"""
This file is part of Automunge which is released under the BSD 3-Clause license.
See file LICENSE or go to https://github.com/Automunge/AutoMunge for full license details.

contact available via automunge.com

Copyright (C) 2018, 2019, 2020, 2021, 2022, 2023 Nicholas Teague - All Rights Reserved

patent pending
"""

#global imports
import numpy as np
import pandas as pd

#this mutes the pandas defragmented dataframe warning
#which is not a bug it is a performance indication
#we are muting until pandas can roll out a fix
from warnings import simplefilter
simplefilter(action="ignore", category=pd.errors.PerformanceWarning)

#imports for automunge
import random
import types
#dt used for time stamp, other datetime methods built on pandas
import datetime as dt

#imports for _evalcategory, _getNArows
# from scipy.stats import shapiro
# from scipy.stats import skew

#imports for _predictinfill, _predictpostinfill, _trainFSmodel, _shuffleaccuracy
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
# from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
#stats may be used for cases where user elects RandomSearchCV hyperparameter tuning
from scipy import stats

#imports for PCA dimensionality reduction applied with application
# from sklearn.decomposition import PCA
# from sklearn.decomposition import SparsePCA
# from sklearn.decomposition import KernelPCA

#imports for DP family of transforms e.g. _process_DP** (DP transforms use np.random)
# import numpy as np

#imports for _process_bxcx 
# from scipy.stats import boxcox

#imports for _process_qttf
# from sklearn.preprocessing import QuantileTransformer

#imports for process_hldy 
# from pandas.tseries.holiday import USFederalHolidayCalendar

#we also have imports for auto ML options in the support functions with their application
#(this allows us to include the option without having their library install as a dependancy)
# import sys
# from flaml import AutoML
# from catboost import CatBoostClassifier
# from catboost import CatBoostRegressor
# import tensorflow as tf
# from xgboost import XGBClassifier
# from xgboost import XGBRegressor
# import optuna

class AutoMunge:
  """
  The AutoMunge class defined functions are broadly organized as:
  #01 - automunge(.) related Function Blocks
  #02 - postmunge(.) related Function Blocks
  #03 - inversion related Function Blocks
  #04 - experimental model training Function Blocks

  Under each of these headers, we've aggregated function definitions by theme
  Where each of these "Function Blocks" can be navigated to by a control F search

  Listed here are all of these Function Blocks with names of included functions

  #________________________________________________
  #01 - automunge(.) related Function Blocks
  #________________________________________________

  #__FunctionBlock: FamilyTrees (transform_dict)
  __assembletransformdict

  #__FunctionBlock: category definitions (process_dict)
  __assembleprocessdict

  #__FunctionBlock: automunge process family functions
  __processfamily
  __circleoflife
  __dictupdate
  __populate_columnkey_dict
  __processcousin
  __processparent
  __grab_params
  __custom_process_wrapper
  __check_for_protected_features
  __messy_data_prep
  __apply_processfamily
  __prepare_labelsencoding_dict

  #__FunctionBlock: suffix overlap and defaultinfill support functions
  __apply_defaultinfill
  __set_indexcolumn
  __set_Binary_column
  __set_PCA_column
  __df_copy_train
  __df_check_suffixoverlap
  __suffix_overlap_final_aggregation_and_printouts

  #__FunctionBlock: automunge dualprocess, singleprocess, and custom_train functions
  _process_NArw
  _process_numerical
  _process_dxdt
  _process_dxd2
  _process_shft
  _process_MADn
  _process_mnmx
  _process_mnm3
  _process_mxab
  _process_retn
  _process_mean
  _process_binary
  _custom_train_onht
  _process_smth
  __GPS_parse
  _custom_train_GPS1
  _process_mlti
  _process_cnsl
  _process_lngt
  _process_bnst
  _process_UPCS
  _process_splt
  _process_spl2
  _process_sp19
  _process_sbst
  _process_sbs3
  _process_hash
  _process_hs10
  _process_srch
  _process_src2
  _process_src3
  _process_src4
  _process_aggt
  _process_strn
  _process_strg
  _process_nmrc
  _process_nmr4
  _custom_train_ordl
  _process_maxb
  _process_ucct
  _custom_train_1010
  _process_bshr
  _process_wkdy
  _process_hldy
  _process_wkds
  _process_mnts
  _process_tmzn
  _process_tmsc
  _process_time
  _process_qttf
  _process_bxcx
  _process_log0
  _process_logn
  _process_sqrt
  _process_addd
  _process_sbtr
  _process_mltp
  _process_divd
  _process_rais
  _process_absl
  _process_pwrs
  _process_pwor
  _process_bins
  _process_bsor
  _process_bnwd
  _process_bnwo
  _process_bnep
  _process_bneo
  _process_tlbn
  _process_bkt1
  _process_bkt2
  _process_bkt3
  _process_bkt4
  _process_DPnb
  _process_DPmm
  _process_DPrt
  _process_DPbn
  _process_DPod
  _process_DPmc
  _process_DPsk
  _process_qbt1
  _process_null
  _process_copy
  _process_excl
  _process_exc2
  _process_exc5
  _process_shfl
  _custom_train_trig

  #__FunctionBlock: evalcategory and NArows
  __evalcategory
  __getNArows
  __assemble_excluded_from_postmunge_getNArows

  #__FunctionBlock: parsing support functions
  __parsedate
  __is_number
  __is_number_comma
  __is_number_EU
  __parsenumeric

  #__FunctionBlock: ML infill master functions
  __MLinfillfunction
  __predictinfill
  __createMLinfillsets
  __insertinfill
  __inspect_ML_cmnd
  __assemble_param_sets
  __optuna_XG1

  #__FunctionBlock: halting criteria support functions
  __consolidate_multicolumn
  __populate_halt_dict
  __calc_stop_result

  #__FunctionBlock: ML infill leakage support
  __check_for_leakage
  __append_full_exclude
  __convert_leakage_dict
  __convert_leakage_sets

  #__FunctionBlock: ML infill stochastic impute support
  __stochastic_impute
  __stochastic_impute_categoric
  __stochastic_impute_numeric

  #__FunctionBlock: ML infill autoML training, inference, and support
  __assemble_autoMLer
  __autoMLer_cleanup
  __populateMLinfilldefaults
  __initRandomForestClassifier
  __initRandomForestRegressor
  _train_randomforest_classifier
  _predict_randomforest_classifier
  _train_randomforest_regressor
  _predict_randomforest_regressor
  _train_autogluon_classifier
  _train_autogluon_regressor
  _predict_autogluon_classifier
  _predict_autogluon_regressor
  _train_flaml_classifier
  _predict_flaml_classifier
  _train_flaml_regressor
  _predict_flaml_regressor
  _train_catboost_classifier
  _predict_catboost_classifier
  _train_catboost_regressor
  _predict_catboost_regressor
  _train_xgboost_classifier
  _train_xgboost_regressor
  __train_xgboost
  _predict_xgboost_classifier
  _predict_xgboost_regressor
  __predict_xgboost

  #__FunctionBlock: customML support functions
  _train_customML_classifier
  _train_customML_regressor
  __train_customML
  _predict_customML_classifier
  _predict_customML_regressor
  __predict_customML
  __call_default_function

  #__FunctionBlock: customML default inference functions
  __customML_tensorflow_defaultpredict_classification
  __customML_tensorflow_defaultpredict_regression
  __customML_xgboost_defaultpredict
  __customML_catboost_defaultpredict
  __customML_flaml_defaultpredict
  __customML_randomforest_defaultpredict

  #__FunctionBlock: data translations support
  __convert_onehot_to_singlecolumn
  __sequential_range_integer
  __sequential_range_integer_invert
  __convert_singlecolumn_to_onehot
  __convert_1010_to_onehot
  __convert_onehot_to_1010
  __onehot_support

  #__FunctionBlock: oversampling support
  __LabelSetGenerator
  __LabelFrequencyLevelizer

  #__FunctionBlock: automunge feature importance
  __trainFSmodel
  __createFSsets
  __createFSsets2
  __shuffleaccuracy
  __assemblemadethecut
  __featureselect

  #__FunctionBlock: assigninfill support functions
  __assemblepostprocess_assigninfill
  __assemble_sorted_columns_by_NaN_dict
  __apply_am_infill
  __apply_pm_infill
  __zeroinfillfunction
  __oneinfillfunction
  __negzeroinfillfunction
  __naninfillfunction
  __adjinfillfunction
  __interpinfillfunction
  __train_medianinfillfunction
  __test_medianinfillfunction
  __train_meaninfillfunction
  __test_meaninfillfunction
  __train_modeinfillfunction
  __test_modeinfillfunction
  __train_lcinfillfunction

  #__FunctionBlock: PCA support functions
  __evalPCA
  __populatePCAdefaults
  __initSparsePCA
  __initKernelPCA
  __initPCA
  __boolexcl
  __createPCAsets
  __PCAfunction

  #__FunctionBlock: validation functions
  __check_am_miscparameters
  __check_pm_miscparameters
  __check_df_type
  __check_np_shape
  __check_ML_infill
  __check_ML_infill_2
  __validate_allvalidnumeric
  __check_assigncat
  __check_assigncat2
  __check_assigncat3
  __create_inverse_assigncat
  __check_assigninfill
  __check_transformdict000
  __check_transformdict00
  __check_transformdict0
  __check_transformdict
  __check_transformdict2
  __check_transform_dict_roots
  __check_haltingproblem
  __check_offspring
  __check_assignnan
  __check_assignnan_injections
  __check_ML_cmnd
  __check_sampling_dict
  __check_entropy_seeds
  __check_random_generator
  __check_assignparam
  __check_columnheaders
  __check_processdict
  __check_processdict3
  __check_processdict3_support
  __check_processdict4

  #__FunctionBlock: entropy_seeds and random_generator support
  __initialize_sampling_report_dict
  __populate_sampling_report_dict
  __prepare_seeds
  __random_parameters_params_append
  __populate_randomseed
  __sample_from_parameter_list
  __sample_from_scipy_stats
  __get_nprandom
  __erase_seeds

  #__FunctionBlock: functionpointer support
  __grab_functionpointer_entries_support
  __grab_functionpointer_entries

  #__FunctionBlock: column assignments string conversion
  __assigncat_str_convert
  __assigninfill_str_convert
  __parameter_str_convert
  __assignparam_str_convert
  __assignnan_str_convert
  __assignnan_list_convert
  __convert_floatnan_to_npnan

  #__FunctionBlock: label smoothing support
  __apply_LabelSmoothing
  __postapply_LabelSmoothing

  #__FunctionBlock: Binary categoric consolidations support
  __BinaryConsolidate
  __postBinaryConsolidate
  __Binary_convert
  __postBinary_convert
  __masterBinaryinvert
  __meta_inverseprocess_Binary
  __inverseprocess_Binary

  #__FunctionBlock: data type management
  __floatprecision_transform
  __convert_to_nan
  __assignnan_convert
  __assignnan_inject

  #__FunctionBlock: data set partitioning and shuffling
  __df_split
  __df_split_specified
  __df_shuffle
  __df_shuffle_series
  __dupl_rows_consolidate

  #__FunctionBlock: populate returned reports
  __populate_columntype_report
  __populate_column_map_report

  #__FunctionBlock: various automunge utility functions
  __list_replace
  __list_sorting
  __autowhere
  __autocopy
  __autoprint
  __logger_append
  __column_convert_support
  __orig_headers_support

  #__FunctionBlock: encryption utility functions
  __encrypt_postprocess_dict
  __decrypt_postprocess_dict

  #__FunctionBlock: automunge ppd_append
  __ppd_append_support

  #__FunctionBlock: automunge(.) definition
  automunge

  #________________________________________________
  #02 - postmunge(.) related Function Blocks
  #________________________________________________

  #__FunctionBlock: postmunge process family functions
  __postprocessfamily
  __postcircleoflife
  __postprocesscousin
  __postprocessparent
  __custom_postprocess_wrapper
  __apply_postprocessfamily

  #__FunctionBlock: postmunge postprocess and custom_test functions
  _postprocess_numerical
  _postprocess_MADn
  _postprocess_mnmx
  _postprocess_mnm3
  _postprocess_mxab
  _postprocess_retn
  _postprocess_mean
  _postprocess_binary
  _custom_test_onht
  _postprocess_smth
  _custom_test_GPS1
  _postprocess_mlti
  _postprocess_cnsl
  _postprocess_splt
  _postprocess_spl2
  _postprocess_sp19
  _postprocess_sbst
  _postprocess_sbs3
  _postprocess_hash
  _postprocess_hs10
  _postprocess_srch
  _postprocess_src2
  _postprocess_src3
  _postprocess_src4
  _postprocess_nmr4
  _custom_test_ordl
  _postprocess_maxb
  _postprocess_ucct
  _custom_test_1010
  _postprocess_tmsc
  _postprocess_time
  _postprocess_qttf
  _postprocess_bxcx
  _postprocess_log0
  _postprocess_logn
  _postprocess_sqrt
  _postprocess_addd
  _postprocess_sbtr
  _postprocess_mltp
  _postprocess_divd
  _postprocess_rais
  _postprocess_absl
  _postprocess_pwrs
  _postprocess_pwor
  _postprocess_bins
  _postprocess_bsor
  _postprocess_bnwd
  _postprocess_bnwo
  _postprocess_bnep
  _postprocess_bneo
  _postprocess_tlbn
  _postprocess_bkt1
  _postprocess_bkt2
  _postprocess_bkt3
  _postprocess_bkt4
  _postprocess_DPnb
  _postprocess_DPmm
  _postprocess_DPrt
  _postprocess_DPbn
  _postprocess_DPod
  _postprocess_DPmc
  _postprocess_DPsk
  _postprocess_exc2
  _postprocess_exc5

  #__FunctionBlock: postmunge ML infill
  __createpostMLinfillsets
  __predictpostinfill
  __postMLinfillfunction

  #__FunctionBlock: postmunge PCA
  __postcreatePCAsets
  __postPCAfunction

  #__FunctionBlock: postmunge feature selection
  __postfeatureselect

  #__FunctionBlock: postmunge driftreport
  __PCA_drift_stats
  __Binary_drift_stats
  __prepare_driftreport

  #__FunctionBlock: postmunge(.) definition
  postmunge

  #________________________________________________
  #03 - inversion related Function Blocks
  #________________________________________________

  #__FunctionBlock: data structure assembly support functions
  __populate_categorytree
  __populate_family
  __get_categorylist
  __populate_inverse_categorytree
  __populate_inverse_family
  __populate_inputcolumn_dict
  __populate_labelsencoding_dict_support
  __populate_labelsencoding_dict_support2
  __populate_labelsencoding_dict_support3

  #__FunctionBlock: inversion preprocessing support
  __LS_invert
  __inversion_header_support

  #__FunctionBlock: custom inversion support
  __custom_inverseprocess_wrapper

  #__FunctionBlock: postmunge inversion inverseprocess and custom_inversion functions
  _inverseprocess_nmbr
  _inverseprocess_year
  _inverseprocess_mean
  _inverseprocess_MADn
  _inverseprocess_MAD3
  _inverseprocess_mnmx
  _inverseprocess_mnm3
  _inverseprocess_mxab
  _inverseprocess_retn
  _inverseprocess_shft
  _inverseprocess_qttf
  _inverseprocess_log0
  _inverseprocess_logn
  _inverseprocess_addd
  _inverseprocess_sbtr
  _inverseprocess_mltp
  _inverseprocess_divd
  _inverseprocess_rais
  _inverseprocess_absl
  _inverseprocess_sqrt
  _inverseprocess_bnst
  _inverseprocess_UPCS
  _inverseprocess_excl
  _inverseprocess_pwr2
  _inverseprocess_pwor
  _inverseprocess_bins
  _inverseprocess_bsor
  _inverseprocess_bnwd
  _inverseprocess_bnwo
  _inverseprocess_bnep
  _inverseprocess_bneo
  _inverseprocess_tlbn
  _inverseprocess_bkt1
  _inverseprocess_bkt2
  _inverseprocess_bkt3
  _inverseprocess_bkt4
  _custom_inversion_onht
  _inverseprocess_smth
  _custom_inversion_GPS1
  _inverseprocess_mlti
  _inverseprocess_cnsl
  _custom_inversion_ordl
  _inverseprocess_strg
  _inverseprocess_bnry
  _custom_inversion_1010
  _inverseprocess_splt
  _inverseprocess_spl2
  _inverseprocess_sp19
  _inverseprocess_sbst
  _inverseprocess_sbs3
  _inverseprocess_srch
  _inverseprocess_src2
  _inverseprocess_src3
  _inverseprocess_src4
  _inverseprocess_nmrc
  _inverseprocess_DPmc
  _inverseprocess_qbt1
  _custom_inversion_trig

  #__FunctionBlock: master inversion functions
  __df_inversion
  __df_inversion_meta
  __inversion_parent

  #________________________________________________
  #04 - experimental model training Function Blocks
  #________________________________________________

  #__FunctionBlock: model training Function Blocks
  __check_model_parameters
  automodel

  #__FunctionBlock: model inference Function Blocks
  autoinference
  """

  #________________________________________________
  #01 - automunge(.) related Function Blocks
  #________________________________________________

  #__FunctionBlock: FamilyTrees (transform_dict)

  def __assembletransformdict(self, binstransform, NArw_marker):
    """
    #populates the transform_dict data structure
    #which is the internal library that is subsequently consolidated 
    #with any user passed transformdict
      
    #transform_dict is for purposes of populating
    #for each transformation category's use as a root category
    #a "family tree" set of associated transformation categories
    #which are for purposes of specifying the type and order of transformation functions
    #to be applied when a transformation category is assigned as a root category
      
    #we'll refer to the category key to a family as the "root category"
    #we'll refer to a transformation category entered into 
    #a family tree primitive as a "tree category"

    #a transformation category may serve as both a root category
    #and a tree category

    #each transformation category will have a set of properties assigned
    #in the corresponding process_dict data structure
    #including associated transformation functions, data properties, and etc.

    #a root category may be assigned to a column with the user passed assigncat
    #or when not specified may be determined under automation via _evalcategory

    #when applying transformations
    #the transformation functions associated with a root category
    #will not be applied unless that same category is populated as a tree category

    #the family tree primitives are for purposes of specifying order of transformations
    #as may include generations and branches of derivations
    #as well as for managing column retentions in the returned data
    #(as in some cases intermediate stages of transformations may or may not have desired retention)

    #the family tree primitives can be distinguished by types of 
    #upstream/downstream, supplement/replace, offsping/no offspring

    #___________
    #'parents' :
    #upstream / first generation / replaces column / with offspring

    #'siblings':
    #upstream / first generation / supplements column / with offspring

    #'auntsuncles' :
    #upstream / first generation / replaces column / no offspring

    #'cousins' :
    #upstream / first generation / supplements column / no offspring

    #'children' :
    #downstream parents / offspring generations / replaces column / with offspring

    #'niecesnephews' :
    #downstream siblings / offspring generations / supplements column / with offspring

    #'coworkers' :
    #downstream auntsuncles / offspring generations / replaces column / no offspring

    #'friends' :
    #downstream cousins / offspring generations / supplements column / no offspring
    #___________

    #each of the family tree primitives associated with a root category
    #may have entries of zero, one, or more transformation categories
      
    #when a root category is assigned to a column
    #the upstream primitives are inspected
      
    #when a tree category is found 
    #as an entry to an upstream primitive associated with the root category
    #the transformation functions associated with the tree category are performed

    #if any tree categories are populated in the upstream replacement primitives
    #their inclusion supersedes supplement primitive entries
    #and so the input column to the transformation is not retained in the returned set
    #with the column replacement either achieved by an inplace transformation
    #or subsequent deletion operation

    #when a tree category is found
    #as an entry to an upstream primitive with offspring
    #after the associated transformation function is performed
    #the downstream primitives of the family tree of the tree category is inspected
    #and those downstream primitives are treated as a subsequent generation's upstream primitives
    #where the input column to that subsequent generation is the column returned 
    #from the transformation function associated with the upstream tree category

    #this is an easy point of confusion so as further clarification on this point
    #the downstream primitives associated with a root category
    #will not be inspected when root category is applied
    #unless that root category is also entered as a tree category entry
    #in one of the root category's upstream primitives with offspring
    """

    transform_dict = {}

    #initialize bins based on what was passed through application of automunge(.)
    if binstransform is True:
      bint = 'bint'
    else:
      bint = None
        
    if NArw_marker is True:
      NArw = 'NArw'
    else:
      NArw = None

    #initialize trasnform_dict. Note in a future extension the range of categories
    #is intended to be built out
    transform_dict.update({'nmbr' : {'parents'       : ['nmbr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : [bint]}})
    
    transform_dict.update({'dxdt' : {'parents'       : ['dxdt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'d2dt' : {'parents'       : ['d2dt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['dxdt'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'d3dt' : {'parents'       : ['d3dt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d2dt'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d4dt' : {'parents'       : ['d4dt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d3dt'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d5dt' : {'parents'       : ['d5dt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d4dt'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d6dt' : {'parents'       : ['d6dt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d5dt'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'dxd2' : {'parents'       : ['dxd2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'d2d2' : {'parents'       : ['d2d2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['dxd2'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'d3d2' : {'parents'       : ['d3d2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d2d2'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d4d2' : {'parents'       : ['d4d2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d3d2'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d5d2' : {'parents'       : ['d5d2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d4d2'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'d6d2' : {'parents'       : ['d6d2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['d5d2'],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'nmdx' : {'parents'       : ['nmdx'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['dxdt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'nmd2' : {'parents'       : ['nmd2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['d2dt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'nmd3' : {'parents'       : ['nmd3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['d3dt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'nmd4' : {'parents'       : ['nmd4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['d4dt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'nmd5' : {'parents'       : ['nmd5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['d5dt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'nmd6' : {'parents'       : ['nmd6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['d6dt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'mmdx' : {'parents'       : ['mmdx'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})
    
    transform_dict.update({'mmd2' : {'parents'       : ['mmd2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['mmdx'],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})
    
    transform_dict.update({'mmd3' : {'parents'       : ['mmd3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['mmd2'],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})

    transform_dict.update({'mmd4' : {'parents'       : ['mmd4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['mmd3'],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})

    transform_dict.update({'mmd5' : {'parents'       : ['mmd5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['mmd4'],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})

    transform_dict.update({'mmd6' : {'parents'       : ['mmd6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['mmd5'],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})
    
    transform_dict.update({'dddt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dddt', 'exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ddd2' : {'parents'       : ['ddd2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['dddt'],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ddd3' : {'parents'       : ['ddd3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ddd2'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ddd4' : {'parents'       : ['ddd4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ddd3'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ddd5' : {'parents'       : ['ddd5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ddd4'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ddd6' : {'parents'       : ['ddd6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ddd5'],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'dedt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dedt', 'exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ded2' : {'parents'       : ['ded2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['dedt'],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ded3' : {'parents'       : ['ded3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ded2'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ded4' : {'parents'       : ['ded4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ded3'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ded5' : {'parents'       : ['ded5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ded4'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ded6' : {'parents'       : ['ded6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['ded5'],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'shft' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['shft'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'shf2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['shf2'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'shf3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['shf3'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'shf4' : {'parents'       : ['shf4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
  
    transform_dict.update({'shf5' : {'parents'       : ['shf5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'shf6' : {'parents'       : ['shf6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'shf7' : {'parents'       : ['shf4', 'shf5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})
    
    transform_dict.update({'shf8' : {'parents'       : ['shf4', 'shf5', 'shf6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['retn'],
                                     'friends'       : []}})

    transform_dict.update({'bnry' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnry'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnr2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'onht' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['onht'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'text' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['text'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'txt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['text'],
                                     'cousins'       : [NArw, 'splt'],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'txt3' : {'parents'       : ['txt3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['text'],
                                     'friends'       : []}})

    #mlti primarily intended for use as a downstream tree category
    transform_dict.update({'mlti' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmbr'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    #mlto primarily intended for use as a downstream tree category
    transform_dict.update({'mlto' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'mlt1' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mlt1'],
                                     'cousins'       : [NArw],
                                     'children'      : ['cnsl'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'mlt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mlt2'],
                                     'cousins'       : [NArw],
                                     'children'      : ['cns2'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'mlt3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mlt3'],
                                     'cousins'       : [NArw],
                                     'children'      : ['cns3'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'cnsl' : {'parents'       : ['mlt1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'cns2' : {'parents'       : ['mlt2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['onht'],
                                     'friends'       : []}})

    transform_dict.update({'cns3' : {'parents'       : ['mlt3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'smth' : {'parents'       : ['smt0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['smth'],
                                     'friends'       : []}})
  
    transform_dict.update({'smt0' : {'parents'       : ['smt0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['smth'],
                                     'friends'       : []}})
  
    transform_dict.update({'fsmh' : {'parents'       : ['fsm0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['fsmh'],
                                     'friends'       : []}})
    
    transform_dict.update({'fsm0' : {'parents'       : ['fsm0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['fsmh'],
                                     'friends'       : []}})

    transform_dict.update({'GPS1' : {'parents'       : ['GPS1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlti'],
                                     'friends'       : []}})

    transform_dict.update({'GPS2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['GPS2'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'GPS3' : {'parents'       : ['GPS3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlti'],
                                     'friends'       : []}})

    transform_dict.update({'GPS4' : {'parents'       : ['GPS4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlti'],
                                     'friends'       : []}})

    transform_dict.update({'GPS5' : {'parents'       : ['GPS5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlto'],
                                     'friends'       : []}})

    transform_dict.update({'GPS6' : {'parents'       : ['GPS6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlto', 'mlti'],
                                     'friends'       : []}})

    transform_dict.update({'lngt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lngt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lngm' : {'parents'       : ['lngm'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
  
    transform_dict.update({'lnlg' : {'parents'       : ['lnlg'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['logn'],
                                     'friends'       : []}})

    transform_dict.update({'bnst' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnst'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
  
    transform_dict.update({'bnso' : {'parents'       : ['bnso'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'UPCS' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['UPCS'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'Unht' : {'parents'       : ['Unht'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['onht'],
                                     'friends'       : []}})
  
    transform_dict.update({'Utxt' : {'parents'       : ['Utxt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['text'],
                                     'friends'       : []}})
    
    transform_dict.update({'Utx2' : {'parents'       : ['Utx2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['text'],
                                     'friends'       : ['splt']}})

    transform_dict.update({'Utx3' : {'parents'       : ['Utx3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['txt3'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'Ucct' : {'parents'       : ['Ucct'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ucct', 'ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'Uord' : {'parents'       : ['Uord'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ordl'],
                                     'friends'       : []}})
        
    transform_dict.update({'Uor2' : {'parents'       : ['Uor2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['ord2'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'Uor3' : {'parents'       : ['Uor3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'Uor6' : {'parents'       : ['Uor6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['spl6'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'U101' : {'parents'       : ['U101'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'splt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['splt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'spl2' : {'parents'       : ['spl2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'spl5' : {'parents'       : ['spl5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'spl6' : {'parents'       : ['spl6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['splt'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['ord3']}})
    
    transform_dict.update({'spl7' : {'parents'       : ['spl7'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'spl8' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['spl8'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'spl9' : {'parents'       : ['spl9'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'sp10' : {'parents'       : ['sp10'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp11' : {'parents'       : ['sp11'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['spl5'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp12' : {'parents'       : ['sp12'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['sp11'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp13' : {'parents'       : ['sp13'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['sp10'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp14' : {'parents'       : ['sp14'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['sp13'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp15' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sp15'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'sp16' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sp16'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'sp17' : {'parents'       : ['sp17'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['spl5'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'sp18' : {'parents'       : ['sp18'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : ['sp17'],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'sp19' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sp19'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sp20' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sp20'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sbst' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sbst'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sbs2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sbs2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sbs3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sbs3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sbs4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sbs4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'hash' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hash'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'hsh2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hsh2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hs10' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hs10'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'Uhsh' : {'parents'       : ['Uhsh'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['hash'],
                                     'friends'       : []}})

    transform_dict.update({'Uhs2' : {'parents'       : ['Uhs2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['hsh2'],
                                     'friends'       : []}})
    
    transform_dict.update({'Uh10' : {'parents'       : ['Uh10'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['hs10'],
                                     'friends'       : []}})
    
    transform_dict.update({'srch' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['srch'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'src2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['src2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'src3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['src3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'src4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['src4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'aggt' : {'parents'       : ['aggt'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'strn' : {'parents'       : ['strn'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ord3'],
                                     'friends'       : []}})

    transform_dict.update({'strg' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['strg'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmrc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmrc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmr2' : {'parents'       : ['nmr2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmr3' : {'parents'       : ['nmr3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'nmr4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmr4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmr5' : {'parents'       : ['nmr5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmr6' : {'parents'       : ['nmr6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmr7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmr7'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmr8' : {'parents'       : ['nmr8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmr9' : {'parents'       : ['nmr9'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmcm' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmcm'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmc2' : {'parents'       : ['nmc2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmc3' : {'parents'       : ['nmc3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'nmc4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmc4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmc5' : {'parents'       : ['nmc5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmc6' : {'parents'       : ['nmc6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'nmc7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmc7'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmc8' : {'parents'       : ['nmc8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmc9' : {'parents'       : ['nmc9'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmEU' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmEU'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmE2' : {'parents'       : ['nmE2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmE3' : {'parents'       : ['nmE3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmE4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmE4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmE5' : {'parents'       : ['nmE5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmE6' : {'parents'       : ['nmE6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmE7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nmE7'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'nmE8' : {'parents'       : ['nmE8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nmE9' : {'parents'       : ['nmE9'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['mnmx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ors7' : {'parents'       : ['spl6', 'nmr2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ors5' : {'parents'       : ['spl5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ors6' : {'parents'       : ['spl6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ordl' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
        
    transform_dict.update({'ord2' : {'parents'       : ['ord2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'ord3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'ord5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord5'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'maxb' : {'parents'       : ['or3b'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'or3b' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['or3b'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['maxb'],
                                     'friends'       : []}})
  
    transform_dict.update({'matx' : {'parents'       : ['or3c'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['onht'],
                                     'friends'       : []}})
    
    transform_dict.update({'or3c' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['or3c'],
                                     'cousins'       : [NArw],
                                     'children'      : ['matx'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ma10' : {'parents'       : ['or3d'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or3d' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['or3d'],
                                     'cousins'       : [NArw],
                                     'children'      : ['ma10'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ucct' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ucct'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
        
    transform_dict.update({'ord4' : {'parents'       : ['ord4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'ors2' : {'parents'       : ['spl2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ord3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'or10' : {'parents'       : ['ord4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'or11' : {'parents'       : ['sp11'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'or12' : {'parents'       : ['nmr2'],
                                     'siblings'      : ['sp11'],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'or13' : {'parents'       : ['sp12'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'or14' : {'parents'       : ['nmr2'],
                                     'siblings'      : ['sp12'],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'or15' : {'parents'       : ['or15'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['sp13'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
  
    transform_dict.update({'or16' : {'parents'       : ['or16'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmr2'],
                                     'niecesnephews' : ['sp13'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or17' : {'parents'       : ['or17'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['sp14'],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or18' : {'parents'       : ['or18'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmr2'],
                                     'niecesnephews' : ['sp14'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'or19' : {'parents'       : ['or19'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmc8'],
                                     'niecesnephews' : ['sp13'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or20' : {'parents'       : ['or20'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmc8'],
                                     'niecesnephews' : ['sp14'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or21' : {'parents'       : ['or21'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmc8'],
                                     'niecesnephews' : ['sp17'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'or22' : {'parents'       : ['or22'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmc8'],
                                     'niecesnephews' : ['sp18'],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'or23' : {'parents'       : ['or23'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['nmcm', 'sp19', 'ord3'],
                                     'friends'       : []}})
    
    transform_dict.update({'om10' : {'parents'       : ['ord4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['1010', 'mnmx'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})

    transform_dict.update({'mmor' : {'parents'       : ['ord4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'1010' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['1010'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'10mz' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['10mz'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'null' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['null'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'NArw' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['NArw'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'NAr2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['NAr2'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'NAr3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['NAr3'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'NAr4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['NAr4'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'NAr5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['NAr5'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nbr2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'nbr3' : {'parents'       : ['nbr3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['bint']}})

    transform_dict.update({'nbr4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nbr4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'MADn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['MADn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'MAD2' : {'parents'       : ['MAD2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'MAD3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['MAD3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnmx' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm2' : {'parents'       : ['nmbr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm3' : {'parents'       : ['nmbr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnm3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnm4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx'],
                                     'cousins'       : ['nmbr', NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm6' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnm6'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnm7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx', 'bins'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'mxab' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mxab'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'retn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'rtbn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn', 'bsor'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'rtb2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn', 'bins'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'mean' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mean'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mea2' : {'parents'       : ['nmbr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mean'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mea3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mean', 'bins'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'tmsc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['tmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'time' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['year'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'tmzn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['tmzn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'date' : {'parents'       : ['date'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mnth', 'days', 'hour', 'mint', 'scnd'],
                                     'friends'       : []}})
  
    transform_dict.update({'dat2' : {'parents'       : ['dat2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['bshr', 'wkdy', 'hldy'],
                                     'friends'       : []}})
    
    transform_dict.update({'dat3' : {'parents'       : ['dat3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mnsn', 'mncs', 'dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'],
                                     'friends'       : []}})
    
    transform_dict.update({'dat4' : {'parents'       : ['dat4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc'],
                                     'friends'       : []}})
    
    transform_dict.update({'dat5' : {'parents'       : ['dat5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mdsn', 'mdcs', 'dysn', 'dycs', 'hmss', 'hmsc'],
                                     'friends'       : []}})
    
    transform_dict.update({'dat6' : {'parents'       : ['dat6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'],
                                     'friends'       : []}})
    
    transform_dict.update({'year' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['year'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'yea2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['year', 'yrsn', 'yrcs', 'mdsn', 'mdcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'yrcs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['yrcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'yrsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['yrsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnth' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnth'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'mnt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnsn', 'mncs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnt3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnsn', 'mncs', 'dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnt4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mdsn', 'mdcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnt5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mdsn', 'mdcs', 'hmss', 'hmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnt6' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mdsn', 'mdcs', 'dysn', 'dycs', 'hmss', 'hmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mnsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mncs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mncs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mdsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mdsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mdcs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mdcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'days' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['days'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'day2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dysn', 'dycs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'day3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'day4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dhms', 'dhmc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'day5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dhms', 'dhmc', 'hmss', 'hmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'dysn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dysn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'dycs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dycs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'dhms' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dhms'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'dhmc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['dhmc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hour' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hour'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'hrs2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hrsn', 'hrcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hrs3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hrs4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hmss', 'hmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hrsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hrsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hrcs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hrcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hmss' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hmss'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hmsc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hmsc'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mint' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mint'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'min2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['misn', 'mics'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'min3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['misn', 'mics', 'scsn', 'sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'min4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mssn', 'mscs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'misn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['misn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mics' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mics'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mssn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mssn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mscs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mscs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'scnd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['scnd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'scn2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['scsn', 'sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'scsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['scsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'sccs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sccs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'qttf' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qttf'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'qtt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qtt2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPqt' : {'parents'       : ['qtt1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'qtt1' : {'parents'       : ['qtt1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPqt'],
                                     'friends'       : []}})
    
    transform_dict.update({'bxcx' : {'parents'       : ['bxcx'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bxc2' : {'parents'       : ['bxc2'],
                                     'siblings'      : ['nmbr'],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bxc3' : {'parents'       : ['bxc3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['nmbr'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bxc4' : {'parents'       : ['bxc4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['nbr2'],
                                     'friends'       : []}})

    transform_dict.update({'bxc5' : {'parents'       : ['bxc5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mnmx'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['nbr2', 'bins'],
                                     'friends'       : []}})

    transform_dict.update({'DPbx' : {'parents'       : ['bxc6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'bxc6' : {'parents'       : ['bxc6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['bxc7'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bxc7' : {'parents'       : ['bxc7'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPbx'],
                                     'friends'       : []}})

    transform_dict.update({'ntgr' : {'parents'       : ['ntgr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn', '1010', 'ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'ntg2' : {'parents'       : ['ntg2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn', '1010', 'ordl', 'pwr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'ntg3' : {'parents'       : ['ntg3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['retn', 'ordl', 'por2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mnmx'],
                                     'friends'       : []}})
    
    transform_dict.update({'pwrs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['pwrs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'pwr2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['pwr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'log0' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['log0'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'log1' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['log0', 'pwr2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'logn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['logn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'lgnm' : {'parents'       : ['lgnm'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['nmbr'],
                                     'friends'       : []}})
    
    transform_dict.update({'sqrt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sqrt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'addd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['addd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'sbtr' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sbtr'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'mltp' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['mltp'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'divd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['divd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'rais' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['rais'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'absl' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['absl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bkt1' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bkt1'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bkt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bkt2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bkt3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bkt3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bkt4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bkt4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'wkdy' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['wkdy'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bshr' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bshr'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'hldy' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['hldy'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'wkds' : {'parents'       : ['wkds'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['text'],
                                     'friends'       : []}})
  
    transform_dict.update({'wkdo' : {'parents'       : ['wkdo'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ordl'],
                                     'friends'       : []}})
    
    transform_dict.update({'mnts' : {'parents'       : ['mnts'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['text'],
                                     'friends'       : []}})
  
    transform_dict.update({'mnto' : {'parents'       : ['mnto'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['ordl'],
                                     'friends'       : []}})
    
    transform_dict.update({'bins' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bins'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'bint' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bint'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bsor' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bsor'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'btor' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['btor'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnwd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnwd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnwK' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnwK'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'bnwM' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnwM'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnwo' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnwo'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'bnKo' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnKo'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnMo' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnMo'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})    
    
    transform_dict.update({'bnep' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnep'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bne7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bne7'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bne9' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bne9'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bneo' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bneo'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bn7o' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bn7o'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'bn9o' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bn9o'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'tlbn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['tlbn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'pwor' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['pwor'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'por2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['por2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'por3' : {'parents'       : ['por3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'bkb3' : {'parents'       : ['bkb3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
  
    transform_dict.update({'bkb4' : {'parents'       : ['bkb4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'bsbn' : {'parents'       : ['bsbn'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'bnwb' : {'parents'       : ['bnwb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'bnKb' : {'parents'       : ['bnKb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'bnMb' : {'parents'       : ['bnMb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'bneb' : {'parents'       : ['bneb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'bn7b' : {'parents'       : ['bn7b'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'bn9b' : {'parents'       : ['bn9b'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})
    
    transform_dict.update({'pwbn' : {'parents'       : ['pwbn'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['1010'],
                                     'friends'       : []}})

    transform_dict.update({'DTnb' : {'parents'       : ['DTn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTn3 primarily intended for use as a tree category
    transform_dict.update({'DTn3' : {'parents'       : ['DTn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTnb'],
                                     'friends'       : []}})

    transform_dict.update({'DTmm' : {'parents'       : ['DTm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTm2 primarily intended for use as a tree category
    transform_dict.update({'DTm2' : {'parents'       : ['DTm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTmm'],
                                     'friends'       : []}})

    transform_dict.update({'DTrt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DTrt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DTbn' : {'parents'       : ['DTb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTb2 primarily intended for use as a tree category
    transform_dict.update({'DTb2' : {'parents'       : ['DTb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTbn'],
                                     'friends'       : []}})
    
    transform_dict.update({'DTod' : {'parents'       : ['DTo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo4 primarily intended for use as a tree category
    transform_dict.update({'DTo4' : {'parents'       : ['DTo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTod'],
                                     'friends'       : []}})

    transform_dict.update({'DToh' : {'parents'       : ['DTo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo5 primarily intended for use as a tree category
    transform_dict.update({'DTo5' : {'parents'       : ['DTo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DToh'],
                                     'friends'       : []}})

    transform_dict.update({'DT10' : {'parents'       : ['DTo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo6 primarily intended for use as a tree category
    transform_dict.update({'DTo6' : {'parents'       : ['DTo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DT10'],
                                     'friends'       : []}})

    transform_dict.update({'DTh1' : {'parents'       : ['DTh1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTmc'],
                                     'friends'       : []}})

    #DTmc primarily intended for use as a downstream tree category
    transform_dict.update({'DTmc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DTmp' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DTmp'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo7 primarily intended for use as a downstream tree category
    transform_dict.update({'DTo7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DThs' : {'parents'       : ['DThs'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mThs'],
                                     'friends'       : []}})
  
    transform_dict.update({'DTh2' : {'parents'       : ['DTh2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTo7'],
                                     'friends'       : []}})

    transform_dict.update({'DTns' : {'parents'       : ['DTn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    #DTn4 primarily intended for use as a tree category
    transform_dict.update({'DTn4' : {'parents'       : ['DTn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DTns'],
                                     'friends'       : []}})

    transform_dict.update({'DT1s' : {'parents'       : ['DTo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo8 primarily intended for use as a tree category
    transform_dict.update({'DTo8' : {'parents'       : ['DTo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DT1s'],
                                     'friends'       : []}})

    transform_dict.update({'DTsk' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DTsk'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DTne' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DTne'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DTse' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DTse'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #mThs primarily intended for use as a downstream tree category
    transform_dict.update({'mThs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DBnb' : {'parents'       : ['DBn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTn3 primarily intended for use as a tree category
    transform_dict.update({'DBn3' : {'parents'       : ['DBn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBnb'],
                                     'friends'       : []}})

    transform_dict.update({'DBmm' : {'parents'       : ['DBm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTm2 primarily intended for use as a tree category
    transform_dict.update({'DBm2' : {'parents'       : ['DBm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBmm'],
                                     'friends'       : []}})

    transform_dict.update({'DBrt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DBrt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DBbn' : {'parents'       : ['DBb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTb2 primarily intended for use as a tree category
    transform_dict.update({'DBb2' : {'parents'       : ['DBb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBbn'],
                                     'friends'       : []}})
    
    transform_dict.update({'DBod' : {'parents'       : ['DBo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo4 primarily intended for use as a tree category
    transform_dict.update({'DBo4' : {'parents'       : ['DBo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBod'],
                                     'friends'       : []}})

    transform_dict.update({'DBoh' : {'parents'       : ['DBo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo5 primarily intended for use as a tree category
    transform_dict.update({'DBo5' : {'parents'       : ['DBo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBoh'],
                                     'friends'       : []}})

    transform_dict.update({'DB10' : {'parents'       : ['DBo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo6 primarily intended for use as a tree category
    transform_dict.update({'DBo6' : {'parents'       : ['DBo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DB10'],
                                     'friends'       : []}})

    transform_dict.update({'DBh1' : {'parents'       : ['DBh1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBmc'],
                                     'friends'       : []}})

    #DTmc primarily intended for use as a downstream tree category
    transform_dict.update({'DBmc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DBmp' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DBmp'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo7 primarily intended for use as a downstream tree category
    transform_dict.update({'DBo7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DBhs' : {'parents'       : ['DBhs'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mBhs'],
                                     'friends'       : []}})
  
    transform_dict.update({'DBh2' : {'parents'       : ['DBh2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBo7'],
                                     'friends'       : []}})

    transform_dict.update({'DBns' : {'parents'       : ['DBn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    #DTn4 primarily intended for use as a tree category
    transform_dict.update({'DBn4' : {'parents'       : ['DBn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DBns'],
                                     'friends'       : []}})

    transform_dict.update({'DB1s' : {'parents'       : ['DBo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DTo8 primarily intended for use as a tree category
    transform_dict.update({'DBo8' : {'parents'       : ['DBo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DB1s'],
                                     'friends'       : []}})

    transform_dict.update({'DBsk' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DBsk'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DBne' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DBne'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DBse' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DBse'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #mThs primarily intended for use as a downstream tree category
    transform_dict.update({'mBhs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPnb' : {'parents'       : ['DPn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPn3 primarily intended for use as a tree category
    transform_dict.update({'DPn3' : {'parents'       : ['DPn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPnb'],
                                     'friends'       : []}})

    transform_dict.update({'DPmm' : {'parents'       : ['DPm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPm2 primarily intended for use as a tree category
    transform_dict.update({'DPm2' : {'parents'       : ['DPm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPmm'],
                                     'friends'       : []}})

    transform_dict.update({'DPrt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPrt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DLnb' : {'parents'       : ['DLn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DLn3 primarily intended for use as a tree category
    transform_dict.update({'DLn3' : {'parents'       : ['DLn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DLnb'],
                                     'friends'       : []}})

    transform_dict.update({'DLmm' : {'parents'       : ['DLm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    #DLm2 primarily intended for use as a tree category
    transform_dict.update({'DLm2' : {'parents'       : ['DLm2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DLmm'],
                                     'friends'       : []}})

    transform_dict.update({'DLrt' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DLrt'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DPbn' : {'parents'       : ['DPb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPb2 primarily intended for use as a tree category
    transform_dict.update({'DPb2' : {'parents'       : ['DPb2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPbn'],
                                     'friends'       : []}})
    
    transform_dict.update({'DPod' : {'parents'       : ['DPo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPo4 primarily intended for use as a tree category
    transform_dict.update({'DPo4' : {'parents'       : ['DPo4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPod'],
                                     'friends'       : []}})

    transform_dict.update({'DPoh' : {'parents'       : ['DPo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPo5 primarily intended for use as a tree category
    transform_dict.update({'DPo5' : {'parents'       : ['DPo5'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPoh'],
                                     'friends'       : []}})

    transform_dict.update({'DP10' : {'parents'       : ['DPo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPo6 primarily intended for use as a tree category
    transform_dict.update({'DPo6' : {'parents'       : ['DPo6'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DP10'],
                                     'friends'       : []}})

    transform_dict.update({'DPh1' : {'parents'       : ['DPh1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPmc'],
                                     'friends'       : []}})

    #DPmc primarily intended for use as a downstream tree category
    transform_dict.update({'DPmc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPmp' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPmp'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPo7 primarily intended for use as a downstream tree category
    transform_dict.update({'DPo7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPhs' : {'parents'       : ['DPhs'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['mlhs'],
                                     'friends'       : []}})
  
    transform_dict.update({'DPh2' : {'parents'       : ['DPh2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPo7'],
                                     'friends'       : []}})

    transform_dict.update({'DPns' : {'parents'       : ['DPn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    #DPn4 primarily intended for use as a tree category
    transform_dict.update({'DPn4' : {'parents'       : ['DPn4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DPns'],
                                     'friends'       : []}})

    transform_dict.update({'DP1s' : {'parents'       : ['DPo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #DPo8 primarily intended for use as a tree category
    transform_dict.update({'DPo8' : {'parents'       : ['DPo8'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['DP1s'],
                                     'friends'       : []}})

    transform_dict.update({'DPsk' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPsk'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPne' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPne'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'DPse' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPse'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'DPpc' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['DPpc'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    #mlhs primarily intended for use as a downstream tree category
    transform_dict.update({'mlhs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordl'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'qbt1' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qbt1'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'qbt2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qbt2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'qbt3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qbt3'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'qbt4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qbt4'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'nmqb' : {'parents'       : ['nmqb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['qbt1'],
                                     'friends'       : []}})
  
    transform_dict.update({'nmq2' : {'parents'       : ['nmq2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['qbt1']}})
  
    transform_dict.update({'mmqb' : {'parents'       : ['mmqb'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['qbt3'],
                                     'friends'       : []}})
    
    transform_dict.update({'mmq2' : {'parents'       : ['mmq2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['qbt3']}})
    
    transform_dict.update({'copy' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['copy'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'excl' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['excl'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #for exc2 without NArw use exc6
    transform_dict.update({'exc2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    #exc3 is like exc2 with downstream standard deviation bins
    transform_dict.update({'exc3' : {'parents'       : ['exc3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['bins']}})
    
    #exc4 is like exc2 with downstream power of ten bins
    transform_dict.update({'exc4' : {'parents'       : ['exc4'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : ['pwr2']}})
    
    #exc5 is passthrough integer categoric, for use without NArw use exc7
    transform_dict.update({'exc5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc5'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'exc6' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc2'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'exc7' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc7'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    #exc8 is passthrough integer continuous, for use without NArw use exc9
    transform_dict.update({'exc8' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc8'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'exc9' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['exc9'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'shfl' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['shfl'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sint' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sint'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'cost' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['cost'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'tant' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['tant'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'arsn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['arsn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'arcs' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['arcs'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'artn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['artn'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'nmbd' : {'parents'       : ['nmbr'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : [bint]}})

    transform_dict.update({'101d' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['101d'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'ordd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['ordd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'texd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['texd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'bnrd' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['bnrd'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'datd' : {'parents'       : ['datd'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'],
                                     'friends'       : []}})
    
    transform_dict.update({'nuld' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['nuld'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'lbnm' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lbnm'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lbnb' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lbnb'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lb10' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lb10'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'lbor' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lbor'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lbos' : {'parents'       : ['lbos'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['strg'],
                                     'friends'       : []}})
    
    transform_dict.update({'lbte' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lbte'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'lbbn' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['lbbn'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lbsm' : {'parents'       : ['smt0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['lbsm'],
                                     'friends'       : []}})
  
    transform_dict.update({'lbfs' : {'parents'       : ['fsm0'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['lbfs'],
                                     'friends'       : []}})
    
    transform_dict.update({'lbda' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'],
                                     'cousins'       : [],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'lgnr' : {'parents'       : ['lgnr', 'sgn3'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : ['lgn2'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
  
    transform_dict.update({'lgn2' : {'parents'       : ['lgn2'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['qbt5'],
                                     'friends'       : []}})
    
    transform_dict.update({'sgn1' : {'parents'       : ['sgn1'],
                                     'siblings'      : [],
                                     'auntsuncles'   : [],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : ['sgn2'],
                                     'friends'       : []}})
    
    transform_dict.update({'qbt5' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['qbt5'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})
    
    transform_dict.update({'sgn2' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sgn2'],
                                     'cousins'       : [NArw],
                                     'children'      : [],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sgn3' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sgn3'],
                                     'cousins'       : [NArw],
                                     'children'      : ['sgn4'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    transform_dict.update({'sgn4' : {'parents'       : [],
                                     'siblings'      : [],
                                     'auntsuncles'   : ['sgn4'],
                                     'cousins'       : [NArw],
                                     'children'      : ['sgn1'],
                                     'niecesnephews' : [],
                                     'coworkers'     : [],
                                     'friends'       : []}})

    return transform_dict

  #__FunctionBlock: category definitions (process_dict)
  
  def __assembleprocessdict(self):
    '''
    #creates a dictionary storing all of the processing functions for each
    #category. Note that the convention is that every dualprocess entry 
    #(to process both train and test data in automunge) is meant
    #to have a corresponding postprocess entry (to process the test set in 
    #postmunge). If the dualprocess/postprocess pair aren't included a 
    #singleprocess function should be instead which processes a single column
    #at a time and is neutral to whether that set is from train or test data.
      
    #note that the functionpointer entry is currently only available for user passed processdict
    #this internal library process_dict does not accept functionpointer entries
      
    #A user should pass either a pair of processing functions to both 
    #dualprocess and postprocess, or alternatively just a single processing
    #function to singleprocess, and omit or pass None to those not used.
    #A user can also pass an inversion function to inverseprocess if available.
    #Most of the transforms defined internal to the library follow this convention.

    #dualprocess: for passing a processing function in which normalization 
    #             parameters are derived from properties of the training set
    #             and jointly process the train set and if available corresponding test set

    #singleprocess: for passing a processing function in which no normalization
    #               parameters are needed from the train set to process the
    #               test set, such that train and test sets processed separately

    #postprocess: for passing a processing function in which normalization 
    #             parameters originally derived from the train set are applied
    #             to separately process a corresponding test set
    #             An entry should correspond to the dualprocess entry.

    #inverseprocess: for passing a processing function used to invert
    #                a corresponding forward pass transform
    #                An entry should correspond to the dualprocess or singleprocess entry.

    #__________________________________________________________________________
    #Alternative streamlined processing function conventions are also available 
    #which may be populated as entries to custom_train / custom_test / custom_inversion.
    #These conventions are documented in the readme section "Custom Transformation Functions".
    #In cases of redundancy custom_train entry specifications take precedence 
    #over dualprocess/singleprocess/postprocess entries.

    #custom_train: for passing a train set processing function in which normalization parameters
    #              are derived from properties of the training set. Will be used to process both 
    #              train and test data when custom_test not provided (in which case similar to singleprocess convention).

    #custom_test: for passing a test set processing function in which normalization parameters
    #             that were derived from properties of the training set are used to process the test data.
    #             When omitted custom_train will be used to process both the train and test data.
    #             An entry should correspond to the custom_train entry.

    #custom_inversion: for passing a processing function used to invert
    #                  a corresponding forward pass transform
    #                  An entry should correspond to the custom_train entry.

    #___________________________________________________________________________
    #The processdict also specifies various properties associated with the transformations. 
    #At a minimum, a user needs to specify NArowtype and MLinfilltype or otherwise
    #include a functionpointer entry.

    #___________________________________________________________________________
    #NArowtype: classifies the type of entries that are targets for infill.
    #           can be entries of {'numeric', 'integer', 'justNaN', 'exclude', 
    #                              'positivenumeric', 'nonnegativenumeric', 
    #                              'nonzeronumeric', 'parsenumeric', 'datetime'}
    #           Note that in the custom_train convention this is used to apply data type casting prior to the transform.
    # - 'numeric' for source columns with expected numeric entries
    # - 'integer' for source columns with expected integer entries
    # - 'justNaN' for source columns that may have expected entries other than numeric
    # - 'binary' similar to justNaN but only the top two most frequent entries are considered valid
    # - 'exclude' for source columns that aren't needing NArow columns derived
    # - 'totalexclude' for source columns that aren't needing NArow columns derived, 
    #                  also excluded from assignnan global option and nan conversions for missing data
    # - 'positivenumeric' for source columns with expected positive numeric entries
    # - 'nonnegativenumeric' for source columns with expected non-negative numeric (zero allowed)
    # - 'nonzeronumeric' for source columns with allowed positive and negative but no zero
    # - 'parsenumeric' marks for infill strings that don't contain any numeric characters
    # - 'datetime' marks for infill cells that aren't recognized as datetime objects

    # ** Note that NArowtype also is used as basis for metrics evaluated in drift assessment of source columns
    # ** Note that by default any np.inf values are converted to NaN for infill
    # ** Note that by default python None entries are treated as targets for infill

    #___________________________________________________________________________
    #MLinfilltype: classifies data types of the returned set, 
    #              as may determine what types of models are trained for ML infill
    #              can be entries {'numeric', 'singlct', 'binary', 'multirt', 'concurrent_act', 'concurrent_nmbr', 
    #                              '1010', 'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}
    #              'numeric' single columns with numeric entries for regression (signed floats)
    #              'singlct' for single column sets with ordinal entries (nonnegative integer classification)
    #              'integer' for single column sets with integer entries (signed integer regression)
    #              'binary'  single column sets with boolean entries (0/1)
    #              'multirt' categoric multicolumn sets with boolean entries (0/1), up to one activation per row
    #              '1010'    for multicolumn sets with binary encoding via 1010, boolean integer entries (0/1), 
    #                        with distinct encoding representations by the set of activations
    #              'concurrent_act' for multicolumn sets with boolean integer entries as may have 
    #                               multiple entries in the same row, different from 1010 
    #                               in that columns are independent
    #              'concurrent_ordl' for multicolumn sets with ordinal encoded entries (nonnegative integer classification)
    #              'concurrent_nmbr' for multicolumn sets with numeric entries (signed floats)
    #              'exclude' for columns which will be excluded from infill, included in other features' ML infill bases
    #                        returned data should be numerically encoded
    #              'boolexclude' boolean integer set suitable for Binary transform but excluded from all infill 
    #                            (e.g. NArw entries), included in other features' ML infill bases
    #              'ordlexclude' ordinal set excluded from infill (note that in some cases in library 
    #                            ordlexclude may return a multi-column set), included in other features' ML infill bases
    #              'totalexclude' for complete passthroughs (excl) without datatype conversions, infill, 
    #                             excluded from other features' ML infill bases

    #___________________________________________________________________________
    #Other optional entries for processdict include:
    #info_retention, inplace_option, defaultparams, labelctgy, 
    #defaultinfill, dtype_convert, functionpointer, and noise_transform.

    #___________________________________________________________________________
    #info_retention: boolean marker associated with an inversion operation that helps inversion prioritize
    #transformation paths with full information recovery. (May pass as True when there is no information loss.)

    #___________________________________________________________________________
    #inplace_option: boolean marker indicating whether a transform supports the inplace parameter received in params.
    #                When not specified this is assumed as True (which is always valid for the custom_train convention).
    #                In other words, in dualprocess/singleprocess convention, if your transform does not support inplace,
    #                need to specify inplace_option as False

    #___________________________________________________________________________
    #defaultparams: a dictionary recording any default assignparam assignments associated with the category. 
    #               Note that deviations in user specifications to assignparam as part of an automunge(.) call
    #               take precedence over defaultparams. Note that when applying functionpointer defaultparams
    #               from the pointer target are also populated when not previously specified.

    #___________________________________________________________________________
    #defaultinfill: this option serves to specify a default infill
    #               applied after NArowtype data type casting and preceding the transformation function.
    #               (defaultinfill is a precursor to ML infill or other infills applied based on assigninfill)
    #               defaults to 'adjinfill' when not specified, can also pass as one of
    #               {'adjinfill', 'meaninfill', 'medianinfill', 'modeinfill', 'lcinfill', 
    #                'interpinfill', 'zeroinfill', 'oneinfill', 'naninfill', 'negzeroinfill'}
    #               Note that 'meaninfill' and 'medianinfill' only work with numeric data (based on NArowtype).
    #               Note that for 'datetime' NArowtype, defaultinfill only supports 'adjinfill' or 'naninfill'
    #               Note that 'naninfill' is intended for cases where user wishes to apply their own default infill 
    #               as part of a custom_train entry

    #___________________________________________________________________________
    #dtype_convert: this option is intended for the custom_train convention, accepts boolean entries,
    #               defaults to True when not specified, False turns off a data type conversion
    #               that is applied after custom_train transformation functions based on MLinfilltype.
    #               May also be used to deactivate a floatprecision conversion for any category. 
    #               This option primarily included to support special cases and not intended for wide use.

    #___________________________________________________________________________
    #labelctgy: an optional entry, should be a string entry of a single transformation category 
    #           as entered in the family tree when the category of the processdict entry is used as a root category. 
    #           Used to determine a basis of feature selection for cases where root 
    #           category is applied to a label set resulting in a set returned in multiple configurations. 
    #           Also used in label frequency levelizer. 
    #           Note that since this is only used for small edge case populating a labelctgy entry is optional. 
    #           If one is not assigned, an arbitrary entry will be accessed from the family tree. 
    #           This option primarily included to support special cases.

    #___________________________________________________________________________
    #functionpointer: A functionpointer entry 
    #                 may be entered in lieu of any or all of these other entries **.
    #                 The functionpointer should be populated with a category that has its own processdict entry 
    #                 (or a category that has its own process_dict entry internal to the library)
    #                 The functionpointer inspects the pointer target and passes those specifications 
    #                 to the origin processdict entry unless previously specified.
    #                 The functionpointer is intended as a shortcut for specifying processdict entries
    #                 that may be helpful in cases where a new entry is very similar to some existing entry.
    #                 (**As the exception labelctgy not accessed from functionpointer 
    #                 since it is specific to a root category's family tree.)

    #___________________________________________________________________________
    #noise_transform: this option serves to specify the noise injection types for noise transforms
    #                 used to support an entropy seeding based on sampling_dict['sampling_type'] specification
    #                 defaults to False when not specified, can also pass as one of
    #                 {'numeric', 'categoric', 'binary', False}
    #                 numeric is for transforms similar to DPnb/DPmm/DPrt which have a binomial and distribution sampling
    #                 categoric is for transforms similar to DPod/DPmc which have a binomial and a choice sampling
    #                 binary is for transforms similar to an alternate DPbn configuration which only have a binomial sampling
    #                 False is for transforms without sampling_dict['sampling_type'] specification support

    #___________________________________________________________________________
    #Other clarifications:
    #Note that NArowtype is associated with transformation inputs
    #including for a category's use as a root category and as a tree category
    #MLinfilltype is associated with transformation outputs
    #for a category's use as a tree category
    '''

    process_dict = {}
    
    process_dict.update({'nmbr' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'dxdt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d2dt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d3dt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d4dt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d5dt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d6dt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'dxd2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d2d2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d3d2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d4d2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d5d2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d6d2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmdx' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd2' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd4' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd5' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd6' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'mmdx' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'dddt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'ddd2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'ddd3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'ddd4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'ddd5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'ddd6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxdt,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dddt'}})
    process_dict.update({'dedt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'ded2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'ded3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'ded4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'ded5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'ded6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_dxd2,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dedt'}})
    process_dict.update({'shft' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 1},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'shft'}})
    process_dict.update({'shf2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 2},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'shf2'}})
    process_dict.update({'shf3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 3},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'shf3'}})
    process_dict.update({'shf4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 1},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 2},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'periods' : 3},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf7' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf8' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shft,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_shft,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nbr2' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nbr2'}})
    process_dict.update({'nbr3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nbr3'}})
    process_dict.update({'nbr4' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'zeroinfill',
                                  'defaultparams' : {'abs_zero' : False},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'MADn' : {'dualprocess' : self._process_MADn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_MADn,
                                  'inverseprocess' : self._inverseprocess_MADn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'center' : 'mean'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'MADn'}})
    process_dict.update({'MAD2' : {'dualprocess' : self._process_MADn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_MADn,
                                  'inverseprocess' : self._inverseprocess_MADn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'center' : 'mean'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'MAD2'}})
    process_dict.update({'MAD3' : {'dualprocess' : self._process_MADn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_MADn,
                                  'inverseprocess' : self._inverseprocess_MADn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'center' : 'max'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'MAD3'}})
    process_dict.update({'mnmx' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm3' : {'dualprocess' : self._process_mnm3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnm3,
                                  'inverseprocess' : self._inverseprocess_mnm3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnm3'}})
    process_dict.update({'mnm4' : {'dualprocess' : self._process_mnm3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnm3,
                                  'inverseprocess' : self._inverseprocess_mnm3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnm4'}})
    process_dict.update({'mnm5' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm6' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'defaultparams' : {'floor' : True},
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnm6'}})
    process_dict.update({'mnm7' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mxab' : {'dualprocess' : self._process_mxab,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mxab,
                                  'inverseprocess' : self._inverseprocess_mxab,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mxab'}})
    process_dict.update({'retn' : {'dualprocess' : self._process_retn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_retn,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'rtbn' : {'dualprocess' : self._process_retn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_retn,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'rtb2' : {'dualprocess' : self._process_retn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_retn,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'retn'}})
    process_dict.update({'mean' : {'dualprocess' : self._process_mean,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mean,
                                  'inverseprocess' : self._inverseprocess_mean,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mean'}})
    process_dict.update({'mea2' : {'dualprocess' : self._process_mean,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mean,
                                  'inverseprocess' : self._inverseprocess_mean,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mean'}})
    process_dict.update({'mea3' : {'dualprocess' : self._process_mean,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mean,
                                  'inverseprocess' : self._inverseprocess_mean,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mean'}})
    process_dict.update({'bnry' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'infillconvention' : 'onevalue'},
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'bnry'}})
    process_dict.update({'bnr2' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'infillconvention' : 'zerovalue'},
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'bnr2'}})
    process_dict.update({'onht' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'suffix_convention' : 'onht'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'onht'}})
    process_dict.update({'text' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'text'}})
    process_dict.update({'txt2' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'text'}})
    process_dict.update({'txt3' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'text'}})
    process_dict.update({'mlti' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlti'}})
    process_dict.update({'mlto' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'dtype' : 'conditionalinteger',
                                                     'norm_category' : 'ord3',
                                                     'norm_params' : {'str_convert' : False}},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_ordl',
                                  'labelctgy' : 'mlto'}})
    process_dict.update({'mlt1' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'dtype' : 'conditionalinteger',
                                                     'norm_category' : 'ord3',
                                                     'norm_params' : {'str_convert' : True}},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_ordl',
                                  'labelctgy' : 'mlt1'}})
    process_dict.update({'mlt2' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'dtype' : 'conditionalinteger',
                                                     'norm_category' : 'ord3',
                                                     'norm_params' : {'str_convert' : True}},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_ordl',
                                  'labelctgy' : 'mlt2'}})
    process_dict.update({'mlt3' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'dtype' : 'conditionalinteger',
                                                     'norm_category' : 'ord3',
                                                     'norm_params' : {'str_convert' : True}},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_ordl',
                                  'labelctgy' : 'mlt3'}})
    process_dict.update({'cnsl' : {'dualprocess' : self._process_cnsl,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_cnsl,
                                  'inverseprocess' : self._inverseprocess_cnsl,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'messy_data_headers' : [],
                                                     'pad_character' : '_',
                                                     'upstream_ordinal' : True},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : '1010'}})
    process_dict.update({'cns2' : {'dualprocess' : self._process_cnsl,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_cnsl,
                                  'inverseprocess' : self._inverseprocess_cnsl,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'messy_data_headers' : [],
                                                     'pad_character' : '_',
                                                     'upstream_ordinal' : True},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'onht'}})
    process_dict.update({'cns3' : {'dualprocess' : self._process_cnsl,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_cnsl,
                                  'inverseprocess' : self._inverseprocess_cnsl,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'messy_data_headers' : [],
                                                     'pad_character' : '_',
                                                     'upstream_ordinal' : True},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'smt0' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'smth'}})
    process_dict.update({'smth' : {'dualprocess' : self._process_smth,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_smth,
                                  'inverseprocess' : self._inverseprocess_smth,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'smth'}})
    process_dict.update({'fsm0' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'fsmh'}})
    process_dict.update({'fsmh' : {'dualprocess' : self._process_smth,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_smth,
                                  'inverseprocess' : self._inverseprocess_smth,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'LSfit' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'fsmh'}})
    process_dict.update({'GPS1' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : None,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'default'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlti'}})
    process_dict.update({'GPS2' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : None,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'default'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlti'}})
    process_dict.update({'GPS3' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : None,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'nonunique'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlti'}})
    process_dict.update({'GPS4' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : self._custom_test_GPS1,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'nonunique'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlti'}})
    process_dict.update({'GPS5' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : None,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'nonunique'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlto'}})
    process_dict.update({'GPS6' : {'custom_train' : self._custom_train_GPS1,
                                  'custom_test' : None,
                                  'custom_inversion' : self._custom_inversion_GPS1,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'GPS_convention' : 'nonunique'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'mlto'}})
    process_dict.update({'lngt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_lngt,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'integer',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'lngm' : {'dualprocess' : None,
                                  'singleprocess' : self._process_lngt,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'integer',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'lnlg' : {'dualprocess' : None,
                                  'singleprocess' : self._process_lngt,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'integer',
                                  'labelctgy' : 'log0'}})
    process_dict.update({'bnst' : {'dualprocess' : None,
                                  'singleprocess' : self._process_bnst,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_bnst,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'bnst'}})
    process_dict.update({'bnso' : {'dualprocess' : None,
                                  'singleprocess' : self._process_bnst,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_bnst,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'UPCS' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'UPCS'}})
    process_dict.update({'Unht' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'onht'}})
    process_dict.update({'Utxt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'text'}})
    process_dict.update({'Utx2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'text'}})
    process_dict.update({'Utx3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'text'}})
    process_dict.update({'Ucct' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'Uord' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'Uor2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'Uor3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'Uor6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'U101' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : '1010'}})
    process_dict.update({'splt' : {'dualprocess' : self._process_splt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_splt,
                                  'inverseprocess' : self._inverseprocess_splt,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'splt'}})
    process_dict.update({'spl2' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl5' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl6' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl7' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : True,
                                                     'minsplit' : 1},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl8' : {'dualprocess' : self._process_splt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_splt,
                                  'inverseprocess' : self._inverseprocess_splt,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'spl8'}})
    process_dict.update({'spl9' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True,
                                                     'consolidate_nonoverlaps' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'sp10' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True,
                                                     'consolidate_nonoverlaps' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'sp11' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : False,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp12' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : False,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp13' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : True,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp14' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : True,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp15' : {'dualprocess' : self._process_splt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_splt,
                                  'inverseprocess' : self._inverseprocess_splt,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'concurrent_activations': True,
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'sp15'}})
    process_dict.update({'sp16' : {'dualprocess' : self._process_splt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_splt,
                                  'inverseprocess' : self._inverseprocess_splt,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'concurrent_activations': True,
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'sp16'}})
    process_dict.update({'sp17' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : False,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp18' : {'dualprocess' : self._process_spl2,
                                   'singleprocess' : None,
                                   'postprocess' : self._postprocess_spl2,
                                   'inverseprocess' : self._inverseprocess_spl2,
                                   'info_retention' : False,
                                   'inplace_option' : False,
                                   'defaultinfill' : 'naninfill',
                                   'defaultparams' : {'test_same_as_train' : False,
                                                      'consolidate_nonoverlaps' : False},
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp19' : {'dualprocess' : self._process_sp19,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sp19,
                                  'inverseprocess' : self._inverseprocess_sp19,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'sp19'}})
    process_dict.update({'sp20' : {'dualprocess' : self._process_sp19,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sp19,
                                  'inverseprocess' : self._inverseprocess_sp19,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'sp20'}})
    process_dict.update({'sbst' : {'dualprocess' : self._process_sbst,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sbst,
                                  'inverseprocess' : self._inverseprocess_sbst,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'sbst'}})
    process_dict.update({'sbs2' : {'dualprocess' : self._process_sbst,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sbst,
                                  'inverseprocess' : self._inverseprocess_sbst,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'sbs2'}})
    process_dict.update({'sbs3' : {'dualprocess' : self._process_sbs3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sbs3,
                                  'inverseprocess' : self._inverseprocess_sbs3,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'sbs3'}})
    process_dict.update({'sbs4' : {'dualprocess' : self._process_sbs3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sbs3,
                                  'inverseprocess' : self._inverseprocess_sbs3,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'sbs4'}})
    process_dict.update({'hash' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'hash'}})
    process_dict.update({'hsh2' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'space' : '',
                                                     'excluded_characters' : []},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'hsh2'}})
    process_dict.update({'hs10' : {'dualprocess' : self._process_hs10,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hs10,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'hs10'}})
    process_dict.update({'Uhsh' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'hash'}})
    process_dict.update({'Uhs2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'hsh2'}})
    process_dict.update({'Uh10' : {'dualprocess' : None,
                                  'singleprocess' : self._process_UPCS,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'hs10'}})
    process_dict.update({'srch' : {'dualprocess' : self._process_srch,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_srch,
                                  'inverseprocess' : self._inverseprocess_srch,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'srch'}})
    process_dict.update({'src2' : {'dualprocess' : self._process_src2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_src2,
                                  'inverseprocess' : self._inverseprocess_src2,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'src2'}})
    process_dict.update({'src3' : {'dualprocess' : self._process_src3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_src3,
                                  'inverseprocess' : self._inverseprocess_src3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'concurrent_act',
                                  'labelctgy' : 'src3'}})
    process_dict.update({'src4' : {'dualprocess' : self._process_src4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_src4,
                                  'inverseprocess' : self._inverseprocess_src4,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'src4'}})
    process_dict.update({'aggt' : {'dualprocess' : None,
                                  'singleprocess' : self._process_aggt,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'strn' : {'dualprocess' : None,
                                  'singleprocess' : self._process_strn,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'strg' : {'dualprocess' : None,
                                  'singleprocess' : self._process_strg,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_strg,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'strg'}})
    process_dict.update({'nmrc' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmr2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmr4' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmr4'}})
    process_dict.update({'nmr5' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr6' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmr7' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmr7'}})
    process_dict.update({'nmr8' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr9' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'numbers',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmcm' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmcm'}})
    process_dict.update({'nmc2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmc4' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmc4'}})
    process_dict.update({'nmc5' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc6' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmc7' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmc7'}})
    process_dict.update({'nmc8' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc9' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'commas',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmEU' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmEU'}})
    process_dict.update({'nmE2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_nmrc,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces'},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmE4' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmE4'}})
    process_dict.update({'nmE5' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE6' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : True},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmE7' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmE7'}})
    process_dict.update({'nmE8' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE9' : {'dualprocess' : self._process_nmr4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_nmr4,
                                  'inverseprocess' : self._inverseprocess_nmrc,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'convention' : 'spaces',
                                                     'test_same_as_train' : False},
                                  'NArowtype' : 'parsenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ors7' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ors5' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ors6' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ordl' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'frequency_sort' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'ord2' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'frequency_sort' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ord3' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ord5' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'frequency_sort' : False,
                                                     'null_activation' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ord5'}})
    process_dict.update({'maxb' : {'dualprocess' : self._process_maxb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_maxb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'maxb'}})
    process_dict.update({'or3b' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'maxb'}})
    process_dict.update({'matx' : {'dualprocess' : self._process_maxb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_maxb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'onht'}})
    process_dict.update({'or3c' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'onht'}})
    process_dict.update({'ma10' : {'dualprocess' : self._process_maxb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_maxb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'or3d' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'ucct' : {'dualprocess' : self._process_ucct,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_ucct,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'ucct'}})
    process_dict.update({'ord4' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ors2' : {'dualprocess' : self._process_spl2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_spl2,
                                  'inverseprocess' : self._inverseprocess_spl2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'test_same_as_train' : False,
                                                     'consolidate_nonoverlaps' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'or10' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'or11' : {'custom_train' : self._custom_train_1010,
                                   'custom_test' : self._custom_test_1010,
                                   'custom_inversion' : self._custom_inversion_1010,
                                   'info_retention' : True,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : '1010',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or12' : {'custom_train' : self._custom_train_1010,
                                   'custom_test' : self._custom_test_1010,
                                   'custom_inversion' : self._custom_inversion_1010,
                                   'info_retention' : True,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : '1010',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or13' : {'custom_train' : self._custom_train_1010,
                                   'custom_test' : self._custom_test_1010,
                                   'custom_inversion' : self._custom_inversion_1010,
                                   'info_retention' : True,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : '1010',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or14' : {'custom_train' : self._custom_train_1010,
                                   'custom_test' : self._custom_test_1010,
                                   'custom_inversion' : self._custom_inversion_1010,
                                   'info_retention' : True,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : '1010',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or15' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or16' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or17' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or18' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or19' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or20' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or21' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or22' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or23' : {'dualprocess' : None,
                                   'singleprocess' : self._process_UPCS,
                                   'postprocess' : None,
                                   'inverseprocess' : self._inverseprocess_UPCS,
                                   'info_retention' : False,
                                   'inplace_option' : True,
                                   'defaultinfill' : 'naninfill',
                                   'NArowtype' : 'justNaN',
                                   'MLinfilltype' : 'totalexclude',
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'om10' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mmor' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'1010' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : '1010'}})
    process_dict.update({'10mz' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'max_zero' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : '10mz'}})
    process_dict.update({'qttf' : {'dualprocess' : self._process_qttf,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_qttf,
                                  'inverseprocess' : self._inverseprocess_qttf,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'output_distribution' : 'normal'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qttf'}})
    process_dict.update({'qtt2' : {'dualprocess' : self._process_qttf,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_qttf,
                                  'inverseprocess' : self._inverseprocess_qttf,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'output_distribution' : 'uniform'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qtt2'}})
    process_dict.update({'qtt1' : {'dualprocess' : self._process_qttf,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_qttf,
                                  'inverseprocess' : self._inverseprocess_qttf,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'output_distribution' : 'normal'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPqt'}})
    process_dict.update({'DPqt' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPqt'}})
    process_dict.update({'bxcx' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'tmsc' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'time' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'inverseprocess' : self._inverseprocess_year,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'scale' : 'year',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'time'}})
    process_dict.update({'tmzn' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'tmzn'}})
    process_dict.update({'date' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'year'}})
    process_dict.update({'dat2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'hldy'}})
    process_dict.update({'dat3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'year'}})
    process_dict.update({'dat4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'year'}})
    process_dict.update({'dat5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'year'}})
    process_dict.update({'dat6' : {'dualprocess' : None,
                                  'singleprocess' : self._process_tmzn,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'year'}})
    process_dict.update({'year' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'inverseprocess' : self._inverseprocess_year,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'scale' : 'year',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'year'}})
    process_dict.update({'yea2' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'inverseprocess' : self._inverseprocess_year,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'scale' : 'year',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'year'}})
    process_dict.update({'yrsn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'year',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'yrsn'}})
    process_dict.update({'yrcs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'year',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'yrcs'}})
    process_dict.update({'mnth' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'defaultparams' : {'scale' : 'month',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnth'}})
    process_dict.update({'mnt2' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnsn'}})
    process_dict.update({'mnt3' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnsn'}})
    process_dict.update({'mnt4' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'mnt5' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'mnt6' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'mnsn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'month',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mnsn'}})
    process_dict.update({'mncs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'month',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mncs'}})
    process_dict.update({'mdsn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'monthday',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'mdcs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'monthday',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mdcs'}})
    process_dict.update({'days' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'defaultparams' : {'scale' : 'day',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'days'}})
    process_dict.update({'day2' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dysn'}})
    process_dict.update({'day3' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dysn'}})
    process_dict.update({'day4' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dhms'}})
    process_dict.update({'day5' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dhms'}})
    process_dict.update({'dysn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'day',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dysn'}})
    process_dict.update({'dycs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'day',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dycs'}})
    process_dict.update({'dhms' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'dayhourminute',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dhms'}})
    process_dict.update({'dhmc' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'dayhourminute',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'dhmc'}})
    process_dict.update({'hour' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'defaultparams' : {'scale' : 'hour',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hour'}})
    process_dict.update({'hrs2' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hrsn'}})
    process_dict.update({'hrs3' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hrsn'}})
    process_dict.update({'hrs4' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hmss'}})
    process_dict.update({'hrsn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'hour',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hrsn'}})
    process_dict.update({'hrcs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'hour',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hrcs'}})
    process_dict.update({'hmss' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'hourminutesecond',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hmss'}})
    process_dict.update({'hmsc' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'hourminutesecond',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'hmsc'}})
    process_dict.update({'mint' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'defaultparams' : {'scale' : 'minute',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mint'}})
    process_dict.update({'min2' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'misn'}})
    process_dict.update({'min3' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'misn'}})
    process_dict.update({'min4' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mssn'}})
    process_dict.update({'misn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'minute',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'misn'}})
    process_dict.update({'mics' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'minute',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mics'}})
    process_dict.update({'mssn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'minutesecond',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mssn'}})
    process_dict.update({'mscs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'minutesecond',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mscs'}})
    process_dict.update({'scnd' : {'dualprocess' : self._process_time,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_time,
                                  'defaultparams' : {'scale' : 'second',
                                                     'normalization' : 'zscore'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'scnd'}})
    process_dict.update({'scn2' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'scsn'}})
    process_dict.update({'scsn' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'second',
                                                     'function' : 'sin'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'scsn'}})
    process_dict.update({'sccs' : {'dualprocess' : self._process_tmsc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tmsc,
                                  'defaultparams' : {'scale' : 'second',
                                                     'function' : 'cos'},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'sccs'}})
    process_dict.update({'bxc2' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'nonzeronumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'bxc3' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'bxc4' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nbr2'}})
    process_dict.update({'bxc5' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nbr2'}})
    process_dict.update({'bxc6' : {'dualprocess' : self._process_bxcx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bxcx,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPbx'}})
    process_dict.update({'bxc7' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPbx'}})
    process_dict.update({'DPbx' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPbx'}})
    process_dict.update({'ntgr' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ntg2' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ntg3' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'pwrs' : {'dualprocess' : self._process_pwrs,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwrs,
                                  'inverseprocess' : self._inverseprocess_pwr2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'negvalues' : False},
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'pwrs'}})
    process_dict.update({'pwr2' : {'dualprocess' : self._process_pwrs,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwrs,
                                  'inverseprocess' : self._inverseprocess_pwr2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'negvalues' : True},
                                  'NArowtype' : 'nonzeronumeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'pwr2'}})
    process_dict.update({'log0' : {'dualprocess' : self._process_log0,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_log0,
                                  'inverseprocess' : self._inverseprocess_log0,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'log0'}})
    process_dict.update({'log1' : {'dualprocess' : self._process_log0,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_log0,
                                  'inverseprocess' : self._inverseprocess_log0,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'log0'}})
    process_dict.update({'logn' : {'dualprocess' : self._process_logn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_logn,
                                  'inverseprocess' : self._inverseprocess_logn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'logn'}})
    process_dict.update({'lgnm' : {'dualprocess' : self._process_logn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_logn,
                                  'inverseprocess' : self._inverseprocess_logn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'sqrt' : {'dualprocess' : self._process_sqrt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sqrt,
                                  'inverseprocess' : self._inverseprocess_sqrt,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'nonnegativenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'sqrt'}})
    process_dict.update({'addd' : {'dualprocess' : self._process_addd,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_addd,
                                  'inverseprocess' : self._inverseprocess_addd,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'addd'}})
    process_dict.update({'sbtr' : {'dualprocess' : self._process_sbtr,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_sbtr,
                                  'inverseprocess' : self._inverseprocess_sbtr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'sbtr'}})
    process_dict.update({'mltp' : {'dualprocess' : self._process_mltp,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mltp,
                                  'inverseprocess' : self._inverseprocess_mltp,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'mltp'}})
    process_dict.update({'divd' : {'dualprocess' : self._process_divd,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_divd,
                                  'inverseprocess' : self._inverseprocess_divd,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'divd'}})
    process_dict.update({'rais' : {'dualprocess' : self._process_rais,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_rais,
                                  'inverseprocess' : self._inverseprocess_rais,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'rais'}})
    process_dict.update({'absl' : {'dualprocess' : self._process_absl,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_absl,
                                  'inverseprocess' : self._inverseprocess_absl,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'absl'}})
    process_dict.update({'bkt1' : {'dualprocess' : self._process_bkt1,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt1,
                                  'inverseprocess' : self._inverseprocess_bkt1,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bkt1'}})
    process_dict.update({'bkt2' : {'dualprocess' : self._process_bkt2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt2,
                                  'inverseprocess' : self._inverseprocess_bkt2,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bkt2'}})
    process_dict.update({'bkt3' : {'dualprocess' : self._process_bkt3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt3,
                                  'inverseprocess' : self._inverseprocess_bkt3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bkt3'}})
    process_dict.update({'bkt4' : {'dualprocess' : self._process_bkt4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt4,
                                  'inverseprocess' : self._inverseprocess_bkt4,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bkt4'}})
    process_dict.update({'wkdy' : {'dualprocess' : None,
                                  'singleprocess' : self._process_wkdy,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'wkdy'}})
    process_dict.update({'bshr' : {'dualprocess' : None,
                                  'singleprocess' : self._process_bshr,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'bshr'}})
    process_dict.update({'hldy' : {'dualprocess' : None,
                                  'singleprocess' : self._process_hldy,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'hldy'}})
    process_dict.update({'wkds' : {'dualprocess' : None,
                                  'singleprocess' : self._process_wkds,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'text'}})
    process_dict.update({'wkdo' : {'dualprocess' : None,
                                  'singleprocess' : self._process_wkds,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'mnts' : {'dualprocess' : None,
                                  'singleprocess' : self._process_mnts,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'text'}})
    process_dict.update({'mnto' : {'dualprocess' : None,
                                  'singleprocess' : self._process_mnts,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'bins' : {'dualprocess' : self._process_bins,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bins,
                                  'inverseprocess' : self._inverseprocess_bins,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bins'}})
    process_dict.update({'bint' : {'dualprocess' : self._process_bins,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bins,
                                  'inverseprocess' : self._inverseprocess_bins,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'normalizedinput' : True},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bint'}})
    process_dict.update({'bsor' : {'dualprocess' : self._process_bsor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bsor,
                                  'inverseprocess' : self._inverseprocess_bsor,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bsor'}})
    process_dict.update({'btor' : {'dualprocess' : self._process_bsor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bsor,
                                  'inverseprocess' : self._inverseprocess_bsor,
                                  'info_retention' : False,
                                  'defaultparams' : {'normalizedinput' : True},
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'btor'}})
    process_dict.update({'bnwd' : {'dualprocess' : self._process_bnwd,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwd,
                                  'inverseprocess' : self._inverseprocess_bnwd,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bnwd'}})
    process_dict.update({'bnwK' : {'dualprocess' : self._process_bnwd,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwd,
                                  'inverseprocess' : self._inverseprocess_bnwd,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bnwK'}})
    process_dict.update({'bnwM' : {'dualprocess' : self._process_bnwd,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwd,
                                  'inverseprocess' : self._inverseprocess_bnwd,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000000},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bnwM'}})
    process_dict.update({'bnwo' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bnwo'}})
    process_dict.update({'bnKo' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bnKo'}})
    process_dict.update({'bnMo' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000000},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bnMo'}})
    process_dict.update({'bnep' : {'dualprocess' : self._process_bnep,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnep,
                                  'inverseprocess' : self._inverseprocess_bnep,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bnep'}})
    process_dict.update({'bne7' : {'dualprocess' : self._process_bnep,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnep,
                                  'inverseprocess' : self._inverseprocess_bnep,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'bincount':7},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bne7'}})
    process_dict.update({'bne9' : {'dualprocess' : self._process_bnep,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnep,
                                  'inverseprocess' : self._inverseprocess_bnep,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'bincount':9},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'bne9'}})
    process_dict.update({'bneo' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bneo'}})
    process_dict.update({'bn7o' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'bincount':7},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bn7o'}})
    process_dict.update({'bn9o' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'bincount':9},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'bn9o'}})
    process_dict.update({'tlbn' : {'dualprocess' : self._process_tlbn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_tlbn,
                                  'inverseprocess' : self._inverseprocess_tlbn,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'concurrent_nmbr',
                                  'labelctgy' : 'tlbn'}})
    process_dict.update({'pwor' : {'dualprocess' : self._process_pwor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwor,
                                  'inverseprocess' : self._inverseprocess_pwor,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'negvalues' : False},
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'pwor'}})
    process_dict.update({'por2' : {'dualprocess' : self._process_pwor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwor,
                                  'inverseprocess' : self._inverseprocess_pwor,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'negvalues' : True},
                                  'NArowtype' : 'nonzeronumeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'por2'}})
    process_dict.update({'por3' : {'dualprocess' : self._process_pwor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwor,
                                  'inverseprocess' : self._inverseprocess_pwor,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'negvalues' : True},
                                  'NArowtype' : 'nonzeronumeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bkb3' : {'dualprocess' : self._process_bkt3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt3,
                                  'inverseprocess' : self._inverseprocess_bkt3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bkb4' : {'dualprocess' : self._process_bkt4,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt4,
                                  'inverseprocess' : self._inverseprocess_bkt4,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bsbn' : {'dualprocess' : self._process_bsor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bsor,
                                  'inverseprocess' : self._inverseprocess_bsor,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnwb' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnKb' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnMb' : {'dualprocess' : self._process_bnwo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bnwo,
                                  'inverseprocess' : self._inverseprocess_bnwo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'width':1000000},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bneb' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bn7b' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'bincount':7},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'bn9b' : {'dualprocess' : self._process_bneo,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bneo,
                                  'inverseprocess' : self._inverseprocess_bneo,
                                  'info_retention' : False,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'bincount':9},
                                  'inplace_option' : True,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'pwbn' : {'dualprocess' : self._process_pwor,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_pwor,
                                  'inverseprocess' : self._inverseprocess_pwor,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : '1010'}})
    process_dict.update({'DTn3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTnb'}})
    process_dict.update({'DTnb' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTnb'}})
    process_dict.update({'DTm2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTmm'}})
    process_dict.update({'DTmm' : {'dualprocess' : self._process_DPmm,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmm,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTmm'}})
    process_dict.update({'DTrt' : {'dualprocess' : self._process_DPrt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPrt,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTrt'}})
    process_dict.update({'DTb2' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DTbn'}})
    process_dict.update({'DTbn' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DTbn'}})
    process_dict.update({'DTo4' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DTod'}})
    process_dict.update({'DTod' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DTod'}})
    process_dict.update({'DTo5' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'suffix_convention' : 'onht'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DToh'}})
    process_dict.update({'DToh' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DToh'}})
    process_dict.update({'DT10' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DT10'}})
    process_dict.update({'DTo6' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DT10'}})
    process_dict.update({'DTh1' : {'dualprocess' : self._process_hs10,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hs10,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DTmc'}})
    #if DTmc to be applied downstream of multirt or 1010 MLinfilltype recommend recasting MLinfilltype to match
    process_dict.update({'DTmc' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DTmc'}})
    #note that inplace set to False when receiving messy_data_headers specification
    process_dict.update({'DTmp' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'messy_data_headers' : []},
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'DTmp'}})
    process_dict.update({'DTo7' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'upstream_hsh2' : True,
                                                     'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DTo7'}})
    process_dict.update({'DThs' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mThs'}})
    process_dict.update({'DTh2' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'space' : '',
                                                     'excluded_characters' : []},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'DTod'}})
    process_dict.update({'DTn4' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTns'}})
    process_dict.update({'DTns' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : False,
                                                     'testnoise' : True},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DTns'}})
    process_dict.update({'DTo8' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DT1s'}})
    process_dict.update({'DT1s' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : False,
                                                     'testnoise' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DT1s'}})
    process_dict.update({'DTsk' : {'dualprocess' : self._process_DPsk,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPsk,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'binary',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DTsk'}})
    process_dict.update({'DTne' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'rescale_sigmas' : True,
                                                     'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DTne'}})
    process_dict.update({'DTse' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : False,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DTse'}})
    #note that the default norm_category DPod is intended for use as a downstream tree category
    process_dict.update({'mThs' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'dtype' : 'mThs',
                                                     'flip_prob' : 0.03,
                                                     'testnoise' : False,
                                                     'norm_params' : {'test_flip_prob' : 0.01,
                                                                      'trainnoise' : False,
                                                                      'testnoise' : True,
                                                                      'test_weighted' : True},
                                                     'norm_category' : 'DTod'},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mThs'}})
    process_dict.update({'DBn3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBnb'}})
    process_dict.update({'DBnb' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBnb'}})
    process_dict.update({'DBm2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBmm'}})
    process_dict.update({'DBmm' : {'dualprocess' : self._process_DPmm,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmm,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBmm'}})
    process_dict.update({'DBrt' : {'dualprocess' : self._process_DPrt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPrt,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBrt'}})
    process_dict.update({'DBb2' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DBbn'}})
    process_dict.update({'DBbn' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DBbn'}})
    process_dict.update({'DBo4' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DBod'}})
    process_dict.update({'DBod' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DBod'}})
    process_dict.update({'DBo5' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'suffix_convention' : 'onht'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DBoh'}})
    process_dict.update({'DBoh' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DBoh'}})
    process_dict.update({'DB10' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DB10'}})
    process_dict.update({'DBo6' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DB10'}})
    process_dict.update({'DBh1' : {'dualprocess' : self._process_hs10,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hs10,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DBmc'}})
    #if DTmc to be applied downstream of multirt or 1010 MLinfilltype recommend recasting MLinfilltype to match
    process_dict.update({'DBmc' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DBmc'}})
    #note that inplace set to False when receiving messy_data_headers specification
    process_dict.update({'DBmp' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'messy_data_headers' : []},
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'DBmp'}})
    process_dict.update({'DBo7' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'upstream_hsh2' : True,
                                                     'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DBo7'}})
    process_dict.update({'DBhs' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mBhs'}})
    process_dict.update({'DBh2' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'space' : '',
                                                     'excluded_characters' : []},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'DBod'}})
    process_dict.update({'DBn4' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBns'}})
    process_dict.update({'DBns' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : True,
                                                     'testnoise' : True},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DBns'}})
    process_dict.update({'DBo8' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DB1s'}})
    process_dict.update({'DB1s' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : True,
                                                     'testnoise' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DB1s'}})
    process_dict.update({'DBsk' : {'dualprocess' : self._process_DPsk,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPsk,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'defaultparams' : {'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'binary',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DBsk'}})
    process_dict.update({'DBne' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'rescale_sigmas' : True,
                                                     'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DBne'}})
    process_dict.update({'DBse' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'swap_noise' : True,
                                                     'trainnoise' : True,
                                                     'testnoise' : True},
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DBse'}})
    #note that the default norm_category DPod is intended for use as a downstream tree category
    process_dict.update({'mBhs' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'dtype' : 'mThs',
                                                     'flip_prob' : 0.03,
                                                     'testnoise' : False,
                                                     'norm_params' : {'flip_prob' : 0.03,
                                                                      'test_flip_prob' : 0.01,
                                                                      'trainnoise' : True,
                                                                      'testnoise' : True,
                                                                      'weighted' : True,
                                                                      'test_weighted' : True},
                                                     'norm_category' : 'DBod'},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mBhs'}})
    process_dict.update({'DPn3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPnb'}})
    process_dict.update({'DPnb' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPnb'}})
    process_dict.update({'DPm2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPmm'}})
    process_dict.update({'DPmm' : {'dualprocess' : self._process_DPmm,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmm,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'noise_transform' : 'numeric',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPmm'}})
    process_dict.update({'DPrt' : {'dualprocess' : self._process_DPrt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPrt,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'noise_transform' : 'numeric',
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPrt'}})
    process_dict.update({'DLn3' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DLnb'}})
    process_dict.update({'DLnb' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'numeric',
                                  'defaultparams' : {'noisedistribution' : 'laplace'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DLnb'}})
    process_dict.update({'DLm2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DLmm'}})
    process_dict.update({'DLmm' : {'dualprocess' : self._process_DPmm,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmm,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'noise_transform' : 'numeric',
                                  'defaultparams' : {'noisedistribution' : 'laplace'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DLmm'}})
    process_dict.update({'DLrt' : {'dualprocess' : self._process_DPrt,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPrt,
                                  'inverseprocess' : self._inverseprocess_retn,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'noise_transform' : 'numeric',
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'noisedistribution' : 'laplace'},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DLrt'}})
    process_dict.update({'DPb2' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DPbn'}})
    process_dict.update({'DPbn' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'DPbn'}})
    process_dict.update({'DPo4' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DPod'}})
    process_dict.update({'DPod' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DPod'}})
    process_dict.update({'DPo5' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'suffix_convention' : 'onht'},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DPoh'}})
    process_dict.update({'DPoh' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'DPoh'}})
    process_dict.update({'DP10' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DP10'}})
    process_dict.update({'DPo6' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DP10'}})
    process_dict.update({'DPh1' : {'dualprocess' : self._process_hs10,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hs10,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DPmc'}})
    #if DPmc to be applied downstream of multirt or 1010 MLinfilltype recommend recasting MLinfilltype to match
    process_dict.update({'DPmc' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'DPmc'}})
    #note that inplace set to False when receiving messy_data_headers specification
    process_dict.update({'DPmp' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'messy_data_headers' : []},
                                  'noise_transform' : 'categoric',
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'DPmp'}})
    process_dict.update({'DPo7' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'upstream_hsh2' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'DPo7'}})
    process_dict.update({'DPhs' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mlhs'}})
    process_dict.update({'DPh2' : {'dualprocess' : self._process_hash,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_hash,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'space' : '',
                                                     'excluded_characters' : []},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'DPod'}})
    process_dict.update({'DPn4' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPns'}})
    process_dict.update({'DPns' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'DPns'}})
    process_dict.update({'DPo8' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DP1s'}})
    process_dict.update({'DP1s' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'DP1s'}})
    process_dict.update({'DPsk' : {'dualprocess' : self._process_DPsk,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPsk,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : False,
                                  'inplace_option' : False,
                                  'noise_transform' : 'binary',
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DPsk'}})
    process_dict.update({'DPne' : {'dualprocess' : self._process_DPnb,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPnb,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'numeric',
                                  'defaultparams' : {'rescale_sigmas' : True},
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DPne'}})
    process_dict.update({'DPse' : {'dualprocess' : self._process_DPmc,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPmc,
                                  'inverseprocess' : self._inverseprocess_DPmc,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'swap_noise' : True},
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DPse'}})
    process_dict.update({'DPpc' : {'dualprocess' : self._process_DPod,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_DPod,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'passthrough' : True},
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'DPpc'}})
    #note that the default norm_category DPod is intended for use as a downstream tree category
    process_dict.update({'mlhs' : {'dualprocess' : self._process_mlti,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mlti,
                                  'inverseprocess' : self._inverseprocess_mlti,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'noise_transform' : 'categoric',
                                  'defaultparams' : {'dtype' : 'mlhs',
                                                     'flip_prob' : 0.03,
                                                     'testnoise' : False,
                                                     'norm_params' : {'flip_prob' : 0.03,
                                                                      'testnoise' : False,
                                                                      'weighted' : True},
                                                     'norm_category' : 'DPod'},
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'ordlexclude',
                                  'labelctgy' : 'mlhs'}})
    process_dict.update({'qbt1' : {'dualprocess' : None,
                                  'singleprocess' : self._process_qbt1,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_qbt1,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'qbt1'}})
    process_dict.update({'qbt2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_qbt1,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_qbt1,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'defaultparams' : {'integer_bits' : 15,
                                                     'fractional_bits' : 0},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'qbt2'}})
    process_dict.update({'qbt3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_qbt1,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_qbt1,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'zeroinfill',
                                  'defaultparams' : {'sign_bit' : False,
                                                     'fractional_bits' : 13},
                                  'NArowtype' : 'nonnegativenumeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'qbt3'}})
    process_dict.update({'qbt4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_qbt1,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_qbt1,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'zeroinfill',
                                  'defaultparams' : {'sign_bit' : False,
                                                     'integer_bits' : 16,
                                                     'fractional_bits' : 0},
                                  'NArowtype' : 'nonnegativenumeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'qbt4'}})
    process_dict.update({'nmqb' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt1'}})
    process_dict.update({'nmq2' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt1'}})
    process_dict.update({'mmqb' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt3'}})
    process_dict.update({'mmq2' : {'dualprocess' : self._process_mnmx,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mnmx,
                                  'inverseprocess' : self._inverseprocess_mnmx,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt3'}})
    process_dict.update({'NArw' : {'dualprocess' : None,
                                  'singleprocess' : self._process_NArw,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'NAr2' : {'dualprocess' : None,
                                  'singleprocess' : self._process_NArw,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'NAr2'}})
    process_dict.update({'NAr3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_NArw,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'NAr3'}})
    process_dict.update({'NAr4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_NArw,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'nonnegativenumeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'NAr4'}})
    process_dict.update({'NAr5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_NArw,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'NAr5'}})
    process_dict.update({'null' : {'dualprocess' : None,
                                  'singleprocess' : self._process_null,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : None}})
    process_dict.update({'copy' : {'dualprocess' : None,
                                  'singleprocess' : self._process_copy,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'copy'}})
    process_dict.update({'excl' : {'dualprocess' : None,
                                  'singleprocess' : self._process_excl,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_excl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'excl'}})
    process_dict.update({'exc2' : {'dualprocess' : self._process_exc2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc2,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'exc2'}})
    process_dict.update({'exc3' : {'dualprocess' : self._process_exc2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc2,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'exc3'}})
    process_dict.update({'exc4' : {'dualprocess' : self._process_exc2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc2,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'exc4'}})
    process_dict.update({'exc5' : {'dualprocess' : self._process_exc5,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc5,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'integertype' : 'singlct'},
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'exc5'}})
    process_dict.update({'exc6' : {'dualprocess' : self._process_exc2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc2,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'exc6'}})
    process_dict.update({'exc7' : {'dualprocess' : self._process_exc5,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc5,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'integertype' : 'singlct'},
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'exc7'}})
    process_dict.update({'exc8' : {'dualprocess' : self._process_exc5,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc5,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'integertype' : 'integer'},
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'integer',
                                  'labelctgy' : 'exc8'}})
    process_dict.update({'exc9' : {'dualprocess' : self._process_exc5,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc5,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'defaultparams' : {'integertype' : 'integer'},
                                  'NArowtype' : 'integer',
                                  'MLinfilltype' : 'integer',
                                  'labelctgy' : 'exc9'}})
    process_dict.update({'shfl' : {'dualprocess' : None,
                                  'singleprocess' : self._process_shfl,
                                  'postprocess' : None,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'shfl'}})
    process_dict.update({'sint' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'sin'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'sint'}})
    process_dict.update({'cost' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'cos'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'cost'}})
    process_dict.update({'tant' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'tan'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'tant'}})
    process_dict.update({'arsn' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'arcsin'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'arsn'}})
    process_dict.update({'arcs' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'arccos'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'arcs'}})
    process_dict.update({'artn' : {'custom_train' : self._custom_train_trig,
                                  'custom_inversion' : self._custom_inversion_trig,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultparams' : {'operation' : 'arctan'},
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'artn'}})
    process_dict.update({'nmbd' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'101d' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : '101d'}})
    process_dict.update({'ordd' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'ordd'}})
    process_dict.update({'texd' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'texd'}})
    process_dict.update({'bnrd' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'bnrd'}})
    process_dict.update({'datd' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'nuld' : {'dualprocess' : None,
                                  'singleprocess' : self._process_null,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'totalexclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'nuld'}})
    process_dict.update({'lbnm' : {'dualprocess' : self._process_exc2,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_exc2,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'adjinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'lbnm'}})
    process_dict.update({'lbnb' : {'dualprocess' : self._process_numerical,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_numerical,
                                  'inverseprocess' : self._inverseprocess_nmbr,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'lbnb'}})
    process_dict.update({'lb10' : {'custom_train' : self._custom_train_1010,
                                  'custom_test' : self._custom_test_1010,
                                  'custom_inversion' : self._custom_inversion_1010,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : '1010',
                                  'labelctgy' : 'lb10'}})
    process_dict.update({'lbor' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'null_activation' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'lbor'}})
    process_dict.update({'lbos' : {'custom_train' : self._custom_train_ordl,
                                  'custom_test' : self._custom_test_ordl,
                                  'custom_inversion' : self._custom_inversion_ordl,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'null_activation' : False},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'strg'}})
    process_dict.update({'lbte' : {'custom_train' : self._custom_train_onht,
                                  'custom_test' : self._custom_test_onht,
                                  'custom_inversion' : self._custom_inversion_onht,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'multirt',
                                  'labelctgy' : 'lbte'}})
    process_dict.update({'lbbn' : {'dualprocess' : self._process_binary,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_binary,
                                  'inverseprocess' : self._inverseprocess_bnry,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'invert' : True},
                                  'NArowtype' : 'binary',
                                  'MLinfilltype' : 'binary',
                                  'labelctgy' : 'lbbn'}})
    process_dict.update({'lbsm' : {'dualprocess' : self._process_smth,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_smth,
                                  'inverseprocess' : self._inverseprocess_smth,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'lbsm'}})
    process_dict.update({'lbfs' : {'dualprocess' : self._process_smth,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_smth,
                                  'inverseprocess' : self._inverseprocess_smth,
                                  'info_retention' : True,
                                  'inplace_option' : False,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'LSfit' : True},
                                  'NArowtype' : 'justNaN',
                                  'MLinfilltype' : 'exclude',
                                  'labelctgy' : 'lbfs'}})
    process_dict.update({'lbda' : {'dualprocess' : None,
                                  'singleprocess' : None,
                                  'postprocess' : None,
                                  'inplace_option' : False,
                                  'NArowtype' : 'datetime',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'lgnr' : {'dualprocess' : self._process_absl,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_absl,
                                  'inverseprocess' : self._inverseprocess_absl,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'nonzeronumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt5'}})
    process_dict.update({'lgn2' : {'dualprocess' : self._process_logn,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_logn,
                                  'inverseprocess' : self._inverseprocess_logn,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'NArowtype' : 'positivenumeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'qbt5'}})
    process_dict.update({'qbt5' : {'dualprocess' : None,
                                  'singleprocess' : self._process_qbt1,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_qbt1,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'negzeroinfill',
                                  'defaultparams' : {'integer_bits' : 4,
                                                     'fractional_bits' : 3,
                                                     'sign_bit' : True},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'boolexclude',
                                  'labelctgy' : 'qbt5'}})
    process_dict.update({'sgn3' : {'dualprocess' : None,
                                  'singleprocess' : self._process_copy,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'suffix' : ''},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'sgn2'}})
    process_dict.update({'sgn4' : {'dualprocess' : None,
                                  'singleprocess' : self._process_copy,
                                  'postprocess' : None,
                                  'inverseprocess' : self._inverseprocess_UPCS,
                                  'info_retention' : True,
                                  'inplace_option' : True,
                                  'defaultparams' : {'suffix' : ''},
                                  'NArowtype' : 'exclude',
                                  'MLinfilltype' : 'totalexclude',
                                  'labelctgy' : 'sgn2'}})
    process_dict.update({'sgn1' : {'dualprocess' : self._process_mltp,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_mltp,
                                  'inverseprocess' : self._inverseprocess_mltp,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'meaninfill',
                                  'defaultparams' : {'multiply' : -1},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'numeric',
                                  'labelctgy' : 'sgn2'}})
    process_dict.update({'sgn2' : {'dualprocess' : self._process_bkt3,
                                  'singleprocess' : None,
                                  'postprocess' : self._postprocess_bkt3,
                                  'inverseprocess' : self._inverseprocess_bkt3,
                                  'info_retention' : False,
                                  'inplace_option' : True,
                                  'defaultinfill' : 'naninfill',
                                  'defaultparams' : {'buckets' : [0]},
                                  'NArowtype' : 'numeric',
                                  'MLinfilltype' : 'singlct',
                                  'labelctgy' : 'sgn2'}})

    return process_dict

  #__FunctionBlock: automunge process family functions

  def __processfamily(self, df_train, df_test, column, origcategory, \
                    transform_dict, postprocess_dict, assign_param, logger):
    '''
    #as automunge runs a for loop through each column in automunge, this is the master 
    #processing function applied which runs through the different family primitives
    #populated in the transform_dict by assembletransformdict
    #processfamily is only called with column as an inputcolumn and origcategory as a root category
    #downstream generations are applied within processparent
    
    #we will run in order of
    #siblings, cousins, parents, auntsuncles
    '''

    inplaceperformed = False
    
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #when a replacement transform is to be applied
    final_upstream = False
    if len(transform_dict[origcategory]['auntsuncles']) == 0:
      if len(transform_dict[origcategory]['parents']) > 0:
        final_upstream = transform_dict[origcategory]['parents'][-1]
    else:
      if len(transform_dict[origcategory]['auntsuncles']) > 0:
        final_upstream = transform_dict[origcategory]['auntsuncles'][-1]

    #process the siblings (with downstream, supplemental)
    for sibling in transform_dict[origcategory]['siblings']:

      if sibling != None:
        #note we use the processparent function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.__processparent(df_train, df_test, column, sibling, origcategory, final_upstream, \
                          transform_dict, postprocess_dict, assign_param, logger)
    
    #process the cousins (no downstream, supplemental)
    for cousin in transform_dict[origcategory]['cousins']:
      
      #this if statement kind of a placeholder such as for validation of primitive entry
      if cousin != None:

        #note we use the processcousin function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.__processcousin(df_train, df_test, column, cousin, origcategory, final_upstream, \
                            transform_dict, postprocess_dict, assign_param, logger)

    #process the parents (with downstream, with replacement)
    for parent in transform_dict[origcategory]['parents']:

      if parent != None:

        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.__processparent(df_train, df_test, column, parent, origcategory, final_upstream, \
                          transform_dict, postprocess_dict, assign_param, logger)
        
    #process the auntsuncles (no downstream, with replacement)
    for auntuncle in transform_dict[origcategory]['auntsuncles']:

      if auntuncle != None:

        #note we use the processcousin function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.__processcousin(df_train, df_test, column, auntuncle, origcategory, final_upstream, \
                            transform_dict, postprocess_dict, assign_param, logger)

    #if a replacement primitive was applied and inplace wasn't performed register the column to orig_noinplace
    if len(transform_dict[origcategory]['auntsuncles']) \
    + len(transform_dict[origcategory]['parents']) > 0 \
    and inplaceperformed is False:
      if column not in postprocess_dict['orig_noinplace']:
        postprocess_dict['orig_noinplace'] = postprocess_dict['orig_noinplace'] | {column}

    return df_train, df_test, postprocess_dict

  def __circleoflife(self, df_train, df_test, column, category, \
                    transform_dict, postprocess_dict, templist1):
    '''
    #This function deletes source column for cases where family primitives 
    #included replacement, with maintenance of the associated data structures.
    
    #templist1 is the list of df_train columns before processfamily
    '''

    #if we had replacement transformations performed on first generation \
    #then delete the original column
    if len(transform_dict[category]['auntsuncles']) \
    + len(transform_dict[category]['parents']) > 0:
      
      if column in postprocess_dict['orig_noinplace']:
        del df_train[column]
        del df_test[column]

    #if we had replacement transformations performed on downstream generation \
    #then delete the associated parent column 
    
    newcolumns = set(df_train) - set(templist1)
    
    #this one is for columns replaced as part of inplace operation
    if len(newcolumns) > 0:
      anewcolumn = list(newcolumns)[0]
      temp_columnslist = self.__autocopy(postprocess_dict['column_dict'][anewcolumn]['columnslist'])
      for newcolumn in temp_columnslist:
        if postprocess_dict['column_dict'][newcolumn]['deletecolumn'] == 'inplace':
          for newcolumn2 in temp_columnslist:
            if newcolumn in postprocess_dict['column_dict'][newcolumn2]['columnslist']:        
              postprocess_dict['column_dict'][newcolumn2]['columnslist'].remove(newcolumn)
    
    #this one is for columns we manually delete
    for newcolumn in newcolumns:
      if postprocess_dict['column_dict'][newcolumn]['deletecolumn'] is True:
        for newcolumn2 in newcolumns:
          if newcolumn in postprocess_dict['column_dict'][newcolumn2]['columnslist']:
            postprocess_dict['column_dict'][newcolumn2]['columnslist'].remove(newcolumn)
          
        #now we'll delete column
        #note this only worksa on single column  parents, need to incioroprate categorylist
        #for multicolumn parents (future extension)
        if newcolumn in df_train.columns:
          del df_train[newcolumn]
          del df_test[newcolumn]

    return df_train, df_test, postprocess_dict

  def __dictupdate(self, column, column_dict, postprocess_dict):
    '''
    #dictupdate function takes as input column_dict, postprocess_dict, then for cases
    #where origcolumn is the same for the two combines the columnslist and the 
    #normalization_dict, then appends the column_dict onto the postprocess_dict
    #returns the column_dict and postprocess_dict. Note that the passed column name
    #"column" is the column name prior to the applicaiton of processing, and the
    #name of the column after the last processing function is saved as a key
    #in the column_dict
    '''

    #(reason for "key2" instead of key1 is some shuffling during editing)
    for key2 in column_dict:

      #first address carry-though of origcolumn and origcategory from parent to child
      if column in postprocess_dict['column_dict']:

        #if column is not origcolumn in postprocess_dict
        if postprocess_dict['column_dict'][column]['origcolumn'] \
        != column:

          #assign origcolumn from postprocess_dict to column_dict
          column_dict[key2]['origcolumn'] = \
          postprocess_dict['column_dict'][column]['origcolumn']

          #assign origcategory from postprocess_dict to column_dict
          column_dict[key2]['origcategory'] = \
          postprocess_dict['column_dict'][column]['origcategory']

      for key1 in postprocess_dict['column_dict']:

        #if origcolumn is the same between column_dict saved in postprocess_dict and
        #the column_dict outputed from our processing, we'll combine a few values
        if postprocess_dict['column_dict'][key1]['origcolumn'] == column_dict[key2]['origcolumn']:
          #first we'll combine the columnslist capturing all columns 
          #originating from same origcolumn for these two sets
          postprocess_dict['column_dict'][key1]['columnslist'] = \
          list(set(postprocess_dict['column_dict'][key1]['columnslist'])|set(column_dict[key2]['columnslist']))
          #apply that value to the column_dict columnslist as well
          column_dict[key2]['columnslist'] = postprocess_dict['column_dict'][key1]['columnslist']

    #now append column_dict onto postprocess_dict
    postprocess_dict['column_dict'].update(column_dict)

    #return column_dict, postprocess_dict
    return postprocess_dict

  def __populate_columnkey_dict(self, column_dict_list, postprocess_dict, transformationcategory):
    """
    #columnkey_dict is used in postprocess functions
    #to derive a normkey when returned column isn't known or may return emtpy set
    #
    #populates in the form:
    #columnkey_dict.update({inputcolumn : {transformationcategory : categorylist}})
    #where categorylist_aggregate is a combination of all categorylists 
    #derived from some functions with recorded category applied to a given input column
    #
    #Note that convention is if the same transformation is applied multiple times in same family tree
    #each application will have a unique input column, otherwise transfomation will be via a unique process_dict transformationcategory
    #  
    #Here column_dict_list is the list of dictionaries returned from a single processing function
    """

    if len(column_dict_list) > 0:

      inputcolumn = column_dict_list[0][list(column_dict_list[0])[0]]['inputcolumn']
      if inputcolumn not in postprocess_dict['columnkey_dict']:
        postprocess_dict['columnkey_dict'].update({inputcolumn : {}})
        
      categorylist = column_dict_list[0][list(column_dict_list[0])[0]]['categorylist']

      if transformationcategory not in postprocess_dict['columnkey_dict'][inputcolumn]:
        postprocess_dict['columnkey_dict'][inputcolumn].update({transformationcategory : categorylist})
      
    return postprocess_dict
  
  def __processcousin(self, df_train, df_test, column, cousin, origcategory, final_upstream, \
                     transform_dict, postprocess_dict, assign_param, logger):
    '''
    #cousin is one of the primitives for processfamily function, and it involves
    #transformations without downstream derivations without replacement of source column
    #although this same funciton can be used with the auntsuncles primitive
    #by following with a deletion of original column, also this funciton can be
    #used on the niecesnephews primitive downstream of parents or siblings since 
    #they don't have children (they're way to young for that)
    #note the processing funcitons are accessed through the process_dict
    #note that validation of processing functions in user passed processdict takes place in _check_processdict4
    
    #note that the presence of a custom_train processing function for a tree category
    #takes precedence over a dualprocess function
    #which takes precedence over a singleprocess function

    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}

    '''

    process_dict = postprocess_dict['process_dict']
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == cousin:
      inplacecandidate = True

    params = self.__grab_params(assign_param, cousin, column, process_dict[cousin], postprocess_dict)

    #now populate params entries associated with entropy seeding for noise transforms
    params, postprocess_dict = \
    self.__random_parameters_params_append(params, postprocess_dict, \
                                      column, cousin, 'traintest', \
                                      df_train.shape[0], df_test.shape[0], \
                                      postprocess_dict['printstatus'], df_train, df_test)
    
    if inplacecandidate is True:
      if 'inplace_option' not in process_dict[cousin] \
      or 'inplace_option' in process_dict[cousin] \
      and process_dict[cousin]['inplace_option'] is True:
        if 'inplace' not in params:
          inplaceperformed = True
          params.update({'inplace' : True})
        elif ('inplace' in params and params['inplace'] != False):
          inplaceperformed = True
          params.update({'inplace' : True})
        else:
          inplaceperformed = False
    else:
      #user cannot manually specify inplace as True by design
      if ('inplace' in params and params['inplace'] is True):
        inplaceperformed = False
        params.update({'inplace' : False})
    
    #if this is a custom process function
    #(convention is that 'custom_train' is populated in both scenarios for dualprocess or singleprocess)
    if 'custom_train' in process_dict[cousin] \
    and process_dict[cousin]['custom_train'] != None:
      
      #note that _custom_process_wrapper accesses the copy of process_dict saved within postprocess_dict
      df_train, df_test, column_dict_list = \
      self.__custom_process_wrapper(df_train, df_test, column, origcategory, \
                                   cousin, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, cousin)
    
    #elif this is a dual process function
    elif 'dualprocess' in process_dict[cousin] \
    and process_dict[cousin]['dualprocess'] != None:

      df_train, df_test, column_dict_list = \
      process_dict[cousin]['dualprocess'](df_train, df_test, column, origcategory, \
                                          cousin, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, cousin)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[cousin] \
    and process_dict[cousin]['singleprocess'] != None:

      df_train, column_dict_list =  \
      process_dict[cousin]['singleprocess'](df_train, column, origcategory, \
                                            cousin, postprocess_dict, params)

      df_test, _1 = \
      process_dict[cousin]['singleprocess'](df_test, column, origcategory, \
                                            cousin, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, cousin)

    else:

      #_____the following 40 rows is a validation result aggregation for cases where processdict was speciifed without a callable trasnformation function
      #since we populate results with unique integer identifier performing here a little dictionary munging
      # if postprocess_dict['printstatus'] != 'silent':
        
      self.__autoprint('warning', 'Please note that a tree category was accessed as category ', cousin)
      self.__autoprint('warning', 'Originating from generations of root category ', origcategory)
      self.__autoprint('warning', 'Which did not have processing functions populated in processdict')
      self.__autoprint('warning', 'Processing functions in custom_train or dual/single/post process convention ')
      self.__autoprint('warning', 'Are required when a category is accessed as a tree category entry in a family tree.')
      self.__autoprint('warning', 'Otherwise tree category is treated as None and no downstream generations are accessed.')
      self.__autoprint('warning', '')

      column_dict_list = []

      #populate validation result
      #we'll have convention of populating an entry each time this validation result identified
      #with a key of an integer iterated with each occurance
      #and populated with details of treecategory without processing functions and associated root category 
      # treecategory_with_empty_processing_functions_valresult = {i : {'treecategory' : treecategory,
      #                                                                'rootcategory' : rootcategory}}

      if 'treecategory_with_empty_processing_functions_valresult' not in logger_dict['validations']:
        logger_dict['validations'].update({'treecategory_with_empty_processing_functions_valresult' : {}})

      #validation results reported with key of an integer incremented with each occurance
      #get value of current max entry 
      #(here using pandas as a hack since python doesn't have clean max item in list, works bacause are numeric)
      max_valkey_i = pd.DataFrame({'temp' : list(logger_dict['validations']['treecategory_with_empty_processing_functions_valresult'])}).max()

      #this returns max_valkey_i as a series with one integer entry, with entry as NaN when no entries present
      if (max_valkey_i != max_valkey_i).iat[0]:
        max_valkey_i = -1
      else:
        max_valkey_i = int(max_valkey_i)

      #initialize our valitation result reporting, which notes the tree category without processing functions
      #as well as the root category in whose generations this tree category was found
      treecategory_with_empty_processing_functions_valresult = {max_valkey_i+1 : {'treecategory' : cousin,
                                                                                  'rootcategory' : origcategory}}

      #populate the validation results in ['treecategory_with_empty_processing_functions_valresult']
      logger_dict['validations']['treecategory_with_empty_processing_functions_valresult'].update(
        treecategory_with_empty_processing_functions_valresult
        )
      logger = self.__logger_append(logger)
      #_____

    #update the columnslist and normalization_dict for both column_dict and postprocess_dict
    for column_dict in column_dict_list:
      postprocess_dict = self.__dictupdate(column, column_dict, postprocess_dict)

    return df_train, df_test, postprocess_dict, inplaceperformed

  def __processparent(self, df_train, df_test, column, parent, origcategory, final_upstream, \
                    transform_dict, postprocess_dict, assign_param, logger):
    '''
    #parent is one of the primitives for processfamily function, and it involves
    #transformations with downstream derivations with replacement of source column
    #although this same funciton can be used with the siblinga primitive
    #by not following with a deletion of original column, also this funciton can be
    #used on the children primitive downstream of parents or siblings, allowing
    #the children to have children of their own, you know, grandchildren and stuff.
    #note the processing functions are accessed through the process_dict
    #note that validation of processing functions in user passed processdict takes place in _check_processdict4
    
    #note that the presence of a custom_train processing function for a tree category
    #takes precedence over a dualprocess function
    #which takes precedence over a singleprocess function
    
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    
    #we want to apply in order of
    #upstream process, niecesnephews, friends, children, coworkers
    '''

    process_dict = postprocess_dict['process_dict']

    #upstream process
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == parent:
      inplacecandidate = True
    
    params = self.__grab_params(assign_param, parent, column, process_dict[parent], postprocess_dict)

    #now populate params entries associated with entropy seeding for noise transforms
    params, postprocess_dict = \
    self.__random_parameters_params_append(params, postprocess_dict, \
                                          column, parent, 'traintest', \
                                          df_train.shape[0], df_test.shape[0], \
                                          postprocess_dict['printstatus'], df_train, df_test)
    
    if inplacecandidate is True:
      if 'inplace_option' not in process_dict[parent] \
      or 'inplace_option' in process_dict[parent] \
      and process_dict[parent]['inplace_option'] is True:
        if 'inplace' not in params:
          inplaceperformed = True
          params.update({'inplace' : True})
        elif ('inplace' in params and params['inplace'] != False):
          inplaceperformed = True
          params.update({'inplace' : True})
        else:
          inplaceperformed = False
    else:
      #user cannot manually specify inplace as True by design
      if ('inplace' in params and params['inplace'] is True):
        inplaceperformed = False
        params.update({'inplace' : False})
    
    #if this is a custom process function 
    #(convention is that 'custom_train' is populated in both scenarios for dualprocess or singleprocess)
    if 'custom_train' in process_dict[parent] \
    and process_dict[parent]['custom_train'] != None:
      
      #note that _custom_process_wrapper accesses the copy of process_dict saved within postprocess_dict
      df_train, df_test, column_dict_list = \
      self.__custom_process_wrapper(df_train, df_test, column, origcategory, \
                                   parent, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, parent)
    
    #elif this is a dual process function
    elif 'dualprocess' in process_dict[parent] \
    and process_dict[parent]['dualprocess'] != None:

      df_train, df_test, column_dict_list = \
      process_dict[parent]['dualprocess'](df_train, df_test, column, origcategory, \
                                          parent, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, parent)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[parent] \
    and process_dict[parent]['singleprocess'] != None:

      df_train, column_dict_list =  \
      process_dict[parent]['singleprocess'](df_train, column, origcategory, \
                                          parent, postprocess_dict, params)

      df_test, _1 = \
      process_dict[parent]['singleprocess'](df_test, column, origcategory, \
                                          parent, postprocess_dict, params)
      
      #columnkey_dict used in postprocess functions in a few cases to derive a normkey
      postprocess_dict = self.__populate_columnkey_dict(column_dict_list, postprocess_dict, parent)

    else:

      #_____aggregating validaiton results for non callable functions in processdict entry
      # if postprocess_dict['printstatus'] != 'silent':

      self.__autoprint('warning', 'Please note that a tree category was accessed as category ', parent)
      self.__autoprint('warning', 'Originating from generations of root category ', origcategory)
      self.__autoprint('warning', 'Which did not have processing functions populated in processdict')
      self.__autoprint('warning', 'Processing functions in custom_train or dual/single/post process convention ')
      self.__autoprint('warning', 'Are required when a category is accessed as a tree category entry in a family tree.')
      self.__autoprint('warning', 'Otherwise tree category is treated as None and no downstream generations are accessed.')
      self.__autoprint('warning', '')

      column_dict_list = []

      #populate validation result
      #we'll have convention of populating an entry each time this validation result identified
      #with a key of an integer iterated with each occurance
      #and populated with details of treecategory without processing functions and associated root category 
      # treecategory_with_empty_processing_functions_valresult = {i : {'treecategory' : treecategory,
      #                                                                'rootcategory' : rootcategory}}

      if 'treecategory_with_empty_processing_functions_valresult' not in logger_dict['validations']:
        logger_dict['validations'].update({'treecategory_with_empty_processing_functions_valresult' : {}})

      #validation results reported with key of an integer incremented with each occurance
      #get value of current max entry
      #(here using pandas as a hack since python doesn't have clean max item in list, works bacause are numeric)
      max_valkey_i = pd.DataFrame({'temp' : list(logger_dict['validations']['treecategory_with_empty_processing_functions_valresult'])}).max()

      #this returns max_valkey_i as a series with one integer entry, with entry as NaN when no entries present
      if (max_valkey_i != max_valkey_i).iat[0]:
        max_valkey_i = -1
      else:
        max_valkey_i = int(max_valkey_i)

      #initialize our valitation result reporting, which notes the tree category without processing functions
      #as well as the root category in whose generations this tree category was found
      treecategory_with_empty_processing_functions_valresult = {max_valkey_i+1 : {'treecategory' : parent,
                                                                                  'rootcategory' : origcategory}}

      #populate the validation results in ['treecategory_with_empty_processing_functions_valresult']
      logger_dict['validations']['treecategory_with_empty_processing_functions_valresult'].update(
        treecategory_with_empty_processing_functions_valresult
        )
      logger = self.__logger_append(logger)
      #_____

    #update the columnslist and normalization_dict for both column_dict and postprocess_dict
    for column_dict in column_dict_list:
      postprocess_dict = self.__dictupdate(column, column_dict, postprocess_dict)

    #we take an arbitrary column returned from upstream transform as first entry in categorylist
    #to serve as parentcolumn input to next generation transform
    #if downstream transform includes support for multi-column input
    #they can use parentcolumn to access from column_dict the upstream categorylist and normalization_dict
    if len(column_dict_list) > 0:
      column_dict = column_dict_list[0]
      support_column = list(column_dict.keys())[0]
      support_categorylist = column_dict[support_column]['categorylist']
      parentcolumn = support_categorylist[0]
      
      #only proceed to next generation if parent did not return an empty set

      #if transform_dict[parent] != None:

      #initialize in case no downstream performed
      parent_inplaceperformed = False

      #process any children

      #final downstream transform from coworkers or children is elligible for inplace
      #when a replacement transform is to be applied
      final_downstream = False
      if len(transform_dict[parent]['coworkers']) == 0:
        if len(transform_dict[parent]['children']) > 0:
          final_downstream = transform_dict[parent]['children'][-1]
      else:
        if len(transform_dict[parent]['coworkers']) > 0:
          final_downstream = transform_dict[parent]['coworkers'][-1]

      #process any niecesnephews
      #note the function applied is comparable to processsibling, just a different
      #parent column
      for niecenephew in transform_dict[parent]['niecesnephews']:

        if niecenephew != None:

          #process the niecenephew
          #note the function applied is processparent (using recursion)
          #parent column
          df_train, df_test, postprocess_dict, parent_inplaceperformed = \
          self.__processparent(df_train, df_test, parentcolumn, niecenephew, origcategory, final_downstream, \
                             transform_dict, postprocess_dict, assign_param, logger)

      #process any friends
      for friend in transform_dict[parent]['friends']:

        if friend != None:

          #process the friend
          #note the function applied is processcousin
          df_train, df_test, postprocess_dict, parent_inplaceperformed = \
          self.__processcousin(df_train, df_test, parentcolumn, friend, origcategory, final_downstream, \
                             transform_dict, postprocess_dict, assign_param, logger)

      for child in transform_dict[parent]['children']:

        if child != None:

          #process the child
          #note the function applied is processparent (using recursion)
          #parent column
          df_train, df_test, postprocess_dict, parent_inplaceperformed = \
          self.__processparent(df_train, df_test, parentcolumn, child, origcategory, final_downstream, \
                             transform_dict, postprocess_dict, assign_param, logger)

      #process any coworkers
      for coworker in transform_dict[parent]['coworkers']:

        if coworker != None:

          #process the coworker
          #note the function applied is processcousin
          df_train, df_test, postprocess_dict, parent_inplaceperformed = \
          self.__processcousin(df_train, df_test, parentcolumn, coworker, origcategory, final_downstream, \
                             transform_dict, postprocess_dict, assign_param, logger)

      #if we had replacement transformations performed then mark column for deletion
      if len(transform_dict[parent]['children']) \
      + len(transform_dict[parent]['coworkers']) > 0 \
      and parent_inplaceperformed is False:
        #here we'll only address downstream generaitons
        if parentcolumn in postprocess_dict['column_dict']:
          for parentcolumn_categorylist_entry in postprocess_dict['column_dict'][parentcolumn]['categorylist']:
            postprocess_dict['column_dict'][parentcolumn_categorylist_entry]['deletecolumn'] = True
        else:
          if parentcolumn not in postprocess_dict['orig_noinplace']:
            postprocess_dict['orig_noinplace'] = postprocess_dict['orig_noinplace'] | {parentcolumn}
      elif parent_inplaceperformed is True:
        if parentcolumn in postprocess_dict['column_dict']:
          for parentcolumn_categorylist_entry in postprocess_dict['column_dict'][parentcolumn]['categorylist']:
            postprocess_dict['column_dict'][parentcolumn_categorylist_entry]['deletecolumn'] = 'inplace'

    return df_train, df_test, postprocess_dict, inplaceperformed

  def __grab_params(self, assign_param, category, column, processdict_entry, postprocess_dict):
    """
    #In order of precendence, parameters assigned to distinct 
    #category/column configurations take precedence 
    #to default_assignparam assigned to categories which take precendence 
    #to global_assignparam assigned to all transformations which take precendence 
    #to parameters set as defaultparams in processdict definition.
    """

    params = {}
    
    if 'defaultparams' in processdict_entry:
      
      #key are parameters
      for key in processdict_entry['defaultparams']:
        
        params.update({key : processdict_entry['defaultparams'][key]})

    #if assign_param is not empty
    if bool(assign_param):
      
      if 'global_assignparam' in assign_param:
        
        #key are parameters
        for key in assign_param['global_assignparam']:
          
          params.update({key : assign_param['global_assignparam'][key]})
          
      if 'default_assignparam' in assign_param:
        
        if category in assign_param['default_assignparam']:
            
          #key are parameters
          for key in assign_param['default_assignparam'][category]:

            params.update({key : assign_param['default_assignparam'][category][key]})
                
      if category in assign_param:
        
        #distinct category/column configurations can either be assigned
        #using the source column or the derived column serving as input to the transform
        #in case both are present the dervied column specifciation takes precedence
          
        #derived column specification with suffix takes precendence over input column specification
        if column in assign_param[category]:

          #key are parameters
          for key in assign_param[category][column]:

            params.update({key : assign_param[category][column][key]})

        #we won't use a specified source column entry if the derived column was already specfied
        elif column in postprocess_dict['column_dict'] \
        and postprocess_dict['column_dict'][column]['origcolumn'] in assign_param[category]:

          #key are parameters
          for key in assign_param[category][postprocess_dict['column_dict'][column]['origcolumn']]:

            params.update({key : assign_param[category][postprocess_dict['column_dict'][column]['origcolumn']][key]})
            
      #messy_data_headers refers to input features in multi column encodings
      if 'messy_data_headers' in assign_param:
        
        #we'll know it is an inputcolumn if not in column_dict
        if column in assign_param['messy_data_headers'] \
        and column not in postprocess_dict['column_dict']:

          #messy convention doesn't have inplace support
          params.update({'inplace' : False})
          params.update({'messy_data_headers' : assign_param['messy_data_headers'][column]})
        
    return params

  def __custom_process_wrapper(self, mdf_train, mdf_test, column, category, \
                              treecategory, postprocess_dict, params = {}):
    """
    A wrapper for custom transformation functions applied in automunge
    Where custom transformations follow templates of custom_train_template 
    and custom_test_template
    
    The purpose of this wrapper is to extend the minimized versions of the 
    custom transform templates
    To include other conventions of the library
    Such as default infill, populating data structures, inplace option, 
    suffix overlap detection, and etc
    
    The form of the _custom_process_wrapper inputs/outputs 
    will be similar to what is applied for dualprocess functions
    
    When a processdict has entries for both custom_train_template 
    and custom_test_template
    custom_train_template will be applied to mdf_train 
    and custom_test_template will be applied to mdf_test
    
    Otherwise if the processdict only has an entry for custom_train_template
    custom_train_template will be applied to both mdf_train and mdf_test
    And the normalization_dict populated based on mdf_train will be the saved version
    
    Receives dataframes of a train and test set as mdf_train and mdf_test
    column is the received column that will serve as target for the transformation
    category is the root category for original upstream primitive entries
    treecategory is the category entry to a family tree primitive that 
    was used to access this transformation
    postprocess_dict is the dictionary data structure passed between transforms
    (Note that at this point postprocess_dict will include an entry for the 
    process_dict including consolidated processdict entries)
    params are the parameters passed through assignparam associated with 
    this specific categoy and column
    
    Returns the resulting transformed dataframes mdf_train and mdf_test
    and a populated column_dict_list data structure
    Where column_dict_list has a column_dict entry for each column returned from transform
    And where general convention in library is that the same normalization_dict 
    is saved in each column_dict populated in a transform*
    (*as an asterisk, in a few cases in library the normalization_dict may differ between column_dicts 
    populated in a transform returning a multi column set due to returned column specific drift stats)
    
    Note that this wrapper includes application of a default infill per any processdict entry for 'defaultinfill'
    Or otherwise performs adjinfill as default
    As well as other infill conventions determined based on the NArowtype associated with the treecategory
    
    We'll also create seperately a similar wrapper for postprocess functions applied 
    in postmunge (_custom_postprocess_wrapper)
    And likewise a wrapper for custom inversions that may be performed 
    in postmunge (_custom_inverseprocess_wrapper)
    """

    suffixoverlap_results = {}
    custom_process_wrapper_dict = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    suffix = treecategory
      
    #note this suffixcolumn convention won't work with the two suffix edge cases of text and excl transform
    #which is fine since this is only for transforms passed through custom_train / custom_test
    suffixcolumn = column + '_' + suffix
    
    #grab a list of current columns for a suffix overlap check performed after applying transforms
    initial_columns = list(mdf_train)
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
      
    #___
      
    #now that we've created a new column, we'll apply a default infill
    #contingent on the 'NArowtype' of the tree category serving as basis for this transform
    NArowtype = postprocess_dict['process_dict'][treecategory]['NArowtype']
    MLinfilltype = postprocess_dict['process_dict'][treecategory]['MLinfilltype']
    
    #first to apply default infill we'll convert any nonvalid entries to nan
    
    if NArowtype in {'numeric'}:
      
      #convert all values to either numeric or NaN
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
    elif NArowtype in {'integer'}:
      
      #convert all values to either numeric or NaN
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #non integers are subject to infill
      mdf_train = self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == mdf_train[suffixcolumn].round(), alternative=np.nan, specified='alternative')
      mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn].round(), alternative=np.nan, specified='alternative')
    
    elif NArowtype in {'positivenumeric'}:
      
      #convert all values to either numeric or NaN
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #values <= 0 subject to infill
      mdf_train.loc[mdf_train[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
      mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
      
    elif NArowtype in {'nonnegativenumeric'}:
      
      #convert all values to either numeric or NaN
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #values < 0 subject to infill
      mdf_train.loc[mdf_train[suffixcolumn] < 0, (suffixcolumn)] = np.nan
      mdf_test.loc[mdf_test[suffixcolumn] < 0, (suffixcolumn)] = np.nan
      
    elif NArowtype in {'nonzeronumeric'}:
      
      #convert all values to either numeric or NaN
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #values == 0 subject to infill
      mdf_train.loc[mdf_train[suffixcolumn] == 0, (suffixcolumn)] = np.nan
      mdf_test.loc[mdf_test[suffixcolumn] == 0, (suffixcolumn)] = np.nan
      
    elif NArowtype in {'datetime'}:
      
      #convert values to datetime
      mdf_train[suffixcolumn] = pd.to_datetime(mdf_train[suffixcolumn], errors = 'coerce', utc=True, format='mixed')
      mdf_test[suffixcolumn] = pd.to_datetime(mdf_test[suffixcolumn], errors = 'coerce', utc=True, format='mixed')
      
    elif NArowtype in {'justNaN', 'binary', 'exclude', 'totalexclude', 'parsenumeric'}:
      
      #nonvalid entries are already nan
      pass
    
    #___
    
    #Now that all nonvalid entries are cast as nan, we'll perform our infill
    #which will default to adjacent cell infill unless otherwise specified in processdict
    
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    custom_process_wrapper_dict.update({'defaultinfill_dict' : defaultinfill_dict})
    
    #___
      
    #great now we can apply the custom_train_template to training data
    #custom_train_template will be saved as entry for 'custom_train'
    mdf_train, normalization_dict = \
    postprocess_dict['process_dict'][treecategory]['custom_train'](mdf_train, suffixcolumn, params)
    
    returned_columns = list(mdf_train)
    
    #newcolumns_list is list of returned columns including suffixcolumn
    #may be empty when suffixcolumn not returned
    #this might be a different order of columns vs the dataframe, we'll then recover the order
    #the ^ operation is to accomodate both inplace scenarios
    newcolumns_set = (set(initial_columns) ^ set(returned_columns)) - set(initial_columns)
    newcolumns_list = []
    #this is to recover the order of columns as found in returned_columns
    for returned_column in returned_columns:
      if returned_column in newcolumns_set:
        newcolumns_list.append(returned_column)

    #to be consistent with postmunge, if newcolumns_list is empty then we'll just delete the suffix column in mdf_test
    if len(newcolumns_list) == 0:
      del mdf_test[suffixcolumn]
    else:
      #We'll have convention that if a corresponding custom_test_template wasn't populated
      #custom_test entry will either not be included or cast as None, in which case we apply custom_train to test data
      if 'custom_test' in postprocess_dict['process_dict'][treecategory] \
      and postprocess_dict['process_dict'][treecategory]['custom_test'] != None:
        mdf_test = \
        postprocess_dict['process_dict'][treecategory]['custom_test'](mdf_test, suffixcolumn, normalization_dict)
      else:
        mdf_test, _1 = \
        postprocess_dict['process_dict'][treecategory]['custom_train'](mdf_test, suffixcolumn, params)
      
    #___
    
    #Now run some suffix overlap tests
    
    #If any temporary columns were created but not returned
    #the convention is user should log in a normalization_dict entry under 'tempcolumns' for suffix overlap detection
    if 'tempcolumns' in normalization_dict:
      tempcolumns = normalization_dict['tempcolumns']
      if not isinstance(tempcolumns, list):
        tempcolumns = [tempcolumns]
        
      suffixoverlap_results = self.__df_check_suffixoverlap(initial_columns, \
                                                           tempcolumns, \
                                                           suffixoverlap_results = suffixoverlap_results, \
                                                           printstatus = postprocess_dict['printstatus'])
    
    #now check suffix overlap for returned columns (noting that suffixcolumn already checked above)
    suffixoverlap_results = self.__df_check_suffixoverlap(initial_columns, \
                                                         newcolumns_list, \
                                                         suffixoverlap_results = suffixoverlap_results, \
                                                         printstatus = postprocess_dict['printstatus'])
    
    #add entries to custom_process_wrapper_dict associated with suffix and inplace for reference
    custom_process_wrapper_dict.update({'suffix'  : suffix,
                                        'inplace' : inplace})
    
    #___
    
    #we'll perform one more infill via adjinfill in case user defined transform had unforseen edge case
    
    if MLinfilltype not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

      #apply ffill to replace NArows with value from adjacent cell in preceding row
      mdf_train[newcolumns_list] = mdf_train[newcolumns_list].fillna(method='ffill')
      mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(method='ffill')

      #we'll follow with a bfill just in case first row had a nan
      mdf_train[newcolumns_list] = mdf_train[newcolumns_list].fillna(method='bfill')
      mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(method='bfill')
      
      #finally if prior infill still resulted in nan we'll just plug with 0
      mdf_train[newcolumns_list] = mdf_train[newcolumns_list].fillna(0)
      mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(0)
    
    #___
    
    #now perform a datatype conversion based on MLinfilltype
    
    dtype_convert = True
    #user can deactivate dtype convert in processdict if desired
    if 'dtype_convert' in postprocess_dict['process_dict'][treecategory]:
      if postprocess_dict['process_dict'][treecategory]['dtype_convert'] is False:
        dtype_convert = False
        
    #we'll store any parameters for conditional dtype in custom_process_wrapper_dict
    custom_process_wrapper_dict.update({'dtype_convert' : dtype_convert})
    
    if dtype_convert is True:

      if MLinfilltype in {'numeric', 'concurrent_nmbr', 'exclude'}:
        #datatype conversion performed elsewhere based on floatprecision parameter
        pass

      if MLinfilltype in {'binary', 'multirt', '1010', 'concurrent_act', 'boolexclude'}:
        #datatype cast as np.int8 since entries are boolean integers
        mdf_train[newcolumns_list] = mdf_train[newcolumns_list].astype(np.int8)
        mdf_test[newcolumns_list] = mdf_test[newcolumns_list].astype(np.int8)
      
      if MLinfilltype in {'singlct', 'ordlexclude', 'concurrent_ordl'}:
        #ordinal sets are given a conditional dtype based on max activation
        #this can be deactivated in processdict with dtype_convert if desired
        
        #for concurrent_ordl we'll populate a dicitonary of max_encodings for each newcolumn
        max_encoding_for_dtype_convert = {}

        for newcolumn in newcolumns_list:

          max_encoding = mdf_train[newcolumn].max()
          max_encoding_for_dtype_convert.update({newcolumn : max_encoding})

          if max_encoding <= 255:
            mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.uint8)
            mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint8)
          elif max_encoding <= 65535:
            mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.uint16)
            mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint16)
          else:
            mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.uint32)
            mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint32)

        custom_process_wrapper_dict.update({'max_encoding_for_dtype_convert' : max_encoding_for_dtype_convert})

      if MLinfilltype in {'integer', 'totalexclude'}:
        #no conversion, assumes any conversion takes place in transformation function
        pass

    #___
    
    #Now populate the returned column_dict_list
    #(note that the column_dict entry 'custom_process_wrapper_dict' is unique to the custom_train convention)
    
    column_dict_list = []
    
    newcolumn_normalization_dict = {}
    if len(newcolumns_list) > 0:
      newcolumn_normalization_dict = {newcolumns_list[0] : normalization_dict}
    
    for newcolumn in newcolumns_list:

      if newcolumn != newcolumns_list[0]:
        newcolumn_normalization_dict = {}
      
      column_dict = {newcolumn : {'category' : treecategory, \
                                 'origcategory' : category, \
                                 'normalization_dict' : newcolumn_normalization_dict, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : newcolumns_list, \
                                 'categorylist' : newcolumns_list, \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False, \
                                 'custom_process_wrapper_dict' : custom_process_wrapper_dict}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list

  def __check_for_protected_features(self, assign_param, processdict):
    """
    for cases where any protected_feature have specification
    either in assign_param or processdict
    those features are prepared in a seperate for loop through columns
    to ensure available in original form when noise trasnforms are applied
    """

    protected_features = []
    
    for entry_1 in assign_param:
      #global specification {global_assignparam : {param : value}}
      if entry_1 == 'global_assignparam':
        if 'protected_feature' in assign_param[entry_1]:
          if assign_param[entry_1]['protected_feature'] is not False:
            protected_features.append(assign_param[entry_1]['protected_feature'])
            
      #default category specification {default_assignparam : {category : {param : value}}}
      elif entry_1 == 'default_assignparam':
        for entry_2 in assign_param[entry_1]:
          if 'protected_feature' in assign_param[entry_1][entry_2] \
          and assign_param[entry_1][entry_2]['protected_feature'] is not False:
            protected_features.append(assign_param[entry_1][entry_2]['protected_feature'])

      #target specification {category : {column : {param : value}}}
      else:
        for entry_2 in assign_param[entry_1]:
          if 'protected_feature' in assign_param[entry_1][entry_2] \
          and assign_param[entry_1][entry_2]['protected_feature'] is not False:
            protected_features.append(assign_param[entry_1][entry_2]['protected_feature'])

    #user specified processdict may have protected_feature specifciation in defaultparams
    for root in processdict:
      if 'defaultparams' in processdict[root]:
        if 'protected_feature' in processdict[root]['defaultparams']:
          if processdict[root]['defaultparams']['protected_feature'] is not False:
            protected_features.append(processdict[root]['defaultparams']['protected_feature'])

    #consolidate redundancies
    #these are sorted outside of this function for consistent order
    protected_features = list(set(protected_features))

    return protected_features

  def __messy_data_prep(self, protected_features, assigncat, assign_param, postprocess_dict, printstatus):
    """
    messy_data refers to non tidy feature sets with more than one column per feature
    we'll have convention that to designate column aggregations to treat as single feature
    user can pass a set of input headers to assigncat as a {set} of headers in place of a string
    
    this funciton serves multiple purposes assocaited with the messy_data convenion
    assigncat is inspected to identify messy specification as sets of strings within the specification list
    and replaces messy sets in assigncat with one of those entries
    
    the other entries from a messy set are added to protected_features
    and also added to assigncat['null'] 
    
    and also added to assign_param['messy_data_headers']
    in the form
    {inputcolumn : {messy_set}}
    
    which is accessed in __grab_params
    where the root category is used to selectively pass to tree categories for that root
    as params['messy_data_headers'] = messy_set
    and also with params['inplace'] = False
    
    then with the protected features the others are prepared in a seperate for loop and deleted with the null
    """

    messy_data_headers_dict = {}
    add_to_null_set = set()
    already_populated_set = set()
    
    messy_set_overlap_valresult = False
    
    #inspect assigncat to identify messy sets
    #convention is is user assigned a single set they are embeded in list brackets in __assigncat_str_convert
    for rootcat in assigncat:
      
      current_list = assigncat[rootcat]
      
      for current_list_set in current_list:
        
        if isinstance(current_list_set, set):
          
          if len(current_list_set - already_populated_set) < len(current_list_set):
            messy_set_overlap_valresult = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "Warning of potential error from overlapping messy_data set specifications in assigncat")
            self.__autoprint('warning', '')   
          already_populated_set = already_populated_set | current_list_set
          
          #access element from set
          for inputcolumn in sorted(list(current_list_set)):
            break
          
          #populate messy_data_headers_dict
          messy_data_headers_dict.update({inputcolumn : sorted(list(current_list_set))})
    
          #add entries to protected_features and assigncat[null]
          for current_list_set_entry in current_list_set:
          
            if current_list_set_entry != inputcolumn:
              
              protected_features = protected_features + [current_list_set_entry]
              
              add_to_null_set = add_to_null_set | {current_list_set_entry}
              
          #replace the asisgncat set specification with the excerpt
          assigncat[rootcat][current_list.index(current_list_set)] = inputcolumn
              
    #now add messy_data_headers_dict to assign_param
    assign_param.update({'messy_data_headers' : messy_data_headers_dict})
    
    #then the null specifications of the new protected_features are added to assigncat
    if 'null' not in assigncat:
      assigncat.update({'null':list(add_to_null_set)})
    else:
      assigncat['null'] = assigncat['null'] + list(add_to_null_set)

    return protected_features, assigncat, assign_param, messy_set_overlap_valresult

  def __apply_processfamily(self,
                            df_train,
                            df_test,
                            columns_train,
                            postprocess_dict,
                            transform_dict,
                            mirror_dict,
                            assigncat,
                            inverse_assigncat,
                            final_assigncat,
                            assignnan,
                            masterNArows_train,
                            masterNArows_test,
                            randomseed,
                            eval_ratio,
                            numbercategoryheuristic,
                            powertransform,
                            evalcat,
                            drift_dict,
                            assign_param,
                            printstatus,
                            inplace,
                            logger,
                            labels = False,
                           ):
    """
    Aggregates the processfamily functions and surrouding actions
    Including evalcategory, getNArows, populating mirror_dict, etc
    the labels case is to distinguish between applying function to features or labels
    """

    #For each column, perform processing 
    #based on either category assignments in assigncat 
    #or under automation based on train set feature evaluation in _evalcategory
    for column in columns_train:
      #
      categorycomplete = False

      if bool(assigncat) is True:
    
        if column in inverse_assigncat:
        
          category = inverse_assigncat[column]
          categorycomplete = True

          #printout display progress
          # if printstatus is True:
          if labels is False:
            self.__autoprint('debug', "evaluating column: ", column)
          else:
            self.__autoprint('debug', "evaluating label column: ", column)

          #assigncat has special case, can assign distinct columns to automated evaluation
          #if user assigned column to 'eval' or 'ptfm'
          #such as to perform eval when default is powertransform or visa versa
          #with _evalcategory distinction based on temp_powertransform_for_evalcategory_call
          if category in {'eval', 'ptfm'}:

            if category == 'eval':
              temp_powertransform_for_evalcategory_call = False
            if category == 'ptfm':
              temp_powertransform_for_evalcategory_call = True

            if evalcat is False:
              category = self.__evalcategory(df_train, column, randomseed, eval_ratio, \
                                           numbercategoryheuristic, temp_powertransform_for_evalcategory_call, labels)
            elif callable(evalcat):
              category = evalcat(df_train, column, randomseed, eval_ratio, \
                                 numbercategoryheuristic, temp_powertransform_for_evalcategory_call, labels)
      #
      if categorycomplete is False:

        #printout display progress
        # if printstatus is True:
        if labels is False:
          self.__autoprint('debug', "evaluating column: ", column)
        else:
          self.__autoprint('debug', "evaluating labels column: ", column)

        if evalcat is False:
          category = self.__evalcategory(df_train, column, randomseed, eval_ratio, \
                                       numbercategoryheuristic, powertransform, labels)
        elif callable(evalcat):
          category = evalcat(df_train, column, randomseed, eval_ratio, \
                             numbercategoryheuristic, powertransform, labels)
          
        #populate the result in the final_assigncat as informational resource
        if category in final_assigncat:
          final_assigncat[category].append(column)
        else:
          final_assigncat.update({category:[column]})

      #Previously had a few methods here to validate consistensy of data between train
      #and test sets. Found it was introducing too much complexity and was having trouble
      #keeping track of all the edge cases. So let's just make outright assumption that
      #test data if passed is consistently formatted as train data (for now)
      #added benefit that this reduces running time

      ##
      #to support the postprocess_dict entry below, let's first create a temp
      #list of columns
      #using a list instead of set here to maintain order, even though set would be a little quicker
      templist1 = list(df_train)
      #tempset1 is similar to templist1, but is based on column_dict so includes derivations that were subject to replacement
      tempset1 = set(postprocess_dict['column_dict'])

      #Before calling getNArows, we'll allow user to designate either by category or column 
      #designated source column values that will be converted to nan for treatment as infill
      #where in case of specification redundancy column designation takes precedence
      #and where category is referring to the root category associated with a column
      #and global just means this value treated universally as nan
      #where values are passed in automunge(.) parameter assignnan e.g.
      #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}

      df_train = self.__assignnan_convert(df_train, column, category, assignnan, postprocess_dict)
      df_test = self.__assignnan_convert(df_test, column, category, assignnan, postprocess_dict)

      #we also have convention that infinity values are by default subjected to infill
      #based on understanding that ML libraries in general do not accept thesae kind of values
      #as well as the python None value

      convert_to_nan_list = [np.inf, -np.inf, None, float("NaN")]
      df_train = self.__convert_to_nan(df_train, column, category, postprocess_dict, convert_to_nan_list)
      df_test = self.__convert_to_nan(df_test, column, category, postprocess_dict, convert_to_nan_list)
      
      if labels is False:

        #create NArows (column of True/False where True coresponds to missing data)
        trainNArows, drift_dict = self.__getNArows(df_train, column, category, postprocess_dict, drift_dict=drift_dict, driftassess=True)
        testNArows = self.__getNArows(df_test, column, category, postprocess_dict)

        #now append that NArows onto a master NA rows df
        #these df's are used to support application of infill 
        #such as for partitioning data sets for ML infill and identifying infill targets
        masterNArows_train = pd.concat([masterNArows_train, trainNArows], axis=1)
        masterNArows_test = pd.concat([masterNArows_test, testNArows], axis=1)

      #printout display progress
      # if printstatus is True:
      if labels is False:
        self.__autoprint('debug', "processing column: ", column)
      else:
        self.__autoprint('debug', "processing labels column: ", column)
      self.__autoprint('debug', "    root category: ", category)

      ##
      #now process family
      df_train, df_test, postprocess_dict = \
      self.__processfamily(df_train, df_test, column, category, \
                        transform_dict, postprocess_dict, assign_param, logger)

      ##
      #now delete columns that were subject to replacement when inplace transform wasn't available
      df_train, df_test, postprocess_dict = \
      self.__circleoflife(df_train, df_test, column, category, \
                        transform_dict, postprocess_dict, templist1)
      ##
      #here's another templist to support the postprocess_dict entry below
      templist2 = list(df_train)
      tempset2 = set(postprocess_dict['column_dict'])

      #ok now we're going to pick one of the new entries in templist2 to serve 
      #as a "columnkey" for pulling datas from the postprocess_dict down the road
      #similar to but retains order: columnkeylist = list(set(templist2) - set(templist1))
      columnkeylist = []
      for templist2_entry in templist2:
        if templist2_entry not in templist1:
          columnkeylist.append(templist2_entry)

      #allderivedlist is similar to columnkeylist but includes headers of derivations that were subject to replacement
      allderivedlist = tempset2 - tempset1
      allderivedlist = \
      self.__list_sorting(columnkeylist, allderivedlist)

      #an arbitrary columnkey is populated in postprocess_dict['origcolumn'] with columnkeylist
      if len(columnkeylist) == 0:
        columnkey = column
      else:
        columnkey = columnkeylist[0]

      ##
      if labels is False:
        origcolumn_type = 'train'
      else:
        origcolumn_type = 'label'
        
      postprocess_dict['origcolumn'].update({column : {'type' : origcolumn_type, \
                                                       'category' : category, \
                                                       'columnkeylist' : columnkeylist, \
                                                       'allderivedlist' : allderivedlist, \
                                                       'columnkey' : columnkey}})

      #populate mirror_dict, which will serve as the returned form of transform_dict and process_dict returned in postprocess_dict
      #mirror_dict is used so that user preserves privacy of custom entries that were not inspected
      mirror_dict, _1 = \
      self.__populate_labelsencoding_dict_support2(mirror_dict, postprocess_dict, transform_dict, category, 0)

      ##
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', " returned columns:")
      self.__autoprint('debug', postprocess_dict['origcolumn'][column]['columnkeylist'])
      self.__autoprint('debug', "")

      if inplace is not True:
        #this defragments the dataframes
        df_train = df_train.copy()
        df_test = df_test.copy()
        
    #one more populate mirror_dict for mlti categories which are categories inspected outside of a root category tree such as for mlti transform
    for mlti_entry in postprocess_dict['mlti_categories']:
      mirror_dict, _1 = \
      self.__populate_labelsencoding_dict_support2(mirror_dict, postprocess_dict, transform_dict, mlti_entry, 0)

    return df_train, \
           df_test, \
           postprocess_dict, \
           mirror_dict, \
           final_assigncat, \
           masterNArows_train, \
           masterNArows_test

  def __prepare_labelsencoding_dict(self, postprocess_dict, processdict, transform_dict, mirror_dict, labels_column_listofcolumns, printstatus, logger):
    """
    labelsencoding_dict is used to support public label inversion with enncryption
    in the privacy_encode = False or True case
    note that we use the term labelsencoding_dict in a few support funcitons
    that also serve other purposes for populating the "mirror_dict"
    the purpose of labelsencoding_dict is to extract portions of the column_dict
    relavant to the label features
    including also relevant entries for transform_dict and process_dict
    and any other postprocess_dict entries needed to support label inversion
    """

    labelsencoding_dict = {'transforms' : {}}
    
    for column in labels_column_listofcolumns:
      
      category = postprocess_dict['origcolumn'][column]['category']
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']

      #labelsencoding_dict is returned in postprocess_dict
      #this is redundant with information stored in postprocess_dict['column_dict']
      #labelsencoding_dict is used for public label inversion without encryption key
      #first we'll populate some entries and then final aggregations performed after fully populating postprocess_dict
      labelsencoding_dict['transforms'].update({column : {category:{}}})
      for finalcolumn_label in columnkeylist:

        #this populates a labelsnormalization_dict as {returnedcolumn : {}}
        #which later in the workflow are translated to column_dict entries in __populate_labelsencoding_dict_support3
        labelsencoding_dict['transforms'][column][category].update({finalcolumn_label : {}})

        #this access any upstream normalization_dict's for columns that were subject to replacement
        inputcolumn = postprocess_dict['column_dict'][finalcolumn_label]['inputcolumn']
        labelsencoding_dict = self.__populate_labelsencoding_dict_support(labelsencoding_dict, postprocess_dict, column, inputcolumn)

      #now populate any transform_dict or process_dict entries that were inspected for label transforms
      labelsencoding_dict, _0 = \
      self.__populate_labelsencoding_dict_support2(labelsencoding_dict, postprocess_dict, transform_dict, category, 0)

      #now that we know the root label category, we'll verify that if this was a custom processdict entry
      #it either includes a labelctgy entry or we'll otherwise populate one based on family tree
      #returning any update in postprocess_dict['process_dict']
      postprocess_dict, mirror_dict, check_processdict3_valresult, check_processdict3_validlabelctgy_valresult = \
      self.__check_processdict3(category, processdict, postprocess_dict, transform_dict, mirror_dict, printstatus)

      logger_dict['validations'].update({'check_processdict3_valresult' : check_processdict3_valresult,
                                                              'check_processdict3_validlabelctgy_valresult' : check_processdict3_validlabelctgy_valresult})
      logger = self.__logger_append(logger)
      
    return labelsencoding_dict, mirror_dict, postprocess_dict

  #__FunctionBlock: suffix overlap and defaultinfill support functions

  def __apply_defaultinfill(self, df, suffixcolumn, postprocess_dict, treecategory = False, defaultinfill_dict = False):
    """
    Applies default infill based on defaultinfill processdict entry
    Where if defaultinfill not populated defers to adjinfill
    
    This function can optionally be called internal to the transformation functions
    In dual/single/post process convention
    Where it is intended for application after suffixcolumn creation
    And after data type casting based on NArowtype
    
    Accepts input of train or test data as df
    suffixcolumn is the target column
    postprocess_dict is used to access process_dict
    treecategory is the tree category associated with the transform, which can be passed as False within postprocess functions
    (treecategory won't be inspected when defaultinfill_dict is populated)
    defaultinfill_dict passed as False for train data, test data receives entries populated from train data
    
    note that defaultinfill_dict should be externally stored in normalization_dict
    
    If custom default infill desired for a transform can designate naninfill (or simply omit this function call)
    
    defaultinfill support / default entry for a transform 
    will be noted in the familytrees process_dict population

    note that zeroinfill and oneinfill may not return a final 0/1 value if further operations applied in transform
    for final returned 0/1 can use assigninfill

    note that negzeroinfill intended for use in qbt1 family of transforms
    
    note that when column is pandas category dtype if an infill value not already a registered entry we add it with add_categories
    """    

    #if this is test data then we are expecting a populated defaultinfill_dict from the train data
    #otherwise we'll initialize
    
    #which will default to adjacent cell infill unless otherwise specified in processdict
    #here for simplicity we'll run through each defaultinfill twice
    #first if is for test data, the else will be for train data
    
    traindata = False
    if defaultinfill_dict is not False and 'defaultinfill' in defaultinfill_dict:
      
      traindata = True
      
      defaultinfill = defaultinfill_dict['defaultinfill']
      
      if defaultinfill in {'adjinfill'}:

        #apply ffill to replace NArows with value from adjacent cell in preceding row
        df[suffixcolumn] = df[suffixcolumn].fillna(method='ffill')

        #we'll follow with a bfill just in case first row had a nan
        df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')
        
      elif defaultinfill in {'interpinfill'}:
        
        df[suffixcolumn] = pd.to_numeric(df[suffixcolumn], errors='coerce').interpolate(method='linear', 
                                                                                        axis=0, 
                                                                                        limit=None, 
                                                                                        inplace=False,
                                                                                        limit_direction='forward',
                                                                                        limit_area=None,
                                                                                        downcast=None,
                                                                                       )
        
        #we'll follow with a adj just in case first or last row had a nan
        df[suffixcolumn] = df[suffixcolumn].fillna(method='ffill')
        df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')

      elif defaultinfill in {'meaninfill'}:

        infill_mean = defaultinfill_dict['infill_mean']
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_mean not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_mean])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_mean)

      elif defaultinfill in {'medianinfill'}:

        infill_median = defaultinfill_dict['infill_median']
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_median not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_median])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_median)

      elif defaultinfill in {'modeinfill'}:

        infill_mode = defaultinfill_dict['infill_mode']
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_mode not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_mode])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_mode)

      elif defaultinfill in {'lcinfill'}:

        infill_lc = defaultinfill_dict['infill_lc']
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_lc not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_lc])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_lc)
        
      elif defaultinfill in {'zeroinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 0 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([0])

        df[suffixcolumn] = df[suffixcolumn].fillna(0)

      elif defaultinfill in {'oneinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 1 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([1])
        
        df[suffixcolumn] = df[suffixcolumn].fillna(1)

      elif defaultinfill in {'negzeroinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if -0. not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([-0.])

        df[suffixcolumn] = df[suffixcolumn].fillna(-0.)

      elif defaultinfill in {'naninfill'}:
        #naninfill is intended for cases when user wishes to apply a custom default infill inside of the transform
        pass

      #finally if prior infill still resulted in nan we'll just plug with 0
      if defaultinfill not in {'naninfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 0 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([0])
        
        df[suffixcolumn] = df[suffixcolumn].fillna(0)

    #else if this is train data we'll also populate the defaultinfill_dict
    else:
      
      #we'll apply a default infill
      #contingent on the 'NArowtype' of the tree category serving as basis for this transform
      NArowtype = postprocess_dict['process_dict'][treecategory]['NArowtype']

      defaultinfill = 'adjinfill'
      if 'defaultinfill' in postprocess_dict['process_dict'][treecategory]:
        if isinstance(postprocess_dict['process_dict'][treecategory]['defaultinfill'], str) \
        and postprocess_dict['process_dict'][treecategory]['defaultinfill'] \
        in {'adjinfill', 'interpinfill', 'meaninfill', 'medianinfill', 'modeinfill', 'lcinfill', 'zeroinfill', 'oneinfill', 'negzeroinfill', 'naninfill'}:
          defaultinfill = postprocess_dict['process_dict'][treecategory]['defaultinfill']

      #a few special cases to accomodate NArowtype / defaultinfill compatibilities

      #'meaninfill' and 'medianinfill' intended for numeric data, if not a numeric NArowtype adjinfill applied
      if defaultinfill in {'meaninfill', 'medianinfill', 'interpinfill'}:
        if NArowtype not in {'numeric', 'integer', 'positivenumeric', 'nonnegativenumeric', 'nonzeronumeric'}:
          defaultinfill = 'adjinfill'

      #'datetime' NArowtype only accepts adjinfill or naninfill
      if NArowtype in {'datetime'}:
        if defaultinfill not in {'adjinfill', 'naninfill'}:
          defaultinfill = 'adjinfill'

      #no infill performed for 'exclude' or 'totalexclude' NArowtype
      if NArowtype in {'exclude', 'totalexclude'}:
        defaultinfill = 'naninfill'

      #initialize a dictionary to pass default infill parameters between automunge and postmunge in column_dict
      defaultinfill_dict = {'defaultinfill' : defaultinfill}
    
      if defaultinfill in {'adjinfill'}:

        #apply ffill to replace NArows with value from adjacent cell in preceding row
        df[suffixcolumn] = df[suffixcolumn].fillna(method='ffill')

        #we'll follow with a bfill just in case first row had a nan
        df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')
        
      elif defaultinfill in {'interpinfill'}:
        
        df[suffixcolumn] = pd.to_numeric(df[suffixcolumn], errors='coerce').interpolate(method='linear', 
                                                                                        axis=0, 
                                                                                        limit=None, 
                                                                                        inplace=False,
                                                                                        limit_direction='forward',
                                                                                        limit_area=None,
                                                                                        downcast=None,
                                                                                       )
        
        #we'll follow with a adj just in case first or last row had a nan
        df[suffixcolumn] = df[suffixcolumn].fillna(method='ffill')
        df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')

      elif defaultinfill in {'meaninfill'}:

        infill_mean = df[suffixcolumn].mean()
        defaultinfill_dict.update({'infill_mean' : infill_mean})
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_mean not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_mean])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_mean)

      elif defaultinfill in {'medianinfill'}:

        infill_median = df[suffixcolumn].median()
        defaultinfill_dict.update({'infill_median' : infill_median})
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_median not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_median])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_median)

      elif defaultinfill in {'modeinfill'}:

        infill_mode = df[suffixcolumn].mode()

        if len(infill_mode) > 0:
          infill_mode = infill_mode[0]
        else:
          infill_mode = 0

        defaultinfill_dict.update({'infill_mode' : infill_mode})
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_mode not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_mode])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_mode)

      elif defaultinfill in {'lcinfill'}:

        #(asdf as used here is just an arbitrary string)
        tempstring = 'asdf'
        if tempstring in df:
          while tempstring in df:
            tempstring += 'z'
        #defaultinfill of lcinfill inserts infill as dtype str when bytes entries are present 
        #to avoid edge case for bytes type entries with value_counts sorting
        if (df[suffixcolumn] == df[suffixcolumn].astype(bytes)).sum() > 0:
          mode_valuecounts_list = pd.DataFrame(df[suffixcolumn].astype(str).value_counts())
        else:
          mode_valuecounts_list = pd.DataFrame(df[suffixcolumn].value_counts())
        mode_valuecounts_list = mode_valuecounts_list.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
        mode_valuecounts_list = list(mode_valuecounts_list.index)

        if len(mode_valuecounts_list) > 0:
          infill_lc = mode_valuecounts_list[-1]
        else:
          infill_lc = 0

        defaultinfill_dict.update({'infill_lc' : infill_lc})
        
        if df[suffixcolumn].dtype.name == 'category':
          if infill_lc not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([infill_lc])

        df[suffixcolumn] = df[suffixcolumn].fillna(infill_lc)

      elif defaultinfill in {'zeroinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 0 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([0])

        #note that transform may not return a 0 value, for final returned 0 use assigninfill
        df[suffixcolumn] = df[suffixcolumn].fillna(0)

      elif defaultinfill in {'oneinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 1 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([1])

        #note that transform may not return a 1 value, for final returned 1 use assigninfill
        df[suffixcolumn] = df[suffixcolumn].fillna(1)

      elif defaultinfill in {'negzeroinfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if -0. not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([-0.])

        df[suffixcolumn] = df[suffixcolumn].fillna(-0.)

      elif defaultinfill in {'naninfill'}:
        #naninfill is intended for cases when user wishes to apply a custom default infill inside of the transform
        #note an adjinfill is applied later in this function, for final returned nan use assigninfill
        pass

      #finally if prior infill still resulted in nan we'll just plug with 0
      if defaultinfill not in {'naninfill'}:
        
        if df[suffixcolumn].dtype.name == 'category':
          if 0 not in df[suffixcolumn].cat.categories:
            df[suffixcolumn] = df[suffixcolumn].cat.add_categories([0])

        df[suffixcolumn] = df[suffixcolumn].fillna(0)
    
    return df, defaultinfill_dict

  def __set_indexcolumn(self, trainID_column, testID_column, application_number, indexcolumn_base = 'Automunge_index'):
    """
    #this either sets indexcolumn as 'Automunge_index' 
    #or 'Automunge_index_' + str(application_number) if 'Automunge_index' is already in ID sets
    #(this helps with a rare potential workflow when data sets are repeatedly run through automunge)
    """

    indexcolumn_valresult = False

    fullset = set()
    
    indexcolumn = indexcolumn_base
    if isinstance(trainID_column, list):
      fullset = fullset | set(trainID_column)
      if indexcolumn_base in trainID_column:
        indexcolumn = indexcolumn_base + '_' + str(application_number)
    elif indexcolumn_base == trainID_column:
      fullset = fullset | {trainID_column}
      indexcolumn = indexcolumn_base + '_' + str(application_number)
        
    testindexcolumn = indexcolumn_base
    if isinstance(testID_column, list):
      fullset = fullset | set(testID_column)
      if indexcolumn_base in testID_column:
        testindexcolumn = indexcolumn_base + '_' + str(application_number)
    elif indexcolumn_base == testID_column:
      fullset = fullset | {testID_column}
      testindexcolumn = indexcolumn_base + '_' + str(application_number)
      
    if testindexcolumn != indexcolumn:
      if len(testindexcolumn) > len(indexcolumn):
        indexcolumn = testindexcolumn

    #this is a very remote edge case, just being comprehensive
    if indexcolumn in fullset:
      
      while indexcolumn in fullset:
        indexcolumn = indexcolumn + ','

    if indexcolumn != indexcolumn_base:
      indexcolumn_valresult = True
    
    return indexcolumn, indexcolumn_valresult

  def __set_Binary_column(self, postprocess_dict, suffixrange, root = 'Binary_'):
    """
    #this support function used in Binary dimensionality reduction
    #which Binary and PCA are unique in library in that they create new column headers by means other than suffix appention
    #so we will compare the root of the new column
    #which for Binary will be 'Binary_'
    #to input columns logged in postprocess_dict['origcolumn']
    #and returned columns logged in postprocess_dict['column_dict']
    #and if overlap found add an appender to the root as string of the application_number
    #which is a 12 digit integer derived by random sample for each automunge(.) call
    #similar to what is done for Automunge_index returned in ID sets
    #note that we'll also check for overlaps between potential configurations of root plus suffix
    #if overlap found a validation result will be returned
    #suffixrange is an integer >= 0 which sets depth of how many overlaps will be checked
    #which we know the number of returned columns from Binary will be <= a value as function of nunique
    """

    #root column will be base of the new column header
    rootcolumn = root
    set_Binary_column_valresult = False
    
    #___
    
    #permutations are potential derivaitons from the root based on different Binary options
    #note that to ensure comprehensive, we set a suffixrange depth of inspection as per _Binary_convert
    permutations = {rootcolumn, rootcolumn+'_ord3'}
    for i in range(suffixrange):
      _1010_permutation = rootcolumn + '_1010_' + str(i)
      _onht_permutation = rootcolumn + '_onht_' + str(i)
      permutations = permutations | {_1010_permutation, _onht_permutation}
      
    origcolumns = set(postprocess_dict['origcolumn'])
    returnedcolumns = set(postprocess_dict['column_dict'])
    fullcolumns = origcolumns | returnedcolumns
    
    #if overlap between fullcolumns and root and permutations add application number appender
    if len(permutations & fullcolumns) > 0:
      rootcolumn = rootcolumn + str(postprocess_dict['application_number'])
      set_Binary_column_valresult = True
      
    #___
    
    if set_Binary_column_valresult is True:
      #now one more check to be comprehensive, this is for a very remote edge case
      permutations = {rootcolumn, rootcolumn+'_ord3'}
      for i in range(suffixrange):
        _1010_permutation = rootcolumn + '_1010_' + str(i)
        _onht_permutation = rootcolumn + '_onht_' + str(i)
        permutations = permutations | {_1010_permutation, _onht_permutation}

      if len(permutations & fullcolumns) > 0:
        while len(permutations & fullcolumns) > 0:

          rootcolumn = rootcolumn + ','

          permutations = {rootcolumn, rootcolumn+'_ord3'}
          for i in range(suffixrange):
            _1010_permutation = rootcolumn + '_1010_' + str(i)
            _onht_permutation = rootcolumn + '_onht_' + str(i)
            permutations = permutations | {_1010_permutation, _onht_permutation}
          
    #___
    
    return rootcolumn, set_Binary_column_valresult

  def __set_PCA_column(self, newcolumncount, postprocess_dict, root='PCA_'):
    """
    #this support function used in PCA dimensionality reduction
    #which Binary and PCA are unique in library in that they create new column headers by means other than suffix appention
    #so we will compare the root of the new column
    #which for PCA will be 'PCA_'
    #to input columns logged in postprocess_dict['origcolumn']
    #and returned columns logged in postprocess_dict['column_dict']
    #and if overlap found add an appender to the root as string of the application_number
    #which is a 12 digit integer derived by random sample for each automunge(.) call
    #similar to what is done for Automunge_index returned in ID sets
    #if overlap found a validation result will be returned
    """

    #the returned column names check for overlap with existing columns to accomodate edge case
    origcolumns = set(postprocess_dict['origcolumn'])
    returnedcolumns = set(postprocess_dict['column_dict'])
    fullcolumns = origcolumns | returnedcolumns
    PCA_columns_valresult = False
    
    #generate a list of column names for the conversion to pandas
    columnnames = [root + '_' + str(y) for y in range(newcolumncount)]
    
    #if overlap found, add the application_number to column headers
    if len((set(columnnames) | {root}) & fullcolumns) > 0:
      PCA_columns_valresult = True
      
      columnnames = [(root + str(postprocess_dict['application_number']) + '_' + str(y)) for y in range(newcolumncount)]
      
    #one more to be comprehensive, this is for a very remote edge case
    if PCA_columns_valresult is True:
      i=1
      if len((set(columnnames) | {root}) & fullcolumns) > 0:
        while len(set(columnnames) & fullcolumns) > 0:
          columnnames = [(root + str(postprocess_dict['application_number']) + i*',' + '_' + str(y)) for y in range(newcolumncount)]
          i+=1
        
    return columnnames, PCA_columns_valresult

  def __df_copy_train(self, df_train, column, newcolumn, suffixoverlap_results = {}, printstatus = False):
    """
    #performs a copy operation to add column to a df_train
    #Before any new columns added to df_train
    #checks that they are not already present in df_train
    #if so returns error message and logs in suffixoverlap_results
    """

    #test for overlap error
    if newcolumn in df_train.columns:
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "*****************")
      self.__autoprint('warning', "Warning of suffix overlap error")
      self.__autoprint('warning', "When creating new column: ", newcolumn)
      self.__autoprint('warning', "The column was already found present in df_train headers.")
      self.__autoprint('warning', "")
      self.__autoprint('warning', "Some potential quick fixes for this error include:")
      self.__autoprint('warning', "- rename columns to integers before passing to automunge(.)")
      self.__autoprint('warning', "- strip underscores '_' from column header titles.")
      self.__autoprint('warning', "(convention is all suffix appenders include an underscore)")
      self.__autoprint('warning', "")
      self.__autoprint('warning', "Please note any updates to column headers will need to be carried through to assignment parameters.")
      self.__autoprint('warning', "*****************")
      self.__autoprint('warning', "")
      
      suffixoverlap_results.update({newcolumn : True})
      
    else:
      
      df_train[newcolumn] = df_train[column].copy()
      
      suffixoverlap_results.update({newcolumn : False})
    
    return df_train, suffixoverlap_results

  def __df_check_suffixoverlap(self, df_train, newcolumns, suffixoverlap_results = {}, printstatus = False):
    """
    #checks that newcolumns list are not already present in df_train
    #logs in suffixoverlap_results

    #we'll have convention that df_train can either be passed as the dataframe
    #or alternatively as a list of column headers
    """

    #this will result in a set of column headers for both cases (if df_train passed as list or dataframe)
    origcolumns = set(df_train)

    if not isinstance(newcolumns, list):
      newcolumns = [newcolumns]
    
    for newcolumn in newcolumns:
      
      if newcolumn in origcolumns:
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "*****************")
        self.__autoprint('warning', "Warning of suffix overlap error")
        self.__autoprint('warning', "When creating new column: ", newcolumn)
        self.__autoprint('warning', "The column was already found present in df_train headers.")
        self.__autoprint('warning', "")
        self.__autoprint('warning', "Some potential quick fixes for this error include:")
        self.__autoprint('warning', "- rename columns to integers before passing to automunge(.)")
        self.__autoprint('warning', "- strip underscores '_' from column header titles.")
        self.__autoprint('warning', "(convention is all suffix appenders include an underscore)")
        self.__autoprint('warning', "")
        self.__autoprint('warning', "Please note any updates to column headers will need to be carried through to assignment parameters.")
        self.__autoprint('warning', "*****************")
        self.__autoprint('warning', "")

        suffixoverlap_results.update({newcolumn : True})

      else:

        suffixoverlap_results.update({newcolumn : False})
        
    return suffixoverlap_results

  def __suffix_overlap_final_aggregation_and_printouts(self, postprocess_dict, logger_dict, logger):
    """
    #Performs a final round of printouts in case of identified suffix overlap error
    #Also aggregates the validation results stored in column_dict
    #to those store with logging_dict
    
    #suffixoverlap_aggregated_result is added to give single boolean signal
    #for presence of suffix overlap error
    #(False is good)
    """

    suffixoverlap_aggregated_result = False
    
    #then at completion of automunge(.), aggregate the suffixoverlap results
    #and do an additional printout if any column overlap error to be sure user sees message
    for entry1 in postprocess_dict['column_dict']:
      for entry2 in postprocess_dict['column_dict'][entry1]['suffixoverlap_results']:
        if postprocess_dict['column_dict'][entry1]['suffixoverlap_results'][entry2] is True:
          
          suffixoverlap_aggregated_result = True
          # if postprocess_dict['printstatus'] != 'silent':
          self.__autoprint('warning', "*****************")
          self.__autoprint('warning', "Warning of suffix overlap error")
          self.__autoprint('warning', "When creating new column: ", entry2)
          self.__autoprint('warning', "The column was already found present in df_train headers.")
          self.__autoprint('warning', "")
          self.__autoprint('warning', "Some potential quick fixes for this error include:")
          self.__autoprint('warning', "- rename columns to integers before passing to automunge(.)")
          self.__autoprint('warning', "- strip underscores '_' from column header titles.")
          self.__autoprint('warning', "(convention is all suffix appenders include an underscore)")
          self.__autoprint('warning', "")
          self.__autoprint('warning', "Please note any updates to column headers will need to be carried through to assignment parameters.")
          self.__autoprint('warning', "*****************")
          self.__autoprint('warning', "")
      
      logger_dict['validations']['suffixoverlap_results'].update(
      postprocess_dict['column_dict'][entry1]['suffixoverlap_results'])
      logger = self.__logger_append(logger)

    #note that we automatically mitigate this channel now by revision of the PCA column root
    if 'PCA_suffixoverlap_results' in logger_dict['validations']:
      for entry1 in logger_dict['validations']['PCA_suffixoverlap_results']:
        if logger_dict['validations']['PCA_suffixoverlap_results'][entry1] is True:

            suffixoverlap_aggregated_result = True
            # if postprocess_dict['printstatus'] != 'silent':
            self.__autoprint('warning', "*****************")
            self.__autoprint('warning', "Warning of suffix overlap error")
            self.__autoprint('warning', "When creating PCA column: ", entry1)
            self.__autoprint('warning', "The column was already found present in df_train headers.")
            self.__autoprint('warning', "")
            self.__autoprint('warning', "Note that PCA returned columns are of form: PCAcol0")
            self.__autoprint('warning', "Where # is integer")
            self.__autoprint('warning', "This form of column header should be avoided in passed data.")
            self.__autoprint('warning', "")

    #note that we automatically mitigate this channel now by revision of the Binary column root
    if 'Binary_suffixoverlap_results' in logger_dict['validations']:
      for entry1 in logger_dict['validations']['Binary_suffixoverlap_results']:
        if logger_dict['validations']['Binary_suffixoverlap_results'][entry1] is True:

            suffixoverlap_aggregated_result = True
            # if postprocess_dict['printstatus'] != 'silent':
            self.__autoprint('warning', "*****************")
            self.__autoprint('warning', "Warning of suffix overlap error")
            self.__autoprint('warning', "When creating Binary column: ", entry1)
            self.__autoprint('warning', "The column was already found present in df_train headers.")
            self.__autoprint('warning', "")
            self.__autoprint('warning', "Note that Binary returned columns are of form: Binary_1010_#")
            self.__autoprint('warning', "Where # is integer")
            self.__autoprint('warning', "This error might have occured if you passed data including column header 'Binary' to '1010' transform")
            self.__autoprint('warning', "This form of column header should be avoided in passed data.")
            self.__autoprint('warning', "")

    if 'excl_suffixoverlap_results' in logger_dict['validations']:
      for entry1 in logger_dict['validations']['excl_suffixoverlap_results']:
        if logger_dict['validations']['excl_suffixoverlap_results'][entry1] is True:

            suffixoverlap_aggregated_result = True
            # if postprocess_dict['printstatus'] != 'silent':
            self.__autoprint('warning', "*****************")
            self.__autoprint('warning', "Warning of suffix overlap error")
            self.__autoprint('warning', "When removing '_excl' suffix for column: ", entry1)
            self.__autoprint('warning', "The column without suffix was already found present in df_train headers.")
            self.__autoprint('warning', "")
            
    logger_dict['validations'].update({'suffixoverlap_aggregated_result':suffixoverlap_aggregated_result})
    logger = self.__logger_append(logger)

    if 'indexcolumn_valresult' in logger_dict['validations'] \
    and logger_dict['validations']['indexcolumn_valresult'] is True:
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "please note that the Automunge_index returned in the ID sets had a header update to avoid overlap")
      self.__autoprint('warning', "updated Automunge_index header is: ", postprocess_dict['indexcolumn'])
      self.__autoprint('warning', '')

    if 'PCA_columns_valresult' in logger_dict['validations'] \
    and logger_dict['validations']['PCA_columns_valresult'] is True:
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "please note that the PCA derivation had a root header update to avoid overlap")
      self.__autoprint('warning', "returned PCA columns are: ", postprocess_dict['returned_PCA_columns'])
      self.__autoprint('warning', '')

    if 'set_Binary_column_valresult' in logger_dict['validations'] \
    and logger_dict['validations']['set_Binary_column_valresult'] is True:
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "please note that the Binary derivation had a root header update to avoid overlap")
      self.__autoprint('warning', "returned Binary columns are: ", postprocess_dict['returned_Binary_columns'])
      self.__autoprint('warning', "returned label Binary columns are: ", postprocess_dict['final_returned_labelBinary_columns'])
      self.__autoprint('warning', '')
    
    return postprocess_dict

  #__FunctionBlock: automunge dualprocess, singleprocess, and custom_train functions

  def _process_NArw(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton that creates a boolean column indicating 1 for rows
    #corresponding to missing or improperly formated data in source column
    #note this uses the NArows function which has a category specific approach
    #returns same dataframe with new column of name column + '_NArw'
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

    df[suffixcolumn] = self.__getNArows(df, column, category, postprocess_dict)

    #change NArows data type to 8-bit (1 byte) integers for memory savings
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)

    #create list of columns
    nmbrcolumns = [suffixcolumn]
    
    #for drift report
    pct_NArw = df[suffixcolumn].sum() / df[suffixcolumn].shape[0]

    #create normalization dictionary
    NArwnormalization_dict = {suffixcolumn : {'pct_NArw':pct_NArw, 'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        NArwnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : NArwnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : [nc], \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_numerical(self, mdf_train, mdf_test, column, category, \
                              treecategory, postprocess_dict, params = {}):
    '''
    #process_numerical(mdf_train, mdf_test, column, category)
    #function to normalize data to mean of 0 and standard deviation of 1 \
    #z score normalization) 
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_nmbr'
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    #initialize parameters
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
    
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #stdev_cap can be passed as integer or float for setting cap based on number of standard deviations from mean
    #only performed with cap=False and floor=False, default of stdev_cap=False deactivates
    if 'stdev_cap' in params:
      stdev_cap = params['stdev_cap']
    else:
      stdev_cap = False

    #abs_zero accepts boolean defaulting to True for conversion of negative zero to positive
    #which is in place to ensure defaultinfill of negzeroinfill returns distinct encoding
    if 'abs_zero' in params:
      abs_zero = params['abs_zero']
    else:
      abs_zero = True

    suffixcolumn = column + '_' + suffix

    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    #median and median absolute deviation
    median = mdf_train[suffixcolumn].median()
    # MAD = mdf_train[suffixcolumn].mad()
    MAD = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()
    
    #if cap/floor set based on number of standard deviations, measure statistics and set variables cap and/or floor
    if stdev_cap is not False and cap is False and floor is False:
      
      pre_capfloor_mean = mdf_train[suffixcolumn].mean()
      pre_capfloor_stdev = mdf_train[suffixcolumn].std()
      
      if pre_capfloor_mean != pre_capfloor_mean:
        pre_capfloor_mean = 0
      if pre_capfloor_stdev != pre_capfloor_stdev:
        pre_capfloor_stdev = 1

      cap = pre_capfloor_mean + stdev_cap * pre_capfloor_stdev
      floor = pre_capfloor_mean - stdev_cap * pre_capfloor_stdev
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
      
      mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[suffixcolumn] < floor, (suffixcolumn)] \
      = floor
      
      mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
      = floor

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()
    if mean != mean:
      mean = 0

    #subtract mean from column for both train and test
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] - mean
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - mean

    #get standard deviation of training data
    std = mdf_train[suffixcolumn].std()
    
    #special case, if standard deviation is 0 we'll set it to 1 to avoid division by 0
    if std == 0:
      std = 1

    #divide column values by std for both training and test data
    #offset, multiplier are parameters that defaults to zero, one
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] / std * multiplier + offset
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / std * multiplier + offset

    #replace any negative zero floats with positive zero. Negative zero is reserved for default infill
    if abs_zero is True:
      mdf_train = self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == 0, 0, specified='replacement')
      mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == 0, 0, specified='replacement')

    #apply defaultinfill based on processdict entry
    #this will default to negzeroinfill
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, 'std' : std, \
                                              'median' : median, 'MAD' : MAD, \
                                              'max' : maximum, 'min' : minimum, \
                                              'offset' : offset, 'multiplier': multiplier, \
                                              'cap' : cap, 'floor' : floor, \
                                              'abs_zero' : abs_zero, \
                                              'inplace' : inplace, 'suffix' : suffix,
                                              'defaultinfill_dict' : defaultinfill_dict}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_dxdt(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_dxdt(df, column, category, postprocess_dict)
    #function to translate a continues variable into a bounded variable
    #by taking delta of row from preceding row
    #assumes the rows are not shuffled and represent a continuous funciton 
    #with consistent time steps
    
    #for missing values, uses adjacent cell infill as default
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 1

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[suffixcolumn] = pd.to_numeric(df[suffixcolumn], errors='coerce')

    #apply defaultinfill based on processdict entry
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    #subtract preceding row
    df[suffixcolumn] = df[suffixcolumn] - df[suffixcolumn].shift(periods = periods)
    
    #first row will have a nan so just one more backfill
    df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[suffixcolumn].iat[0]
    if value != value:
      value = 0

      df[suffixcolumn] = df[suffixcolumn].fillna(value)
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[suffixcolumn] >= 0].shape[0] / df[suffixcolumn].shape[0]
    negativeratio = df[df[suffixcolumn] < 0].shape[0] / df[suffixcolumn].shape[0]
    zeroratio = df[df[suffixcolumn] == 0].shape[0] / df[suffixcolumn].shape[0]
    minimum = df[suffixcolumn].min()
    maximum = df[suffixcolumn].max()
    mean = df[suffixcolumn].mean()
    std = df[suffixcolumn].std()

    nmbrnormalization_dict = {suffixcolumn : {'positiveratio' : positiveratio, \
                                              'negativeratio' : negativeratio, \
                                              'zeroratio' : zeroratio, \
                                              'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'periods' : periods, \
                                              'inplace' : inplace,
                                              'defaultinfill_dict' : defaultinfill_dict}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _process_dxd2(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_dxd2(df, column, category, postprocess_dict)
    #function to translate a continues variable into a bounded variable
    #by taking delta of average of last two rows minus 
    #average of preceding two rows before that
    #should take a littel noise out of noisy data
    #assumes the rows are not shuffled and represent a continuous funciton 
    #with consistent time steps
    
    #for missing values, uses adjacent cell infill as default
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 2

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[suffixcolumn] = pd.to_numeric(df[suffixcolumn], errors='coerce')
    
    #apply defaultinfill based on processdict entry
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
#     #we're going to take difference of average of last two rows with two rows preceding
#     df[column + '_dxd2'] = (df[column + '_dxd2'] + df[column + '_dxd2'].shift()) / 2 \
#                            - ((df[column + '_dxd2'].shift(periods=2) + df[column + '_dxd2'].shift(periods=3)) / 2)

    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, [column + '_temp1'], suffixoverlap_results, postprocess_dict['printstatus'])

    df[column + '_temp1'] = df[suffixcolumn].copy()
    # df_train['number7_temp3'] = df_train['number7'].copy()

    for i in range(periods-1):
      df[column + '_temp1'] = df[column + '_temp1'] + df[suffixcolumn].shift(periods = i+1)

    df[suffixcolumn] = (df[column + '_temp1'] - df[column + '_temp1'].shift(periods = periods)) / periods
    
    #first row will have a nan so just one more backfill
    df[suffixcolumn] = df[suffixcolumn].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[suffixcolumn].iat[0]
    if value != value:
      value = 0

      df[suffixcolumn] = df[suffixcolumn].fillna(value)
    
    del df[column + '_temp1']
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxd2 without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[suffixcolumn] >= 0].shape[0] / df[suffixcolumn].shape[0]
    negativeratio = df[df[suffixcolumn] < 0].shape[0] / df[suffixcolumn].shape[0]
    zeroratio = df[df[suffixcolumn] == 0].shape[0] / df[suffixcolumn].shape[0]
    minimum = df[suffixcolumn].min()
    maximum = df[suffixcolumn].max()
    mean = df[suffixcolumn].mean()
    std = df[suffixcolumn].std()
  
    nmbrnormalization_dict = {suffixcolumn : {'positiveratio' : positiveratio, \
                                              'negativeratio' : negativeratio, \
                                              'zeroratio' : zeroratio, \
                                              'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'periods' : periods, \
                                              'inplace' : inplace,
                                              'suffix' : suffix,
                                              'defaultinfill_dict' : defaultinfill_dict}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _process_shft(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_shft(df, column, category, postprocess_dict)
    #function to shift a sequential set forward by one or more time steps    
    #for missing values, uses adjacent cell infill as default
    #accepts parameter 'periods' for number of time steps, defaults to one
    #accepts parameter 'suffix' for column suffix appender
    #such as may be useful if applying this transform to the same column more than once

    #shft family of transforms is special case in the library since an aggregated NArw
    #wouldn't align with missing data entries (as rows are shifted by some period)
    #since we need to set NArowtype to exclude (which prevents aggregation of missing data markers)
    #we will omit defaultinfill support which does not apply infill to exclude NArowtype
    #and apply adjinfill here in transform
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 1
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    shft_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, shft_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, shft_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : shft_column}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[shft_column] = pd.to_numeric(df[shft_column], errors='coerce')
    
    #shift from preceding row
    df[shft_column] = df[shft_column].shift(periods = periods)

    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[shft_column] = df[shft_column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[shft_column] = df[shft_column].fillna(method='bfill')

    #then a final infill in case data was non-numeric
    df[shft_column] = df[shft_column].fillna(0)

    #create list of columns
    nmbrcolumns = [shft_column]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[shft_column] >= 0].shape[0] / df[shft_column].shape[0]
    negativeratio = df[df[shft_column] < 0].shape[0] / df[shft_column].shape[0]
    zeroratio = df[df[shft_column] == 0].shape[0] / df[shft_column].shape[0]
    minimum = df[shft_column].min()
    maximum = df[shft_column].max()
    mean = df[shft_column].mean()
    std = df[shft_column].std()

    nmbrnormalization_dict = {shft_column :      {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods, \
                                                  'suffix' : suffix, \
                                                  'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _process_MADn(self, mdf_train, mdf_test, column, category, \
                              treecategory, postprocess_dict, params = {}):
    '''
    #process_MADn(mdf_train, mdf_test, column, category)
    #function to normalize data to mean of 0 and mean absolute deviation of 1
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_MADn'
    #note this is a "dualprocess" function since is applied to both train and test dataframes
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    #center can be passed as 'mean' or 'max', which serves as subtractor in numerator of normalization 
    #MADn applies mean, MAD3 applies max (max option inspired by discussions at Real World Risk Institute)
    if 'center' in params:
      center = params['center']
    else:
      center = 'mean'
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    median = mdf_train[suffixcolumn].median()

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean() 
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    if center == 'mean':

      #subtract mean from column for both train and test
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn] - mean
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - mean
      
    if center == 'max':
      
      if maximum != maximum:
        maximum = 0

      #subtract maximum from column for both train and test
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn] - maximum
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - maximum

    #get mean absolute deviation of training data
    # MAD = mdf_train[suffixcolumn].mad()
    MAD = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()
    
    #special case to avoid div by 0
    if MAD == 0:
      MAD = 1

    #divide column values by mad for both training and test data
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] / MAD
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / MAD

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, 'MAD' : MAD, 'center' : center, \
                                              'maximum':maximum, 'minimum':minimum, 'median' : median, \
                                              'inplace' : inplace, 'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_mnmx(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_mnmx(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #based on min/max values from training set for this column
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both
    #dataframe inputs
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #for cap ands floor, False means not applied, True means based on set's found max/min in train set
    
    #initialize parameters
    #cap can be passed as True for max of training data or as a specific value prior to normalizaiton, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalizaiton, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #stdev_cap can be passed as integer or float for setting cap based on number of standard deviations from mean
    #only performed with cap=False and floor=False, default of stdev_cap=False deactivates
    if 'stdev_cap' in params:
      stdev_cap = params['stdev_cap']
    else:
      stdev_cap = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[suffixcolumn].std()
    if std != std or std == 0:
      std = 1

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()   
    if mean != mean:
      mean = 0

    #additional drift stats
    median = mdf_train[suffixcolumn].median()
    # MAD = mdf_train[suffixcolumn].mad()
    MAD = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    
    #if cap/floor set based on number of standard deviations, measure statistics and set variables cap and/or floor
    if stdev_cap is not False and cap is False and floor is False:
      
      pre_capfloor_mean = mean
      pre_capfloor_stdev = std
      
      if pre_capfloor_mean != pre_capfloor_mean:
        pre_capfloor_mean = 0
      if pre_capfloor_stdev != pre_capfloor_stdev:
        pre_capfloor_stdev = 1

      cap = pre_capfloor_mean + stdev_cap * pre_capfloor_stdev
      floor = pre_capfloor_mean - stdev_cap * pre_capfloor_stdev
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0:
      maxminusmin = 1
    
    #perform min-max scaling to train and test sets using values from train
    mdf_train[suffixcolumn] = (mdf_train[suffixcolumn] - minimum) / \
                                  (maxminusmin)
    
    mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - minimum) / \
                                 (maxminusmin)

    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[suffixcolumn] > (cap - minimum)/maxminusmin, (suffixcolumn)] \
      = (cap - minimum)/maxminusmin
      
      mdf_test.loc[mdf_test[suffixcolumn] > (cap - minimum)/maxminusmin, (suffixcolumn)] \
      = (cap - minimum)/maxminusmin
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[suffixcolumn] < (floor - minimum)/maxminusmin, (suffixcolumn)] \
      = (floor - minimum)/maxminusmin
      
      mdf_test.loc[mdf_test[suffixcolumn] < (floor - minimum)/maxminusmin, (suffixcolumn)] \
      = (floor - minimum)/maxminusmin
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'maxminusmin' : maxminusmin, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'median' : median, \
                                              'MAD' : MAD, \
                                              'cap' : cap, \
                                              'floor' : floor, \
                                              'suffix' : suffix, \
                                              'inplace' : inplace, \
                                              'defaultinfill_dict' : defaultinfill_dict}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_mnm3(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_mnmx(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #after replacing extreme values above the 0.99 quantile with
    #the value of 0.99 quantile and extreme values below the 0.01
    #quantile with the value of 0.01 quantile
    #(accepts parameters qmax and qmin to customize these 0.99/0.01 values)
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    #initialize parameters
    if 'qmax' in params:
      qmax = params['qmax']
    else:
      qmax = 0.99
      
    if 'qmin' in params:
      qmin = params['qmin']
    else:
      qmin = 0.01

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[suffixcolumn].std()

    #get maximum value of training column
    quantilemax = mdf_train[suffixcolumn].quantile(qmax)
    
    if quantilemax != quantilemax:
      quantilemax = 0

    #get minimum value of training column
    quantilemin = mdf_train[suffixcolumn].quantile(qmin)
    
    if quantilemin != quantilemin:
      quantilemin = 0

    #replace values > quantilemax with quantilemax
    mdf_train.loc[mdf_train[suffixcolumn] > quantilemax, (suffixcolumn)] \
    = quantilemax
    mdf_test.loc[mdf_test[suffixcolumn] > quantilemax, (suffixcolumn)] \
    = quantilemax
    #replace values < quantile10 with quantile10
    mdf_train.loc[mdf_train[suffixcolumn] < quantilemin, (suffixcolumn)] \
    = quantilemin
    mdf_test.loc[mdf_test[suffixcolumn] < quantilemin, (suffixcolumn)] \
    = quantilemin

    #note this step is now performed after the quantile evaluation / replacement

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()    
    if mean != mean:
      mean = 0
    
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #avoid outlier div by zero when max = min
    maxminusmin = quantilemax - quantilemin
    if maxminusmin == 0:
      maxminusmin = 1

    #perform min-max scaling to train and test sets using values from train
    mdf_train[suffixcolumn] = (mdf_train[suffixcolumn] - quantilemin) / \
                                  (maxminusmin)

    mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - quantilemin) / \
                                 (maxminusmin)

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'quantilemin' : quantilemin, \
                                              'quantilemax' : quantilemax, \
                                              'maxminusmin' : maxminusmin, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'qmax' : qmax, \
                                              'qmin' : qmin, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict, \
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list

  def _process_mxab(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_mxab(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of -1 and maximum of 1 \
    #based on division by max absolute values from training set.
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[suffixcolumn].std()
    median = mdf_train[suffixcolumn].median()
    # MAD = mdf_train[suffixcolumn].mad()
    MAD = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()   
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    
    #get max absolute
    maxabs = max(abs(maximum), abs(minimum))
    
    #avoid outlier div by zero when max = min
    if maxabs == 0:
      maxabs = 1
    
    #perform maxabs scaling to train and test sets using values from train
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] / \
                                  (maxabs)
    
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / \
                                 (maxabs)
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'maxabs' : maxabs, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'median' : median, \
                                              'MAD' : MAD, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict, 
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_retn(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    """
    #process_retn(mdf_train, mdf_test, column, category)
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #replaces missing or improperly formatted data with mean of remaining values
    
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    
    #accepts divisor parameters of 'minmax' or 'std', eg divisor for normalization equation
    #note that standard deviation doesn't have same properties for sign retention when all values > or < 0
    if 'divisor' in params:
      divisor = params['divisor']
    else:
      divisor = 'minmax'
    
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
    
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
    
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #stdev_cap can be passed as integer or float for setting cap based on number of standard deviations from mean
    #only performed with cap=False and floor=False, default of stdev_cap=False deactivates
    if 'stdev_cap' in params:
      stdev_cap = params['stdev_cap']
    else:
      stdev_cap = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #a few more metrics collected for driftreport
    #stdev, median, mad
    std = mdf_train[suffixcolumn].std()
    median = mdf_train[suffixcolumn].median()
    # mad = mdf_train[suffixcolumn].mad()
    mad = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()
    
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
      
    if std != std or std == 0:
      std = 1
      
    if mad != mad or mad == 0:
      mad = 1
      
    #if cap/floor set based on number of standard deviations, measure statistics and set variables cap and/or floor
    if stdev_cap is not False and cap is False and floor is False:
      
      pre_capfloor_mean = mdf_train[suffixcolumn].mean()
      pre_capfloor_stdev = std
      
      if pre_capfloor_mean != pre_capfloor_mean:
        pre_capfloor_mean = 0
      if pre_capfloor_stdev != pre_capfloor_stdev:
        pre_capfloor_stdev = 1

      cap = pre_capfloor_mean + stdev_cap * pre_capfloor_stdev
      floor = pre_capfloor_mean - stdev_cap * pre_capfloor_stdev
      
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
      
      mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[suffixcolumn] < floor, (suffixcolumn)] \
      = floor
      
      mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
      = floor
      
    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #edge case (only neccesary so scalingapproach is assigned)
    if maximum != maximum:
      maximum = 0
    if minimum != minimum:
      minimum = 0
    
    #divisor
    if divisor not in {'minmax', 'std', 'mad'}:
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "Error: retn transform parameter 'divisor' only accepts entries of 'minmax' 'mad' or 'std'")
      self.__autoprint('warning', '')
    if divisor == 'minmax':
      divisor = maxminusmin
    elif divisor == 'mad':
      divisor = mad
    else:
      divisor = std
      
    if divisor == 0 or divisor != divisor:
      divisor = 1
    
    #driftreport metric scalingapproach returned as 'retn' or 'mnmx' or 'mxmn'
    #where mnmx is for cases where all values in train set are positive
    #mxmn is for cases where all values in train set are negative
    
    if maximum >= 0 and minimum <= 0:
      
      mdf_train[suffixcolumn] = (mdf_train[suffixcolumn]) / \
                                    (divisor) * multiplier + offset
      
      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn]) / \
                                    (divisor) * multiplier + offset
      
      scalingapproach = 'retn'
      
    elif maximum >= 0 and minimum >= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[suffixcolumn] = (mdf_train[suffixcolumn] - minimum) / \
                                    (divisor) * multiplier + offset

      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - minimum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mnmx'
      
    elif maximum <= 0 and minimum <= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[suffixcolumn] = (mdf_train[suffixcolumn] - maximum) / \
                                    (divisor) * multiplier + offset

      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - maximum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mxmn'

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'median' : median, \
                                              'MAD' : mad, \
                                              'scalingapproach' : scalingapproach, \
                                              'offset' : offset, \
                                              'multiplier': multiplier, \
                                              'cap' : cap, \
                                              'floor' : floor, \
                                              'divisor' : divisor, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict, 
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_mean(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_mean(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #based on min/max values from training set for this column
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both
    #dataframe inputs
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
      
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #stdev_cap can be passed as integer or float for setting cap based on number of standard deviations from mean
    #only performed with cap=False and floor=False, default of stdev_cap=False deactivates
    if 'stdev_cap' in params:
      stdev_cap = params['stdev_cap']
    else:
      stdev_cap = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #get maximum value of training column
    maximum = mdf_train[suffixcolumn].max()
    
    #get minimum value of training column
    minimum = mdf_train[suffixcolumn].min()
    
    #if cap/floor set based on number of standard deviations, measure statistics and set variables cap and/or floor
    if stdev_cap is not False and cap is False and floor is False:
      
      pre_capfloor_mean = mdf_train[suffixcolumn].mean()
      pre_capfloor_stdev = mdf_train[suffixcolumn].std()
      
      if pre_capfloor_mean != pre_capfloor_mean:
        pre_capfloor_mean = 0
      if pre_capfloor_stdev != pre_capfloor_stdev:
        pre_capfloor_stdev = 1

      cap = pre_capfloor_mean + stdev_cap * pre_capfloor_stdev
      floor = pre_capfloor_mean - stdev_cap * pre_capfloor_stdev
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
      
      mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[suffixcolumn] < floor, (suffixcolumn)] \
      = floor
      
      mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
      = floor
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[suffixcolumn].std()
    #median and median absolute deviation
    median = mdf_train[suffixcolumn].median()
    # MAD = mdf_train[suffixcolumn].mad()
    MAD = (mdf_train[suffixcolumn] - mdf_train[suffixcolumn].median(axis=0)).abs().median()

    #get mean of training data
    mean = mdf_train[suffixcolumn].mean()
    if mean != mean:
      mean = 0
      
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
    
    #perform min-max scaling to train and test sets using values from train
    mdf_train[suffixcolumn] = (mdf_train[suffixcolumn] - mean) / \
                                  (maxminusmin) * multiplier + offset
    
    mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - mean) / \
                                 (maxminusmin) * multiplier + offset

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'minimum' : minimum, \
                                              'maximum' : maximum, \
                                              'maxminusmin' : maxminusmin, \
                                              'mean' : mean, \
                                              'std' : std, \
                                              'median' : median, \
                                              'MAD' : MAD, \
                                              'offset' : offset, \
                                              'multiplier': multiplier, \
                                              'cap' : cap, \
                                              'floor' : floor, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_binary(self, mdf_train, mdf_test, column, category, \
                           treecategory, postprocess_dict, params = {}):
    '''
    #process_binary(mdf, column, missing)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_train, mdf_test), \
    #the name of the column string ('column') \
    #and the category from parent columkn (category)
    #fills missing valules with most common value
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    #str convert defaults to True to avoid edge case associated with bytes type entries
    #accepts one of {True, False, 'conditional_on_bytes'}
    #where conditional_on_bytes sets as False unless bytes entries are present in train data then sets to True
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    #can pass as 'onevalue' or 'zerovalue'
    #where onevalue used for bnry and zerovalue used for bnr2
    if 'infillconvention' in params:
      infillconvention = params['infillconvention']
    else:
      infillconvention = 'onevalue'

    if 'invert' in params:
      invert = params['invert']
    else:
      invert = False
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #conditional_on_bytes str_convert scenario resets to True if train set bytes entries present, otherwise resets to False
    if str_convert == 'conditional_on_bytes' \
    and (mdf_train[suffixcolumn] == mdf_train[suffixcolumn].astype(bytes)).sum() > 0:
      str_convert = True
    elif str_convert == 'conditional_on_bytes':
      str_convert = False

    if str_convert is True:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(str)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)

    tempstring = 'asdf'
    if tempstring in mdf_train:
      while tempstring in mdf_train:
        tempstring += 'z'
    #valuecounts is a list of unique entries sorted by frequency (from most to least) and then alphabetic, excluding nan
    #note this sorting has an edge case for bytes type entries when str_convert passed as False
    valuecounts = pd.DataFrame(mdf_train[suffixcolumn].value_counts())
    valuecounts = valuecounts.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
    valuecounts = list(valuecounts.index)
    
    if len(valuecounts) > 0:

      if len(valuecounts) > 1:
        if infillconvention == 'onevalue':
          binary_missing_plug = valuecounts[0]
        elif infillconvention == 'zerovalue':
          binary_missing_plug = valuecounts[1]

      if len(valuecounts) == 1:
        #making an executive decision here to deviate from standardinfill of most common value
        #for this edge case where a column evaluated as binary has only single value and NaN's
        if valuecounts[0] == valuecounts[0]:
          binary_missing_plug = np.nan
        else:
          binary_missing_plug = 'zzzinfill'
      #test for nan
      elif binary_missing_plug != binary_missing_plug:
        if infillconvention == 'onevalue':
          binary_missing_plug = valuecounts[1]
        elif infillconvention == 'zerovalue':
          binary_missing_plug = valuecounts[0]

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      extravalues = []
      if len(valuecounts) > 2:
        i=0
        for value in valuecounts:
          if i>1:
            extravalues.append(value)
          i+=1

      #replace nan in valuecounts with binary_missing_plug so we can sort
      valuecounts = [x if x == x else binary_missing_plug for x in valuecounts]
  #     #convert everything to string for sort
  #     valuecounts = [str(x) for x in valuecounts]

      #note LabelBinarizer encodes alphabetically, with 1 assigned to first and 0 to second
      #we'll take different approach of going by most common value to 1 unless 0 or 1
      #are already in the set then we'll defer to keeping those designations in place
      #there's some added complexity here to deal with edge case of passing this function
      #to a set with >2 values as we might run into when caluclating drift in postmunge

  #     valuecounts.sort()
  #     valuecounts = sorted(valuecounts)
      #in case this includes both strings and integers for instance we'll sort this way
  #     valuecounts = sorted(valuecounts, key=lambda p: str(p))

      #we'll save these in the normalization dictionary for future reference
      onevalue = valuecounts[0]
      if len(valuecounts) > 1:
        zerovalue = valuecounts[1]
      else:
        zerovalue = binary_missing_plug

      #special case for when the source column is already encoded as 0/1

      if len(valuecounts) <= 2:

        if 0 in valuecounts:
          zerovalue = 0
          if 1 in valuecounts:
            onevalue = 1
          else:
            if valuecounts[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts[1]
              else:
                onevalue = binary_missing_plug

        if 1 in valuecounts:
          if 0 not in valuecounts:
            if valuecounts[0] != 1:
              onevalue = 1
              zerovalue = valuecounts[0]

      #edge case same as above but when values of 0 or 1. are in set and 
      #len(valuecounts) > 2
      if len(valuecounts) > 2:
        valuecounts2 = valuecounts[:2]

        if 0 in valuecounts2:
          zerovalue = 0
          if 1 in valuecounts2:
            onevalue = 1
          else:
            if valuecounts2[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts2[1]
              else:
                onevalue = binary_missing_plug

        if 1 in valuecounts2:
          if 0 not in valuecounts2:
            if valuecounts2[0] != 1:
              onevalue = 1
              zerovalue = valuecounts2[0]

      #edge case that might come up in drift report
      if binary_missing_plug not in [onevalue, zerovalue]:
        if infillconvention == 'onevalue':
          binary_missing_plug = onevalue
        elif infillconvention == 'zerovalue':
          binary_missing_plug = zerovalue

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      if len(valuecounts) > 2:
        for value in extravalues:

          mdf_train = self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == value, binary_missing_plug, specified='replacement')
          mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == value, binary_missing_plug, specified='replacement')

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill in which casee the next line will dictate)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

      #replace missing data with specified classification
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].fillna(binary_missing_plug)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(binary_missing_plug)

      #this addressess issue where nunique for mdftest > than that for mdf_train
      #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
      #for user specified infill, and as currently addressed will default to infill with most common value
      #in the mean time a workaround could be for user to manually replace extra values with nan prior to
      #postmunge application such as if they want to apply ML infill
      #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
      #if len(mdf_test[suffixcolumn].unique()) > 2:
      uniqueintest = mdf_test[suffixcolumn].unique()
      for unique in uniqueintest:
        if unique not in {onevalue, zerovalue}:

          mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == unique, binary_missing_plug, specified='replacement')

      #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
      if invert is False:
        mdf_train = self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == onevalue, 1, 0)
        mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == onevalue, 1, 0)
      elif invert is True:
        mdf_train = self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == onevalue, 0, 1)
        mdf_test = self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == onevalue, 0, 1)

      #create list of columns
      bnrycolumns = [suffixcolumn]

      #change data types to 8-bit (1 byte) integers for memory savings
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.int8)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.int8)

      #a few more metrics collected for driftreport
      oneratio = mdf_train[suffixcolumn].sum() / mdf_train[suffixcolumn].shape[0]
      zeroratio = (mdf_train[suffixcolumn].shape[0] - mdf_train[suffixcolumn].sum() )\
                  / mdf_train[suffixcolumn].shape[0]

      #create list of columns associated with categorical transform (blank for now)
      categorylist = []
    
    else:
      mdf_train.loc[:, suffixcolumn] = 0
      mdf_test.loc[:, suffixcolumn] = 0
      
      binary_missing_plug = 0
      onevalue = 1
      zerovalue = 0
      extravalues = 0
      oneratio = 0
      zeroratio = 0
      bnrycolumns = [suffixcolumn]
      defaultinfill_dict = {'defaultinfill' : 'naninfill'}

  #     bnrynormalization_dict = {suffixcolumn : {'missing' : binary_missing_plug, \
  #                                                   'onevalue' : onevalue, \
  #                                                   'zerovalue' : zerovalue}}
    
    bnrynormalization_dict = {suffixcolumn : {'missing' : binary_missing_plug, \
                                              'infillconvention' : infillconvention, \
                                              1 : onevalue, \
                                              0 : zerovalue, \
                                              'extravalues' : extravalues, \
                                              'oneratio' : oneratio, \
                                              'zeroratio' : zeroratio, \
                                              'str_convert' : str_convert, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'invert' : invert,
                                              'inplace' : inplace}}

    #store some values in the column_dict{} for use later in ML infill methods
    column_dict_list = []

    for bc in bnrycolumns:

      if bc != bnrycolumns[0]:
        bnrynormalization_dict = {}

      column_dict = { bc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : bnrynormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : bnrycolumns, \
                           'categorylist' : bnrycolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list

  def _custom_train_onht(self, df, column, normalization_dict):
    """
    #a rewrite of onht transform in the custom train convention
    #note that this version combines text vs onht suffix conentions, now distinguishable by parameter
    #this should benefit latency
    #return comparable form of output
    #and accept comaprable parameters
    #primary difference is cleaner code and trimmed a little fat
    """

    #suffix_convention is to distinguish between text and onht suffixes
    #eg text: column + _str(entry) vs onht: column + _str(int)
    #(originally text followed conention column(minus suffix) + _entry)
    #suffix_convention accepts one of {'text', 'onht'}, and defaults to text
    if 'suffix_convention' in normalization_dict:
      suffix_convention = normalization_dict['suffix_convention']
    else:
      suffix_convention = 'text'
      normalization_dict.update({'suffix_convention' : suffix_convention})
    
    #note that when suffix_convention passed as text, 
    #str_convert hard resets to True, and null_activation hard resets to False
    if suffix_convention == 'text':
      normalization_dict.update({'str_convert' : True})
      normalization_dict.update({'null_activation' : False})
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    #str_convert defaults to True to avoid edge case assocaited with bytes type entries
    #accepts one of {True, False, 'conditional_on_bytes'}
    #where conditional_on_bytes sets as False unless bytes entries are present in train data then sets to True
    if 'str_convert' in normalization_dict:
      str_convert = normalization_dict['str_convert']
    else:
      str_convert = True
      normalization_dict.update({'str_convert' : str_convert})
    
    #ordered_overide is boolean to indicate if order of integer encoding basis will 
    #defer to cases when a column is a pandas categorical ordered set
    if 'ordered_overide' in normalization_dict:
      ordered_overide = normalization_dict['ordered_overide']
    else:
      ordered_overide = True
      normalization_dict.update({'ordered_overide' : ordered_overide})

    #all_activations is to base the full set of activations on user specification instead of training set
    if 'all_activations' in normalization_dict:
      #accepts False or a list of activation targets
      all_activations = normalization_dict['all_activations']
    else:
      all_activations = False
      normalization_dict.update({'all_activations' : all_activations})

    #add_activations is to include additional columns for entry activations even when not found in train set
    if 'add_activations' in normalization_dict:
      #accepts False or a list of added activation targets
      add_activations = normalization_dict['add_activations']
    else:
      add_activations = False
      normalization_dict.update({'add_activations' : add_activations})

    #less_activations is to remove entry activaiton columns even when entry found in train set
    if 'less_activations' in normalization_dict:
      #accepts False or a list of removed activation targets
      less_activations = normalization_dict['less_activations']
    else:
      less_activations = False
      normalization_dict.update({'less_activations' : less_activations})

    #consolidated_activations is to consolidate entries to a single common activation
    if 'consolidated_activations' in normalization_dict:
      #accepts False or a list of consolidated activation targets or a list of lists
      consolidated_activations = normalization_dict['consolidated_activations']
    else:
      consolidated_activations = False
      normalization_dict.update({'consolidated_activations' : consolidated_activations})
      
    #null_activation is to have a distinct column for missing data
    #note that when activated, entries in test set not found in train set will still be returned without activation
    #also accepts as 'Binary', which is used in Binary dimensionality reduction to return from inversion all zeros
    #for cases where test activation sets don't match any found in train activation sets
    #note that in Binary scenario we don't accept NaN entries in input
    if 'null_activation' in normalization_dict:
      null_activation = normalization_dict['null_activation']
    else:
      null_activation = False
      normalization_dict.update({'null_activation' : null_activation})
      
    #frequency_sort changes the sorting of encodings from alphabetical to frequency of entries in train set
    if 'frequency_sort' in normalization_dict:
      frequency_sort = normalization_dict['frequency_sort']
    else:
      frequency_sort = True
      normalization_dict.update({'frequency_sort' : frequency_sort})
      
    #_____
    
    #for every derivation related to the set labels_train, we'll remove missing_marker and add once prior to assembling binaryencoding_dict
    #which helps accomodate a few peculiarities related to python sets with NaN inclusion
    missing_marker = None
    if null_activation == 'Binary':
      missing_marker = '0' * len(str(df[column].iat[0]))

    normalization_dict.update({'missing_marker' : missing_marker})

    #conditional_on_bytes str_convert scenario resets to True if train set bytes entries present, otherwise resets to False
    if str_convert == 'conditional_on_bytes' \
    and (df[column] == df[column].astype(bytes)).sum() > 0:
      str_convert = True
      normalization_dict.update({'str_convert' : str_convert})
    elif str_convert == 'conditional_on_bytes':
      str_convert = False
      normalization_dict.update({'str_convert' : str_convert})
    
    #labels_train will be adjusted through derivation and serves as basis for binarization encoding
    labels_train = set()
    
    ordered = False
    
    #ordered_overide is not compatible with activation parameters
    if ordered_overide:
      if df[column].dtype.name == 'category' and df[column].cat.ordered:
        ordered = True
        labels_train_ordered = list(df[column].cat.categories)
        #by convention NaN is reserved for use with missing data
        labels_train_ordered = [x for x in labels_train_ordered if x==x]
        if str_convert is True:
          labels_train_ordered = [str(x) for x in labels_train_ordered]
        
    #frequency_sort derives a sorting order based on frequency of entries found in set
    if ordered is False and frequency_sort is True:
      ordered = True
      
      tempstring = 'asdf'
      if tempstring in df:
        while tempstring in df:
          tempstring += 'z'
      
      if str_convert is True:
        labels_train_ordered = pd.DataFrame(df[column].astype(str).value_counts())
      else:
        labels_train_ordered = pd.DataFrame(df[column].value_counts())
      #note this sorting has an edge case for bytes type entries when str_convert passed as False
      labels_train_ordered = labels_train_ordered.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
      labels_train_ordered = list(labels_train_ordered.index)
      #by convention NaN is reserved for use with missing data
      labels_train_ordered = [x for x in labels_train_ordered if x==x]
      if str_convert is True:
        labels_train_ordered = [str(x) for x in labels_train_ordered]
    
    #_____
            
    if df[column].dtype.name == 'category':
      labels_train = set(df[column].cat.categories)
      labels_train = {x for x in labels_train if x==x}

    if labels_train == set():  
      labels_train = set(df[column].unique())
      labels_train = {x for x in labels_train if x==x}

    #if str_convert parameter activated replace numerical with string equivalent (for common encoding between e.g. 2=='2')
    if str_convert is True:
      #note this set excludes missing_marker
      labels_train = {str(x) for x in labels_train}
      #only convert non-NaN entries in target column
      df = self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')

    #now we have a few activation set related parameters, applied by adjusting labels_train
    #we'll have convention that in cases where activation parameters are assigned, will overide ordered_overide (for alphabetic sorting)

    if all_activations is not False or less_activations is not False:
      #labels_train_orig is a support record that won't be returned
      labels_train_orig = labels_train.copy()

    if all_activations is not False:
      all_activations = {x for x in set(all_activations) if x==x}
      labels_train = all_activations

    if add_activations is not False:
      add_activations = {x for x in set(add_activations) if x==x}
      labels_train = labels_train | add_activations

    if less_activations is not False:
      less_activations = {x for x in set(less_activations) if x==x}
      labels_train = labels_train - less_activations

    #______

    #now we'll take account for any activation consolidations from consolidated_activations parameter

    #as part of this implementation, we next want to derive
    #a version of labels_train excluding consolidations (labels_train)
    #a list of consolidations associated with each returned_consolidation mapped to the returned_consolidation (consolidation_translate_dict)
    #and an inverse_consolidation_translate_dict mapping consolidated entries to their activations
    #which we'll then apply with a replace operation

    labels_train_before_consolidation = labels_train.copy()
    consolidation_translate_dict = {}
    inverse_consolidation_translate_dict = {}

    if consolidated_activations is not False:

      #if user passes a single tier list instead of list of lists we'll embed in a list
      if isinstance(consolidated_activations, list) and len(consolidated_activations) > 0:
        if not isinstance(consolidated_activations[0], list):
          consolidated_activations = [consolidated_activations]
          normalization_dict.update({'consolidated_activations' : consolidated_activations})

      for consolidation_list in consolidated_activations:

        #here is where we add any consolidation targets that weren't present in labels_train
        if str_convert is True:
          consolidation_list = [str(x) for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)
        else:
          #by convention missing data marker not elligible for inclusion in consolidation_list due to NaN/set peculiarities
          #consolidations with NaN can be accomodated by assignnan to treat desired entries as missing data
          consolidation_list = [x for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)

        #no prepare a version of labels_train excluding consolidations (labels_train)

        #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
        returned_consolidation = consolidation_list[0]

        #now remove consolidated entries from labels_train
        #and map a list of consolidations associated with each returned_consolidation to the returned_consolidation (consolidation_translate_dict)
        for consolidation_list in consolidated_activations:

          if len(consolidation_list) > 1:

            #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
            returned_consolidation = consolidation_list[0]

            labels_train = labels_train - set(consolidation_list[1:])

            consolidation_translate_dict.update({returned_consolidation : consolidation_list[1:]})

      #now populate an inverse_consolidation_translate_dict mapping consolidated entries to their activations
      for key,value in consolidation_translate_dict.items():
        for consolidation_list_entry in value:
          inverse_consolidation_translate_dict.update({consolidation_list_entry : key})

      #we can then apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)

    del consolidation_translate_dict
    normalization_dict.update({'inverse_consolidation_translate_dict' : inverse_consolidation_translate_dict})
    del inverse_consolidation_translate_dict

    #____

    #there are a few activation parameter scenarios where we may want to replace train set entries with missing data marker
    if all_activations is not False or less_activations is not False:
      extra_entries = labels_train_orig - labels_train
      extra_entries = list({x for x in extra_entries if x==x})
      if len(extra_entries) > 0:
        plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
        df[column] = df[column].astype('object').replace(plug_dict)

      del labels_train_orig

    #____

    #now prepare our onehot encoding
    
    #if there is a particular order to encodings we'll sort labels_train on basis of labels_train_ordered
    if ordered is True:
      #this converts labels_train to a sorted list
      labels_train = self.__list_sorting(labels_train_ordered, labels_train)
    
    elif ordered is False:
      #convert labels_train to list 
      #and add the missing data marker to first position which will result in all zero binarized representation
      labels_train = list(labels_train)
      labels_train = sorted(labels_train, key=str)
      
    #add our missing_marker, note adding as last position will result in last column even in ordered scenario
    if null_activation is True:
      labels_train = labels_train + [missing_marker]

    #one hot encoding support function
    df_cat = self.__onehot_support(df, column, scenario=1, activations_list=labels_train)
    
    #change data types to int8 for memory savings
    for activation_column in labels_train:
      df_cat[activation_column] = df_cat[activation_column].astype(np.int8)
    
    labels_dict = {}
    onht_returned_columns = []
    if suffix_convention == 'onht':
      i = 0
      for entry in labels_train:
        labels_dict.update({entry : column + '_' + str(i)})
        onht_returned_columns.append(column + '_' + str(i))
        i += 1
    elif suffix_convention == 'text':
      for entry in labels_train:
        #str conversion is to accomodate missing data marker which will return as column + '_nan'
        labels_dict.update({entry : column + '_' + entry})
        onht_returned_columns.append(column + '_' + entry)
      
    normalization_dict.update({'labels_dict' : labels_dict,
                               'onht_returned_columns' : onht_returned_columns})
      
    #concatinate the sparse set with the rest of our training data
    df = pd.concat([df, df_cat], axis=1)
    
    del df[column]
    
    #now convert coloumn headers from text convention to onht convention
    df = df.rename(columns=labels_dict)

    #this is a drift stat for activaiton ratios
    onht_activations_dict = {}
    for onht_returned_column in onht_returned_columns:
      sumcalc = (df[onht_returned_column] == 1).sum() 
      ratio = sumcalc / df[onht_returned_column].shape[0]
      onht_activations_dict.update({onht_returned_column : ratio})
      
    normalization_dict.update({'onht_activations_dict' : onht_activations_dict})
    
    return df, normalization_dict

  def _process_smth(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_smth(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #intended for applicant downstream of a multirt MLinfilltype encoding (e.g. one hot encoding)
    #since this is specifically intended as a downstream trasnform, we'll assume infill already applied
    #applies label smoothing
    #accepts parameters for activation value (float 0.5-1), 
    #LSfit parameter to activate fitted smoothing
    #testsmooth parameter to activate consistently smoothing test data
    '''

    suffixoverlap_results = {}
      
    if 'activation' in params:
      activation = params['activation']
    else:
      activation = 0.9

    if 'LSfit' in params:
      LSfit = params['LSfit']
    else:
      LSfit = False
      
    if 'testsmooth' in params:
      testsmooth = params['testsmooth']
    else:
      testsmooth = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #this function is intended to be applied donstream of a multirt encoding
    #meaning there may be multiple columns serving as target
    #we'll access the upstream categorylist stored in postprocess_dict
    inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
    
    #the returned columns will each have consistent suffix appending
    textcolumns = [(x + '_' + suffix) for x in inputtextcolumns]
    
    #this maps between received and returned columns
    textlabelsdict = dict(zip(inputtextcolumns, textcolumns))
    
    #inplace convention is a little different in that we are inspecting suffixoverlap before copying
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    if inplace is not True:
      
      for inputtextcolumn in inputtextcolumns:
        
        mdf_train[textlabelsdict[inputtextcolumn]] = mdf_train[inputtextcolumn].copy()
        mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()
    
    else:
      
      mdf_train.rename(columns = textlabelsdict, inplace = True)
      mdf_test.rename(columns = textlabelsdict, inplace = True)
    
    #___
    
    #now apply label smoothing
#     category = 'smth'
    LSfitparams_dict = {}

    categorycomplete_dict = dict(zip(textcolumns, [False]*len(textcolumns)))

    for labelsmoothingcolumn in textcolumns:

      if categorycomplete_dict[labelsmoothingcolumn] is False:

        mdf_train, categorycomplete_dict, LSfitparams_dict = \
        self.__apply_LabelSmoothing(mdf_train, 
                                  labelsmoothingcolumn, 
                                  activation, 
                                  textcolumns, 
                                  category, 
                                  categorycomplete_dict, 
                                  LSfit, 
                                  LSfitparams_dict)

    #smoothing not applied to test data consistent with postmunge convention
    #(postmunge can apply based on traindata parameter or by activating testsmooth)
    if testsmooth is True:
      
      categorycomplete_test_dict = dict(zip(textcolumns, [False]*len(textcolumns)))
      
      for labelsmoothingcolumn in textcolumns:

        if categorycomplete_test_dict[labelsmoothingcolumn] is False:

          mdf_test, categorycomplete_dict = \
          self.__postapply_LabelSmoothing(mdf_test, 
                                        labelsmoothingcolumn, 
                                        categorycomplete_test_dict, 
                                        LSfitparams_dict)
    
    column_dict_list = []
    textnormalization_dict = {}

    if len(textcolumns) > 0:
      textnormalization_dict = {textcolumns[0] : {'textlabelsdict' : textlabelsdict, \
                                      'textcolumns' : textcolumns, \
                                      'inputtextcolumns' : inputtextcolumns, \
                                      'LSfitparams_dict' : LSfitparams_dict, \
                                      'activation' : activation, \
                                      'LSfit' : LSfit, \
                                      'testsmooth' : testsmooth, \
                                      'suffix' : suffix, \
                                      'inplace' : inplace}}

    for tc in textcolumns:

      if tc != textcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : textcolumns, \
                           'categorylist' : textcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def __GPS_parse(self, coordinates, firstcomma, seccondcomma, negdirection):
    """
    This is a support function used in GPS1

    based on "$GPGGA message"
    applied to one set of coordinates
    receives coordinates as a str for parsing

    note that we are applying the full conversion in this function
    instead of extracting serately support columns for degrees, minutes, and direction
    and applying pandas operations to convert entries in parallel
    it is possible doing the latter may be more efficient
    for now we are following the full conversion in this function approach for cleaner code
    we speculate some further optimizations for processing efficiency may be possible
    by conducting portions of the DDMM operations in pandas instead of to each entry seperately with .transform
    which would require seperate parsing functions to populate each support column
    but is also possible the benefit would be offset by the redundant parsing
    """

    comma_counter = 0
    maxparsed_address = 0
    str_length = len(coordinates)
    parsing_complete = False
    DDMM = np.nan
    direction = np.nan
    latt = np.nan

    for address in range(str_length):

      if address > maxparsed_address and parsing_complete is False:

        current_char = coordinates[address]
        maxparsed_address = address

        if current_char == ',':
          comma_counter += 1

        else:

          #direction is single character string as one of {'N', 'S'}
          if comma_counter == seccondcomma:

            for address3 in range(address, str_length):
              if comma_counter == seccondcomma:
                current_char3 = coordinates[address3]

                if current_char3 == ',':
                  comma_counter += 1
                  maxparsed_address = address3
                  parsing_complete = True

                  excerpt = coordinates[address : address3]

                  if len(excerpt) > 0 and excerpt in {'N', 'S', 'E', 'W'}:

                    direction = excerpt

          #latitude in the DDMM.MMMMM (variable decimal) populated after second comma
          if comma_counter == firstcomma:

            for address2 in range(address, str_length):
              if comma_counter == firstcomma:
                current_char2 = coordinates[address2]

                if current_char2 == ',':
                  comma_counter += 1
                  maxparsed_address = address2

                  excerpt = coordinates[address : address2]

                  if len(excerpt) > 0 and self.__is_number(excerpt):
#                       if len(excerpt) > 0 and _is_number(excerpt):
                    DDMM = excerpt

    degrees = np.nan
    minutes = np.nan

    #the potential scenarios we will accomodate for characters preceding the decimal are DDMM. / DDDMM.
    if DDMM == DDMM:
      dot_index = np.nan
      if '.' in list(DDMM):
        dot_index = list(DDMM).index('.')
        if dot_index == 4:
          degrees = int(DDMM[0:2])
          minutes = float(DDMM[2:])
        elif dot_index == 5:
          degrees = int(DDMM[0:3])
          minutes = float(DDMM[3:])
      elif len(DDMM) == 4:
        degrees = int(DDMM[0:2])
        minutes = float(DDMM[2:])
      elif len(DDMM) == 5:
        degrees = int(DDMM[0:3])
        minutes = float(DDMM[3:])

      latt = degrees*60 + minutes

      if direction == negdirection:
        latt *= -1

    return latt
  
  def _custom_train_GPS1(self, df, column, normalization_dict):
    """
    GPS1 is for converting sets of received GPS coordinates into format of two columns of lattitude and longitude
    accepts input of string GPS coordinates
    
    accepts parameter GPS_convention, which currently only supports the base configuration of 'default'
    which in future extensions may allow selection between alternate GPS reporting conventions
    'default' is based on structure of the "$GPGGA message" which was output from an RTK GPS receiver
    which follows NMEA conventions, and has lattitude in between commas 2-3, and longitude between 4-5
    reference description available at https://www.gpsworld.com/what-exactly-is-gps-nmea-data/
    allows for variations in precisions of reported coordinates (i.e. number of significant figures)
    or variations in degree magnitude, such as between DDMM. or DDDMM.
    relies on comma seperated inputs
    accepts parameter comma_addresses to designate locations for lattitude/direction/longitude/direction
    which consistent with the demonstration defaults to [2,3,4,5]
    i.e. lattitude is after comma 2, direction after comma 3, longitude after 4, direction after 5
    also accepts parameter comma_count, defaulting to 14, which is used for inversion
    returns lattitude and longitude coordinates as +/- floats in units of arc minutes
    
    we believe there are other conventions for GPS reporting
    we don't yet have sufficient insight into mainstream practice to formalize alternatives
    this implementation is intended as a proof of concept
    
    note that in cases where comma seperated entries are blank, nonnumeric, or omitted
    the returned entries may be represented as NaN for missing data
    in the base root category definition, this transform is followed by a mlti transform 
    for independent normalization of the lattitude and longitude sets
    this will return consistent output independent of data properties of two columns 
    with suffixes _latt and _long to distinguish lattitude and longitude
    so no corresponding _custom_test function is needed

    note that the NArw aggregation will only recognize received NaN points
    (e.g. it won't recognize cases where lattitude or longitude weren't recorded)
    so if ML infill is desired on returned sets, received missing data should be NaN encoded

    Note that this function uses the support function _GPS_parse
    """

    #GPS_convention is to distinguish between conventions for format of received GPS coordinates
    #default is based on structure of the "$GPGGA message"
    #which was output from an RTK GPS receiver
    #in 'default' each row is individually parsed
    #in 'nonunique', only unique entries are parsed
    if 'GPS_convention' in normalization_dict:
      GPS_convention = normalization_dict['GPS_convention']
    else:
      GPS_convention = 'default'
      normalization_dict.update({'GPS_convention' : GPS_convention})
      
    if 'comma_addresses' in normalization_dict:
      comma_addresses = normalization_dict['comma_addresses']
    else:
      #this corresponds to default where DDMM.MMMM lattitude after comma 2, lattitude direction after comma 3
      #DDMM.MMMM longitude after comma 4, longitude direction after comma 5
      comma_addresses = [2,3,4,5]
      normalization_dict.update({'comma_addresses' : comma_addresses})

    #comma_count used in inversion to pad out the recovered form with commas
    if 'comma_count' in normalization_dict:
      comma_count = normalization_dict['comma_count']
    else:
      comma_count = 14
      normalization_dict.update({'comma_count' : comma_count})

    #_____

    def default_GPS_parse_latt(coordinates1):
      return self.__GPS_parse(coordinates1, comma_addresses[0], comma_addresses[1], 'S')

    def default_GPS_parse_long(coordinates1):
      return self.__GPS_parse(coordinates1, comma_addresses[2], comma_addresses[3], 'W')

    #_____

    #in the default scenario, each row is parsed, which may be appropriate for all unique entries
    if GPS_convention == 'default':

      #here are returned column headers
      latt_column = column + '_latt'
      long_column = column + '_long'

      normalization_dict.update({'latt_column' : latt_column,
                                 'long_column' : long_column})

      # df[column] = df[column].astype(str)

      df[latt_column] = pd.Series(df[column].astype(str)).transform(default_GPS_parse_latt)

      df[long_column] = pd.Series(df[column].astype(str)).transform(default_GPS_parse_long)
      
      del df[column]

    #_____
    
    #in the nonunique scenario, only unique entries are parsed, which may be appropriate for common points
    if GPS_convention == 'nonunique':

      unique_entries = list(df[column].astype(str).unique())

      latt_replace_dict = {}
      long_replace_dict = {}

      for unique_entry in unique_entries:

        unique_latt = default_GPS_parse_latt(unique_entry)
        latt_replace_dict.update({unique_entry : unique_latt})

        unique_long = default_GPS_parse_long(unique_entry)
        long_replace_dict.update({unique_entry : unique_long})

      #here are returned column headers
      latt_column = column + '_latt'
      long_column = column + '_long'

      df[latt_column] = df[column].astype(str).astype('object').replace(latt_replace_dict)
      df[long_column] = df[column].astype(str).astype('object').replace(long_replace_dict)

      del df[column]

      normalization_dict.update({'latt_column' : latt_column,
                                 'long_column' : long_column,
                                 'latt_replace_dict' : latt_replace_dict,
                                 'long_replace_dict' : long_replace_dict})

    return df, normalization_dict

  def _process_mlti(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #intended for applicant downstream of a concurrent_nmbr MLinfilltype encoding 
    #(e.g. downstream of multicolumn continuous numeric sets)
    #since this is specifically intended as a downstream trasnform, we'll assume infill already applied
    #applies normalization to each of the columns in upstream transform returned categorylist
    #the type of normalization applied is based on assignparam parameter norm_category
    #which defaults to nmbr, as in trasnforms are accessed from nmbr process_dict entry
    #parameters to the normalization trasnform can also be passed as dictionary to parameter norm_params, e.g. in the form
    #assignparam = {'mlti' : {'(column)' : {'norm_params' : {'(parameter)' : '(value)'}}}
    #where '(column)' is input column associated with the root category
    #and parameter / value are associated with the norm_category
    #the normalizaiton applied treats the norm_category as a tree category without offspring
    #and the input columns are either retained or replaced based on parameter norm_retain, defaulting ot False for replaced
    #note that if an alternate treatment is desired where to apply a family tree of transforms to each column
    #user should instead structure upstream trasnform as a set of numeric mlinfilltype trasnforms.
    '''

    suffixoverlap_results = {}
      
    if 'norm_category' in params:
      norm_category = params['norm_category']
    else:
      norm_category = 'nmbr'

    postprocess_dict['mlti_categories'] = postprocess_dict['mlti_categories'] | {norm_category}

    if 'norm_params' in params:
      norm_params = params['norm_params']
    else:
      norm_params = {}

    #dtype accepts one of {'float', 'conditionalinteger', 'mlhs'}
    #where 'float' is for use with mlti applied with concurrent_nmbr MLinfilltype
    #and 'conditionalinteger' is for use with concurrent_ordl MLinfilltype
    #and 'hash' is for use when upstream trasnform is an ordinal hashing (uses different method to identify encoding space for conditional type)
    #(may be needed for transforms in custom_trian convention, otherwise can just leave as 'float' to defer to transform basis)
    #note conditionalinteger is only applied when processdict[norm_category]['dtype_convert'] is not False
    if 'dtype' in params:
      dtype = params['dtype']
    else:
      dtype = 'float'

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    if 'messy_data_headers' in params:
      messy_data_headers = params['messy_data_headers']
      if messy_data_headers == []:
        messy_data_headers = [column]
    else:
      messy_data_headers = False
      
    #________

    #when applying mlti as a noise transform mlhs for DPhs
    #redundant noise parameters need to be passed both to mlti params and to norm_params
    #mlti params for purposes of deriving entroy seed requirements
    #and norm_params for purposes of passing to norm_category
    #(as alternative to norm_params global_assignparam params or default_assignparam will also work)

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True

    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = 0.01

    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
    
#     #note that in the random_generator == False scenario 
#     #__random_parameters_params_append will have set to np.random.PCG64
#     nprandom_dict = {'custom' : random_generator,
#                      'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated for noise transforms in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #mlti doesn't register the missing_process_dict_noise_transform validation result since it is also used for non-noise trasnforms
      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________

    #this is to support accessing defaultparams from process_dict based on norm_category
    mlti_norm_params = self.__grab_params({norm_category : {column : norm_params}}, 
                                          norm_category, 
                                          column, 
                                          postprocess_dict['process_dict'][norm_category],
                                          postprocess_dict
                                         )
      
    #this function is intended to be applied donstream of a multirt encoding
    #meaning there may be multiple columns serving as target
    #we'll access the upstream categorylist stored in postprocess_dict
    #or otherwise if this is an upstream trasnform just apply to the received column
    if column in postprocess_dict['column_dict']:
      inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
    else:
      #origcolumn scenario when not applied as downstream transform
      inputtextcolumns = [column]
      
    if isinstance(messy_data_headers, list) and column not in postprocess_dict['column_dict']:
      inputtextcolumns = messy_data_headers.copy()
    
    #the returned columns will each have consistent suffix appending
    textcolumns = [(x + '_' + suffix) for x in inputtextcolumns]

    #now update the mlti_norm_params based on any relevant assignparam specifications to global and default
    assignparam = postprocess_dict['assign_param']
    if 'global_assignparam' in assignparam:
      mlti_norm_params.update(assignparam['global_assignparam'])
    if 'default_assignparam' in assignparam:
      if norm_category in assignparam['default_assignparam']:
        mlti_norm_params.update(assignparam['default_assignparam'][norm_category])
        
    #for now operating on convention that the same entropy seeding parameters are passed to each instance
    #traindata inspection is applicable to postmunge, in automunge will not be present
    if 'noise_transform' in postprocess_dict['process_dict'][norm_category] \
    and postprocess_dict['process_dict'][norm_category]['noise_transform'] is not False \
    and ( 'traindata' not in postprocess_dict or 'traindata' in postprocess_dict \
    and postprocess_dict['traindata'] not in {'train_no_noise', 'test_no_noise'}):
      
      mlti_norm_params.update({'random_generator' : random_generator,
                               'sampling_resource_dict' : sampling_resource_dict})

    #textcolumns are never returned, they just used to applky the intermediate suffix appender associated with this tree category
    #so we'll always pass inplace as True in norm_params and if not accepted based on inplace_option in process_dict delete textcolumns
    mlti_norm_params.update({'inplace' : True})
    
    #this maps between received and returned columns
    textlabelsdict = dict(zip(inputtextcolumns, textcolumns))
    
    #inplace convention is a little different in that we are inspecting suffixoverlap before copying
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    if inplace is not True:
      
      for inputtextcolumn in inputtextcolumns:
        
        mdf_train[textlabelsdict[inputtextcolumn]] = mdf_train[inputtextcolumn].copy()
        mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()
    
    else:
      
      mdf_train.rename(columns = textlabelsdict, inplace = True)
      mdf_test.rename(columns = textlabelsdict, inplace = True)
      
    mlti_norm_params_column_dict = {}
    for inputcolumn in textcolumns:
      mlti_norm_params_column_dict.update({inputcolumn : self.__autocopy(mlti_norm_params)})
    
    #now apply one of custom_train / custom_test or dualprocess based on norm_category processdict entry
    
    norm_column_dict_list = []
    norm_columnkey_dict = {'columnkey_dict' : {}}
    
    if 'custom_train' in postprocess_dict['process_dict'][norm_category] \
    and callable(postprocess_dict['process_dict'][norm_category]['custom_train']):
      
      for inputcolumn in textcolumns:
        
        #any assignparam specifications to specific inputcolumn added here
        if norm_category in assignparam:
          if inputcolumn in assignparam[norm_category]:
            mlti_norm_params_column_dict[inputcolumn].update(assignparam[norm_category][inputcolumn])
            
        #for entropy seeding we'll shuffle for each application
        if 'sampling_resource_dict' in mlti_norm_params_column_dict[inputcolumn]:
          for entry in mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict']:
            if entry[-5:] == 'seeds' \
            and isinstance(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
              np.random.shuffle(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry])
        
        mdf_train, mdf_test, column_dict_list_portion = \
        self.__custom_process_wrapper(mdf_train, mdf_test, inputcolumn, category, \
                                     norm_category, postprocess_dict, mlti_norm_params_column_dict[inputcolumn])
    
        norm_column_dict_list += self.__autocopy(column_dict_list_portion)
      
        norm_columnkey_dict = self.__populate_columnkey_dict(column_dict_list_portion, norm_columnkey_dict, norm_category)

      if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
      and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
        for textcolumn in textcolumns:
          del mdf_train[textcolumn]
          del mdf_test[textcolumn]

    #elif this is a dual process function
    elif 'dualprocess' in postprocess_dict['process_dict'][norm_category] \
    and callable(postprocess_dict['process_dict'][norm_category]['dualprocess']):
      
      for inputcolumn in textcolumns:
        
        #any assignparam specifications to specific inputcolumn added here
        if norm_category in assignparam:
          if inputcolumn in assignparam[norm_category]:
            mlti_norm_params_column_dict[inputcolumn].update(assignparam[norm_category][inputcolumn])
            
        #for entropy seeding we'll shuffle for each application
        if 'sampling_resource_dict' in mlti_norm_params_column_dict[inputcolumn]:
          for entry in mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict']:
            if entry[-5:] == 'seeds' \
            and isinstance(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
              np.random.shuffle(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry])

        mdf_train, mdf_test, column_dict_list_portion = \
        postprocess_dict['process_dict'][norm_category]['dualprocess'](mdf_train, mdf_test, inputcolumn, category, \
                                                                       norm_category, postprocess_dict, mlti_norm_params_column_dict[inputcolumn])

        norm_column_dict_list += self.__autocopy(column_dict_list_portion)
      
        norm_columnkey_dict = self.__populate_columnkey_dict(column_dict_list_portion, norm_columnkey_dict, norm_category)

      if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
      and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
        for textcolumn in textcolumns:
          del mdf_train[textcolumn]
          del mdf_test[textcolumn]
    
    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in postprocess_dict['process_dict'][norm_category] \
    and callable(postprocess_dict['process_dict'][norm_category]['singleprocess']):
      
      for inputcolumn in textcolumns:

        #any assignparam specifications to specific inputcolumn added here
        if norm_category in assignparam:
          if inputcolumn in assignparam[norm_category]:
            mlti_norm_params_column_dict[inputcolumn].update(assignparam[norm_category][inputcolumn])
            
        #for entropy seeding we'll shuffle for each application
        if 'sampling_resource_dict' in mlti_norm_params_column_dict[inputcolumn]:
          for entry in mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict']:
            if entry[-5:] == 'seeds' \
            and isinstance(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
              np.random.shuffle(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'][entry])

        mdf_train, column_dict_list_portion =  \
        postprocess_dict['process_dict'][norm_category]['singleprocess'](mdf_train, inputcolumn, category, \
                                                                         norm_category, postprocess_dict, mlti_norm_params_column_dict[inputcolumn])

        mdf_test, _1 = \
        postprocess_dict['process_dict'][norm_category]['singleprocess'](mdf_test, inputcolumn, category, \
                                                                         norm_category, postprocess_dict, mlti_norm_params_column_dict[inputcolumn])

        norm_column_dict_list += self.__autocopy(column_dict_list_portion)
      
        norm_columnkey_dict = self.__populate_columnkey_dict(column_dict_list_portion, norm_columnkey_dict, norm_category)

      if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
      and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
        for textcolumn in textcolumns:
          del mdf_train[textcolumn]
          del mdf_test[textcolumn]
    
    final_returned_columns = []
    for norm_column_dict_list_entry in norm_column_dict_list:
      final_returned_columns.append(list(norm_column_dict_list_entry)[0])

    #if this is an ordinal encoded set returned form transform in custom_train convention we may need to apply dtype convert
    max_encoding_for_dtype_convert_dict = {}
    if dtype in {'conditionalinteger', 'mlhs'}:

      if 'dtype_convert' not in postprocess_dict['process_dict'][norm_category] \
      or 'dtype_convert' in postprocess_dict['process_dict'][norm_category] \
      and postprocess_dict['process_dict'][norm_category]['dtype_convert'] is not False:

        for final_returned_column in final_returned_columns:

          max_encoding_for_dtype_convert = mdf_train[final_returned_column].max()

          #max encoding space for dtype = mlhs scnenario assumes upstream hashing and norm_category that retains size of encoding space
          if dtype == 'mlhs':

            if column in postprocess_dict['column_dict'] \
            and column in postprocess_dict['column_dict'][column]['normalization_dict'] \
            and 'vocab_size' in postprocess_dict['column_dict'][column]['normalization_dict'][column]:
              max_encoding_for_dtype_convert = postprocess_dict['column_dict'][column]['normalization_dict'][column]['vocab_size']

          #save the max encoding with key of final_returned_column for use in postmunge
          max_encoding_for_dtype_convert_dict.update({final_returned_column : max_encoding_for_dtype_convert})

          if max_encoding_for_dtype_convert <= 255:
            mdf_train[final_returned_column] = mdf_train[final_returned_column].astype(np.uint8)
            mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint8)
          elif max_encoding_for_dtype_convert <= 65535:
            mdf_train[final_returned_column] = mdf_train[final_returned_column].astype(np.uint16)
            mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint16)
          else:
            mdf_train[final_returned_column] = mdf_train[final_returned_column].astype(np.uint32)
            mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint32)
    
    #_____
    #entropy seeding support
      
    #erase seeds
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)
    
    if 'noise_transform' in postprocess_dict['process_dict'][norm_category] \
    and postprocess_dict['process_dict'][norm_category]['noise_transform'] is not False \
    and ( 'traindata' not in postprocess_dict or 'traindata' in postprocess_dict \
    and postprocess_dict['traindata'] not in {'train_no_noise', 'test_no_noise'}):
      
      #erase seeds that were saved in mlti_norm_params_column_dict
      for inputcolumn in textcolumns:
        if 'sampling_resource_dict' in mlti_norm_params_column_dict[inputcolumn]:
          mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'] = \
          self.__erase_seeds(mlti_norm_params_column_dict[inputcolumn]['sampling_resource_dict'])
        
      #aggregate counts into the returned sampling_resource_dict from norm_category reports
      for norm_column_dict in norm_column_dict_list:
        for norm_column_dict_key in norm_column_dict:
          if 'normalization_dict' in norm_column_dict[norm_column_dict_key] \
          and norm_column_dict_key in norm_column_dict[norm_column_dict_key]['normalization_dict'] \
          and 'sampling_resource_dict' in norm_column_dict[norm_column_dict_key]['normalization_dict'][norm_column_dict_key]:
            sampling_resource_dict_single = norm_column_dict[norm_column_dict_key]['normalization_dict'][norm_column_dict_key]['sampling_resource_dict']
            
            #here are the counts we'll aggregate, which will have integer entries (seeds will be empty list or array)
            aggregated_counts = []
            for entry in sampling_resource_dict:
              if isinstance(sampling_resource_dict[entry], (int, float)):
                aggregated_counts.append(entry)
                
            for aggregated_count in aggregated_counts:
              if aggregated_count in sampling_resource_dict_single:
                sampling_resource_dict[aggregated_count] += sampling_resource_dict_single[aggregated_count]
    #_____

    textnormalization_dict = {}
    if len(final_returned_columns) > 0:
      textnormalization_dict = {final_returned_columns[0] : {'norm_category' : norm_category, \
                                                              'norm_params' : mlti_norm_params_column_dict, \
                                                              'textlabelsdict' : textlabelsdict, \
                                                              'textcolumns' : textcolumns, \
                                                              'inputtextcolumns' : inputtextcolumns, \
                                                              'norm_columnkey_dict' : norm_columnkey_dict, \
                                                              'norm_column_dict_list' : norm_column_dict_list, \
                                                              'messy_data_headers' : messy_data_headers, \
                                                              'dtype' : dtype, \
                                                              'max_encoding_for_dtype_convert_dict' : max_encoding_for_dtype_convert_dict, \
                                                              'final_returned_columns' : final_returned_columns, \
                                                              'sampling_resource_dict' : sampling_resource_dict, \
                                                              'trainnoise' : trainnoise,
                                                              'testnoise' : testnoise,
                                                              'flip_prob' : flip_prob,
                                                              'test_flip_prob' : test_flip_prob,
                                                              'suffix' : suffix, \
                                                              'inplace' : inplace}}

    column_dict_list = []
    for tc in final_returned_columns:

      if tc != final_returned_columns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : final_returned_columns, \
                           'categorylist' : final_returned_columns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_cnsl(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_cnsl(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to consolidate multiple categoric columns into a single categoric column
    #this is similar to what is done in Binary
    #now that we have the messy data convention from 8.10 wanted to get comparable functionality in family trees
    #recommend applying an upstream mlto to align to a common form between input columns as ordinal
    #note that this will convert all entries into strings prior to consolidation
    
    #note that cnsl pads out the representation of each input with underscore characters
    #which relies on convention of ordinal encoded input to avoid edge case
    #based on padding to width of max string with in encoding space
    '''

    suffixoverlap_results = {}
    
    #initialize parameters

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'messy_data_headers' in params:
      messy_data_headers = params['messy_data_headers']
      if messy_data_headers == []:
        messy_data_headers = [column]
    else:
      messy_data_headers = False
      
#     if 'inplace' in params:
#       inplace = params['inplace']
#     else:
#       inplace = False
      
    if 'pad_character' in params:
      pad_character = params['pad_character']
    else:
      pad_character = '_'
      
    if 'upstream_ordinal' in params:
      upstream_ordinal = params['upstream_ordinal']
    else:
      upstream_ordinal = True
      
    #this function is intended to be applied donstream of a potentially multi column encoding 
    #meaning there may be multiple columns serving as target
    #we'll access the upstream categorylist stored in postprocess_dict
    #or otherwise if this is an upstream trasnform just apply to the received column
    if column in postprocess_dict['column_dict']:
      inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
    else:
      #origcolumn scenario when not applied as downstream transform
      #note that this scenario also is automatically applied 
      #when this transform applied downstream of a concurrent MLinfilltype with mlti
      #since at that point column won't be in column_dict
      #which is desired behavior for that use case
      inputtextcolumns = [column]
      
    if isinstance(messy_data_headers, list) and column not in postprocess_dict['column_dict']:
      inputtextcolumns = messy_data_headers.copy()
    
    #the returned columns will each have consistent suffix appending
    textcolumn = (column + '_' + suffix)
    
    #we don't offer inplace support for most DP transforms since they usually involve support columns
    #we might be able to make inplace work for this one
    
    #check suffixoverlap before copying
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_train[textcolumn] = ''
    mdf_test[textcolumn] = ''
    
    #we'll do two versions with and without integer conversion to remove float trailing 0's
    if upstream_ordinal is True:

      #find the max string length to pad out with underscroe character
      #an alternate character can be selected by parameter pad_character
      unique_list = []
      for inputtextcolumn in inputtextcolumns:
        unique_list += list(mdf_train[inputtextcolumn].unique())
      #get longest length entry
      longest_len = len([str(int(x)) for x in unique_list if unique_list.index(x) == 0 or len(str(int(x))) > len(str(int(unique_list[unique_list.index(x)-1])))])
      
    else:

      #find the max string length to pad out with underscroe character
      #an alternate character can be selected by parameter pad_character
      unique_list = list(mdf_train[inputtextcolumns].unique())
      #get longest length entry
      longest_len = len([str(x) for x in unique_list if unique_list.index(x) == 0 or len(int(x)) > len(str(unique_list[unique_list.index(x)-1]))])

    #add a contingency factor for case where encoding space wasn't fully represented
    #this scenario is not common in our library
    #as used here if ordinal encoding space found in train set was 10^3, this will support encodings up to 10^10
    longest_len += 7
    
    #now consolidate with padding
    #we'll do two versions with and without integer conversion to remove float trailing 0's  
    for inputtextcolumn in inputtextcolumns:

      if upstream_ordinal is True:
        mdf_train[textcolumn] += mdf_train[inputtextcolumn].astype(int).astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)
        mdf_test[textcolumn] += mdf_test[inputtextcolumn].astype(int).astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)
        
      else:
        mdf_train[textcolumn] += mdf_train[inputtextcolumn].astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)
        mdf_test[textcolumn] += mdf_test[inputtextcolumn].astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)

    column_dict_list = []

    normalization_dict = {}

    normalization_dict = {textcolumn : {'longest_len' : longest_len,
                                        'pad_character' : pad_character,
                                        'upstream_ordinal' : upstream_ordinal,
                                        'messy_data_headers' : messy_data_headers,
                                        'suffix' : suffix,
                                        'inputtextcolumns' : inputtextcolumns,
                                       }}
    
    for textcolumn in [textcolumn]:
    
      column_dict = {textcolumn : {'category' : treecategory, \
                                   'origcategory' : category, \
                                   'normalization_dict' : normalization_dict, \
                                   'origcolumn' : column, \
                                   'inputcolumn' : column, \
                                   'columnslist' : [textcolumn], \
                                   'categorylist' : [textcolumn], \
                                   'infillmodel' : False, \
                                   'infillcomplete' : False, \
                                   'suffixoverlap_results' : suffixoverlap_results, \
                                   'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_lngt(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton that length of string for each entry
    #such as a heuristic for information content
    #default infill is len(str(np.nan)) = 3
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    df[suffixcolumn] = df[suffixcolumn].astype(str).transform(len)
    
    #grab a fe4w driftreport metrics:
    #get maximum value of training column
    maximum = df[suffixcolumn].max()
    
    #get minimum value of training column
    minimum = df[suffixcolumn].min()
    
    #get minimum value of training column
    mean = df[suffixcolumn].mean()
    
    #get standard deviation of training column
    std = df[suffixcolumn].std()

    #create list of columns
    columns = [suffixcolumn]

    #library defaults to int32 data type for integer mlinfilltype
    df[suffixcolumn] = df[suffixcolumn].astype(np.int32)

    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'maximum' : maximum, \
                                          'minimum' : minimum, \
                                          'mean' : mean, \
                                          'std' : std, \
                                          'suffix' : suffix, \
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'inplace' : inplace }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in columns:

      if nc != columns[0]:
        normalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : columns, \
                           'categorylist' : columns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_bnst(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    intended for application downstream of a 1010, multirt, or concurrent_act MLinfilltype encoding (with boolean integer entries)
    bnst converts multicolumn categoric representations to a single column representation
    returning a single column string aggregation of the received multi column input
    this may be useful if the upstream categoric transform isn't available in ordinal form
    as some  learning libraries may not accept multi column categoric encodings for label sets
    note that an additional downstream ordinal encoding could be specified in family tree to convert output to numeric
    
    note that bnst is very similar to what has been used in prior transforms such as inverseprocess_1010 which preceded custom_inversion_1010
    
    parameter upstreaminteger should be passed as True when upstream contains integers (such as boolean integer or ordinal int)
    else can pass as False when upstream contains string entries
    
    since reducing the number of columns does not support inplace
    
    implemented in the singleprocess convention
    """

    suffixoverlap_results = {}
    
    if 'upstreaminteger' in params:
      upstreaminteger = params['upstreaminteger']
    else:
      upstreaminteger = True
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    #this function is intended to be applied donstream of a 1010, multirt, or concurrent encoding
    #meaning there may be multiple columns serving as target
    #we'll access the upstream categorylist stored in postprocess_dict
    if column in postprocess_dict['column_dict']:
      inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
    else:
      inputtextcolumns = [column]
    
    #note that column will be the first column in the upstream categorylist
    suffixcolumn = column + '_' + suffix
    
    #check for suffix overlap before initizlizing suffixcolumn
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    df[suffixcolumn] = ''

    for inputtextcolumn in inputtextcolumns:
      
      if upstreaminteger is True:

        #only supports inputtextcolumn containing entries of 0 and 1, assumes infill performed upstream
        #partly motivated by inversion requires a fixed character width 
        df[suffixcolumn] = df[suffixcolumn] + pd.to_numeric(df[inputtextcolumn], errors='coerce').fillna(0).abs().astype(int).clip(lower=0, upper=1).astype(str)

      #we need a fixed character width for inversion        
      else:
        df[suffixcolumn] = df[suffixcolumn] + df[inputtextcolumn].astype(str)
        
    normalization_dict = {suffixcolumn : {'upstreaminteger' : upstreaminteger, \
                                          'suffix' : suffix, \
                                          'inputtextcolumns' : inputtextcolumns}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in [suffixcolumn]:

      if nc != [suffixcolumn][0]:
        normalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : [suffixcolumn], \
                           'categorylist' : [suffixcolumn], \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list
  
  def _process_UPCS(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton that converts columns to uppercase strings
    #such as to allow consistnet encoding if data has upper/lower case discrepencies
    #default infill is a distinct entry as string NAN
    #note that with assigninfill this can be converted to other infill methods
    #returns same dataframe with new column of name suffixcolumn
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'activate' in params:
      activate = params['activate']
    else:
      activate = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    #convert to uppercase string based on activate parameter
    if activate is True:
      #convert column to uppercase string except for nan infill points
      df = \
      self.__autowhere(df, suffixcolumn, df[suffixcolumn] == df[suffixcolumn], df[suffixcolumn].astype(str).str.upper(), specified='replacement')

    #create list of columns
    UPCScolumns = [suffixcolumn]

    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'activate' : activate, 
                                          'defaultinfill_dict' : defaultinfill_dict, 
                                          'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in UPCScolumns:

      if nc != UPCScolumns[0]:
        normalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : UPCScolumns, \
                           'categorylist' : UPCScolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False, \
                           'inplace' : inplace}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_splt(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_splt(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 
    #'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    '''

    suffixoverlap_results = {}
    
    #overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]
    
    if 'minsplit' in params:
      minsplit = params['minsplit'] - 1
    else:
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = False
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]
                  
                  if concurrent_activations is False:

                    if extract == extract3:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                  elif concurrent_activations is True:
                    
                    if extract == extract3 and unique in overlap_dict[key]:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
     
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
    
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_' + suffix + '_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_test[newcolumn] = mdf_test[column].copy()

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_' + suffix + '_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'suffix' : suffix, \
                                                'test_same_as_train' : test_same_as_train, \
                                                'overlap_dict' : overlap_dict, \
                                                'splt_newcolumns_splt'   : newcolumns, \
                                                'minsplit' : minsplit, \
                                                'concurrent_activations' : concurrent_activations, \
                                                'preint_newcolumns' : preint_newcolumns, \
                                                'int_headers' : int_headers, \
                                                'int_labels_dict' : int_labels_dict, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'inverse_int_labels_dict' : inverse_int_labels_dict}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_spl2(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_spl2(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and replaces entries 
    #with their redecued overlap
    #replaces entries without overlap to 0 (unique to spl5)
    #for example, if a categorical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created in which the entry 'north' 
    #replaced cells with north in their entries
    #(here for north and northeast)
    #and cells with west would be set to 0
    #returns as column titled origcolumn_spl2
    #missing values are ignored by default
    #this alternative to splt may be benficial for instance if one wanted 
    #to follow with an ordl encoding
    '''

    suffixoverlap_results = {}
    
#     overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]

    if 'minsplit' in params:
        
      minsplit = params['minsplit'] - 1
    
    else:
      
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
      
    if 'consolidate_nonoverlaps' in params:
      consolidate_nonoverlaps = params['consolidate_nonoverlaps']
    else:
      consolidate_nonoverlaps = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]

                  if extract == extract3:

                    extract_already_in_overlap_dict = True
                    
                    break
                    
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          break
      
    #now for mdf_test we'll only consider those overlaps already 
    #identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:
    
      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                break
    
    #so that was all comparable to splt, now for spl2 we'll create a new 
    #dictionary structred as
    #{original unique value : overlap extract for replacement}
    
    #since one original unique value may have entries as multiple overlaps, 
    #we'll prioritize overlaps with
    #longer string lengths and then alphabetical
    
    spl2_overlap_dict = {}
    
    overlap_key_list = list(overlap_dict)
    
    overlap_key_list.sort()
    overlap_key_list.sort(key = len, reverse=True)
    
    for overlap_key in overlap_key_list:
      
      for entry in overlap_dict[overlap_key]:
        
        if entry not in spl2_overlap_dict:
          
          spl2_overlap_dict.update({entry : overlap_key})
    
    #here's where we identify values to set to 0 for spl5
    spl5_zero_dict = {}
    if consolidate_nonoverlaps is True:
      for entry in unique_list:
        if entry not in spl2_overlap_dict:
          spl5_zero_dict.update({entry : 0})
    
    #then we'll do same for test set
    
    spl2_test_overlap_dict = {}
    
    test_overlap_key_list = list(test_overlap_dict)
    
    test_overlap_key_list.sort()
    test_overlap_key_list.sort(key = len, reverse=True)
    
    for overlap_key in test_overlap_key_list:
      
      for entry in test_overlap_dict[overlap_key]:
        
        if entry not in spl2_test_overlap_dict:
          
          spl2_test_overlap_dict.update({entry : overlap_key})
    
    #here's where we identify values to set to 0 for spl5
    spl5_test_zero_dict = {}
    if consolidate_nonoverlaps is True:

      if test_same_as_train is True:
        unique_list_test = list(mdf_test[column].unique())
        unique_list_test = list(map(str, unique_list_test))

      for entry in unique_list_test:
        if entry not in spl2_test_overlap_dict:
          spl5_test_zero_dict.update({entry : 0})
    
    newcolumns = []

#     for dict_key in overlap_dict:

    newcolumn = column + '_' + suffix
    
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[newcolumn] = mdf_test[column].copy()

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
    mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

    mdf_train[newcolumn] = mdf_train[newcolumn].replace(spl2_overlap_dict)
    mdf_train[newcolumn] = mdf_train[newcolumn].replace(spl5_zero_dict)

#       mdf_train[newcolumn] = mdf_train[column].isin(overlap_dict[dict_key])
#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

    mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl2_test_overlap_dict)
    mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl5_test_zero_dict)

#       mdf_test[newcolumn] = mdf_test[column].isin(test_overlap_dict[dict_key])
#       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

    newcolumns.append(newcolumn)

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'suffix' : suffix, \
                                                'test_same_as_train' : test_same_as_train, \
                                                'consolidate_nonoverlaps' : consolidate_nonoverlaps, \
                                                'overlap_dict' : overlap_dict, \
                                                'spl2_newcolumns'   : newcolumns, 
                                                'spl2_overlap_dict' : spl2_overlap_dict, \
                                                'spl2_test_overlap_dict' : spl2_test_overlap_dict, \
                                                'spl5_zero_dict' : spl5_zero_dict, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'minsplit' : minsplit}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []

    return mdf_train, mdf_test, column_dict_list

  def _process_sp19(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_splt(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 
    #'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #sp15 is comparable to splt but multiple concurrent activations allowed
    #so requires a different MLinfilltype in processdict
    
    #sp19 is comparable to sp15 but with a returned binary encoding aggregation
    '''

    suffixoverlap_results = {}
    
    #overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]
    
    if 'minsplit' in params:
      minsplit = params['minsplit'] - 1
    else:
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False
      
    #note that same MLinfilltype in processdict ('1010')
    #may be used for both configurations but applying concurrent_activations = False
    #with sp11 is less efficient then running splt
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]
                  
                  if concurrent_activations is False:

                    if extract == extract3:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                  elif concurrent_activations is True:
                    
                    if extract == extract3 and unique in overlap_dict[key]:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
        
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
    
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_sp15_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
#       mdf_train[newcolumn] = mdf_train[column].copy()
      
      mdf_test[newcolumn] = mdf_test[column].copy()

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_sp15_' + str(i)})
        i += 1
        
      #now convert column headers from string to int convention
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      newcolumns = [int_labels_dict[entry] for entry in newcolumns]

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []
    
    #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
    
    if len(newcolumns) > 0:
      
      sp19_column = column + '_' + suffix
    
      #aggregate collection of activations as string set
      #the suffix 'activations_' is to avoid potential of overlap with binary encoding and aggregated activations
      mdf_train[sp19_column] = 'activations_'
      mdf_test[sp19_column] = 'activations_'

      for entry in newcolumns:
        mdf_train[sp19_column] = mdf_train[sp19_column] + mdf_train[entry].astype(str)
        mdf_test[sp19_column] = mdf_test[sp19_column] + mdf_test[entry].astype(str)

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[sp19_column].unique())
      labels_train.sort()
      labels_test = list(mdf_test[sp19_column].unique())
      labels_test.sort()

      #labels_train is a list of strings, insert missing data marker
      labels_train = labels_train + [np.nan]
      labels_test = labels_test + [np.nan]

      #get length of the list
      listlength = len(labels_train)

      #calculate number of columns we'll need
      binary_column_count = int(np.ceil(np.log2(listlength)))

      #initialize dictionaryt to store encodings
      binary_encoding_dict = {}
      encoding_list = []

      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")

        if i < len(labels_train):

          #store the encoding in a dictionary
          binary_encoding_dict.update({labels_train[i] : encoding})

          #store the encoding in a list for checking in next step
          encoding_list.append(encoding)

      #clear up memory
      del encoding_list
  #     del overlap_list

      #new driftreport metric _1010_activations_dict
      _1010_activations_dict = {}
      for key in binary_encoding_dict:
        sumcalc = (mdf_train[sp19_column] == key).sum() 
        ratio = sumcalc / mdf_train[sp19_column].shape[0]
        _1010_activations_dict.update({key:ratio})

      #replace the cateogries in train set via ordinal trasnformation
      mdf_train[sp19_column] = mdf_train[sp19_column].astype('object').replace(binary_encoding_dict) 

      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))

      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
      mdf_test[sp19_column] = mdf_test[sp19_column].astype('object').replace(testplug_dict)  

      #now we'll apply the 1010 transformation to the test set
      mdf_test[sp19_column] = mdf_test[sp19_column].astype('object').replace(binary_encoding_dict)    

      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []

      for i in range(binary_column_count):

        _1010_columnlist.append(column + '_' + suffix + '_' + str(i))

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, _1010_columnlist, suffixoverlap_results, postprocess_dict['printstatus'])

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_train[_1010_column] = mdf_train[sp19_column].str.slice(i,i+1).astype(np.int8)

        mdf_test[_1010_column] = mdf_test[sp19_column].str.slice(i,i+1).astype(np.int8)

        i+=1

      #now delete the support column
      del mdf_train[sp19_column]
      del mdf_test[sp19_column]

      for entry in newcolumns:
        del mdf_train[entry]
        del mdf_test[entry]

      #now store the column_dict entries
      categorylist = _1010_columnlist

      normalization_dict = {}
      if len(categorylist) > 0:
        normalization_dict = {categorylist[0] : {'suffix' : suffix, \
                                                'test_same_as_train' : test_same_as_train, \
                                                '_1010_binary_encoding_dict' : binary_encoding_dict, \
                                                '_1010_binary_column_count' : binary_column_count, \
                                                '_1010_activations_dict' : _1010_activations_dict, \
                                                'categorylist' : categorylist, \
                                                'overlap_dict' : overlap_dict, \
                                                'splt_newcolumns_sp19'   : newcolumns, \
                                                'minsplit' : minsplit, \
                                                'concurrent_activations' : concurrent_activations, \
                                                'preint_newcolumns' : preint_newcolumns, \
                                                'int_headers' : int_headers, \
                                                'int_labels_dict' : int_labels_dict, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'inverse_int_labels_dict' : inverse_int_labels_dict}}

      column_dict_list = []

      for tc in categorylist:

        if tc != categorylist[0]:
          normalization_dict = {}

        column_dict = {tc : {'category' : treecategory, \
                             'origcategory' : category, \
                             'normalization_dict' : normalization_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict)
      
    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_sbst(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_sbst(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    '''

    suffixoverlap_results = {}
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False

    if 'minsplit' in params:
      minsplit = params['minsplit']
    else:
      minsplit = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    unique_list = sorted(unique_list, key=len, reverse=True)
    
#     maxlength = max(len(x) for x in unique_list)
    
#     minlength = min(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minlength, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    #unique is what we are searching for
    for unique in unique_list:
      len_unique = len(unique)

      if len_unique >= minsplit:
      
        #unique2 is where we are searching
        for unique2 in unique_list:
          len_unique2 = len(unique2)
          
          if len_unique2 > len_unique:
            
            nbr_iterations = len_unique2 - len_unique + 1
            
            for i in range(nbr_iterations):
              
              extract = unique2[i:(len_unique+i)]
              
              extract_already_in_overlap_dict = False
                    
              if extract_already_in_overlap_dict is False:
                
                if extract == unique:
                  
                  if extract in overlap_dict:
                    
                    if unique2 not in overlap_dict[extract]:
                      
                      overlap_dict[extract].append(unique2)
                      
                      if concurrent_activations is False:

                        break
                        
                    # if unique not in overlap_dict[extract]:
                      
                    #   overlap_dict[extract].append(unique)

                    #   if concurrent_activations is False:

                    #     break
                        
                  #else if we don't have a key for extract
                  else:

                    overlap_dict.update({extract : [unique, unique2]})

                    if concurrent_activations is False:

                      break
                    
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      unique_list_test = sorted(unique_list_test, key=len, reverse=True)

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
                
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_' + suffix + '_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
#       mdf_train[newcolumn] = mdf_train[column].copy()
  
      mdf_test[newcolumn] = mdf_test[column].copy()

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_' + suffix + '_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'suffix' : suffix, \
                                                'test_same_as_train' : test_same_as_train, \
                                                'overlap_dict' : overlap_dict, \
                                                'splt_newcolumns_sbst'   : newcolumns, \
                                                'minsplit' : minsplit, \
                                                'concurrent_activations' : concurrent_activations, \
                                                'preint_newcolumns' : preint_newcolumns, \
                                                'int_headers' : int_headers, \
                                                'int_labels_dict' : int_labels_dict, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'inverse_int_labels_dict' : inverse_int_labels_dict}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_sbs3(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_sbst(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    
    #sbs3 is comparable to sbst but with a returned binary encoding aggregation
    '''

    suffixoverlap_results = {}
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False

    if 'minsplit' in params:
      minsplit = params['minsplit']
    else:
      minsplit = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    unique_list = sorted(unique_list, key=len, reverse=True)
    
#     maxlength = max(len(x) for x in unique_list)
    
#     minlength = min(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minlength, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    #unique is what we are searching for
    for unique in unique_list:
      len_unique = len(unique)

      if len_unique >= minsplit:
      
        #unique2 is where we are searching
        for unique2 in unique_list:
          len_unique2 = len(unique2)
          
          if len_unique2 > len_unique:
            
            nbr_iterations = len_unique2 - len_unique + 1
            
            for i in range(nbr_iterations):
              
              extract = unique2[i:(len_unique+i)]
              
              extract_already_in_overlap_dict = False
                    
              if extract_already_in_overlap_dict is False:
                
                if extract == unique:
                  
                  if extract in overlap_dict:
                    
                    if unique2 not in overlap_dict[extract]:
                      
                      overlap_dict[extract].append(unique2)
                      
                      if concurrent_activations is False:

                        break
                        
                    # if unique not in overlap_dict[extract]:
                      
                    #   overlap_dict[extract].append(unique)

                    #   if concurrent_activations is False:

                    #     break
                        
                  #else if we don't have a key for extract
                  else:

                    overlap_dict.update({extract : [unique, unique2]})

                    if concurrent_activations is False:

                      break
                    
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      unique_list_test = sorted(unique_list_test, key=len, reverse=True)

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
                
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_sbst_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
#       mdf_train[newcolumn] = mdf_train[column].copy()
  
      mdf_test[newcolumn] = mdf_test[column].copy()

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_sbst_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []
    
    #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
    
    if len(newcolumns) > 0:
      
      sbs3_column = column + suffix

      #aggregate collection of activations as string set
      #the suffix 'activations_' is to avoid potential of overlap with binary encoding and aggregated activations
      mdf_train[sbs3_column] = 'activations_'
      mdf_test[sbs3_column] = 'activations_'

      for entry in newcolumns:
        mdf_train[sbs3_column] = mdf_train[sbs3_column] + mdf_train[entry].astype(str)
        mdf_test[sbs3_column] = mdf_test[sbs3_column] + mdf_test[entry].astype(str)

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[sbs3_column].unique())
      labels_train.sort()
      labels_test = list(mdf_test[sbs3_column].unique())
      labels_test.sort()

      #labels_train is a list of strings, insert missing data marker
      labels_train = labels_train + [np.nan]
      labels_test = labels_test + [np.nan]

      #get length of the list
      listlength = len(labels_train)

      #calculate number of columns we'll need
      binary_column_count = int(np.ceil(np.log2(listlength)))

      #initialize dictionaryt to store encodings
      binary_encoding_dict = {}
      encoding_list = []

      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")

        if i < len(labels_train):

          #store the encoding in a dictionary
          binary_encoding_dict.update({labels_train[i] : encoding})

          #store the encoding in a list for checking in next step
          encoding_list.append(encoding)

      #clear up memory
      del encoding_list
  #     del overlap_list

      #new driftreport metric _1010_activations_dict
      _1010_activations_dict = {}
      for key in binary_encoding_dict:
        sumcalc = (mdf_train[sbs3_column] == key).sum() 
        ratio = sumcalc / mdf_train[sbs3_column].shape[0]
        _1010_activations_dict.update({key:ratio})

      #replace the cateogries in train set via ordinal trasnformation
      mdf_train[sbs3_column] = mdf_train[sbs3_column].astype('object').replace(binary_encoding_dict) 

      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))

      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
      mdf_test[sbs3_column] = mdf_test[sbs3_column].astype('object').replace(testplug_dict)  

      #now we'll apply the 1010 transformation to the test set
      mdf_test[sbs3_column] = mdf_test[sbs3_column].astype('object').replace(binary_encoding_dict)    

      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []

      for i in range(binary_column_count):

        _1010_columnlist.append(column + '_' + suffix + '_' + str(i))

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, _1010_columnlist, suffixoverlap_results, postprocess_dict['printstatus'])

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_train[_1010_column] = mdf_train[sbs3_column].str.slice(i,i+1).astype(np.int8)

        mdf_test[_1010_column] = mdf_test[sbs3_column].str.slice(i,i+1).astype(np.int8)

        i+=1

      #now delete the support column
      del mdf_train[sbs3_column]
      del mdf_test[sbs3_column]

      for entry in newcolumns:
        del mdf_train[entry]
        del mdf_test[entry]

      #now store the column_dict entries
      categorylist = _1010_columnlist

      normalization_dict = {}
      if len(categorylist) > 0:
        normalization_dict = {categorylist[0] : {'suffix' : suffix, \
                                                'test_same_as_train' : test_same_as_train, \
                                                '_1010_binary_encoding_dict' : binary_encoding_dict, \
                                                '_1010_binary_column_count' : binary_column_count, \
                                                '_1010_activations_dict' : _1010_activations_dict, \
                                                'categorylist' : categorylist, \
                                                'overlap_dict' : overlap_dict, \
                                                'splt_newcolumns_sbs3'   : newcolumns, \
                                                'concurrent_activations' : concurrent_activations, \
                                                'minsplit' : minsplit, \
                                                'preint_newcolumns' : preint_newcolumns, \
                                                'int_headers' : int_headers, \
                                                'int_labels_dict' : int_labels_dict, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'inverse_int_labels_dict' : inverse_int_labels_dict}}

      column_dict_list = []

      for tc in categorylist:

        if tc != categorylist[0]:
          normalization_dict = {}

        column_dict = {tc : {'category' : treecategory, \
                             'origcategory' : category, \
                             'normalization_dict' : normalization_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict)
      
    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_hash(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    """
    #applies the "hashing trick" to encode categoric sets
    #returning a set of columns with integers corresponding to words from set vocabulary
    #this is intended for sets with very high cardinality
    #note that the same integer may be returned in different columns 
    #for same word found in different entries
    #works by segregating entries into a list of words based on space seperator
    #stripping any special characters
    #and hashing each word with hashlib md5 hashing algorithm
    #which is converted to integer and taken remainder from a division by vocab_size
    #where vocab_size is based on heuristic
    #the heuristic derives vocab_size based on number of unique entries found in train set times the multipler
    #where if that result is greater than the cap then the heuristic reverts to the cap as vocab_size
    #where for hash the number of unique entries is calculated after extracting words from entries
    #note that if vocab_size is not large enough some of words may be returned with encoding overlap
    #returns set of columns with suffix appenders '_hash_#' where # is integer
    #unless only returning one column then suffix appender is just '_hash'
    #note that entries with fewer words than max word count are padded out with 0
    #also accepts parameter for excluded_characters, space
    #uppercase conversion if desired is performed externally by the UPCS transform
    #if space passed as '' then word extraction doesn't take place
    #user can manually specify a vocab_size with vocab-size parameter
    """

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    if 'vocab_size' in params:
      vocab_size = params['vocab_size']
    else:
      vocab_size = False
      
    if 'heuristic_multiplier' in params:
      heuristic_multiplier = params['heuristic_multiplier']
    else:
      heuristic_multiplier = 2
      
    if 'heuristic_cap' in params:
      heuristic_cap = params['heuristic_cap']
    else:
      heuristic_cap = 1024
    
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [',', '.', '?', '!', '(', ')']

    if 'space' in params:
      space = params['space']
    else:
      space = ' '

    if 'max_column_count' in params:
      max_column_count = params['max_column_count']
    else:
      max_column_count = False

    #salt can be passed as arbitrary string to ensure privacy of encoding basis
    if 'salt' in params:
      salt = params['salt']
    else:
      salt = ''
      
    #accepts either 'hash' or 'md5', 
    #where hash is quicker since uses native python function instead of hashlib
    if 'hash_alg' in params:
      hash_alg = params['hash_alg']
    else:
      hash_alg = 'hash'

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix

    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)    
      
    #convert column to string, note this means that missing data converted to 'nan'
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(str)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)
    
    #now scrub special characters
    for scrub_punctuation in excluded_characters:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)
      
    if hash_alg == 'md5':
      from hashlib import md5
    
    #define some support functions
    def _assemble_wordlist(string):
      """
      converts a string to list of words by splitting words from space characters
      assumes any desired special characters have already been stripped
      """

      wordlist = []
      j = 0
      
      if max_column_count is False:
        for i in range(len(string)+1):
          if i < len(string):
            if string[i] == space:
              if i > 0:
                if string[j] != space:
                  wordlist.append(string[j:i])
              j = i+1

          else:
            if j < len(string):
              if string[j] != space:
                wordlist.append(string[j:i])
      
      #else if we have a cap on number of returned columns
      else:
        wordlist_length = 0
        for i in range(len(string)+1):
          if i < len(string):
            if string[i] == space:
              if i > 0:
                if string[j] != space:
                  wordlist.append(string[j:i])
                  wordlist_length += 1
                  if wordlist_length == max_column_count - 1:
                    j = i+1
                    break
              j = i+1

          else:
            if j < len(string):
              if string[j] != space:
                wordlist.append(string[j:i])
                wordlist_length += 1
                j = i+1
                if wordlist_length == max_column_count - 1:
                  break

        if wordlist_length == max_column_count - 1:
          if j < len(string):
            wordlist.append(string[j:len(string)])

      return wordlist
    
    #now convert entries to lists of words
    #e.g. this converts "Two words" to ['Two', 'words']
    #if you don't want to split words can pass space = ''
    if space != '':
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(_assemble_wordlist)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_assemble_wordlist)
    else:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(lambda x: [x])
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(lambda x: [x])
      
    #if user didn't specify vocab_size then derive based on heuristic
    if vocab_size is False:
      #now let's derive vocab_size from train set, first convert all entries to a list of lists
      temp_list = pd.Series(mdf_train[suffixcolumn]).tolist()
      #this flattens the list of lists
      temp_list = [item for items in temp_list for item in items]
      #consolidate redundant entries
      temp_list = set(temp_list)
      #get length
      vocab_size = len(temp_list)
      del temp_list
      #calculate the vocab_size based on heuristic_multiplier and heuristic_cap
      vocab_size = int(vocab_size * heuristic_multiplier)
      if vocab_size > heuristic_cap:
        vocab_size = int(heuristic_cap)
        
    def _md5_hash(wordlist):
      """
      applies either md5 hashing to the list of words or python default hashing
      this conversion to integers is known as "the hashing trick"
      md5 is partly inspired by tensorflow keras_preprocessing hashing_trick function
      requires importing from hashlib import md5 which is performed above
      here n is the range of integers for vocabulary
      0 is reserved for use to pad lists of shorter length
      (salt and hash_alg are accessible even though not explicitly passed)
      """
      if hash_alg == 'md5':
        #this is the line (excluding the salt option) that was partly inspired by a line in a Keras function
        return [int(md5((salt + word).encode()).hexdigest(), 16) % (vocab_size-1) + 1 for word in wordlist]
      else:
        return [hash(salt + word) % (vocab_size-1) + 1 for word in wordlist]
        
    #now apply hashing to convert to integers based on vocab_size
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(_md5_hash)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_md5_hash)
    
    #get max length, i.e. for entry with most words
    max_length = mdf_train[suffixcolumn].transform(len).max()
    
    def _pad_hash(hash_list):
      """
      ensures hashing lists are all same length by padding shorter length lists with 0
      """
      padcount = max_length - len(hash_list)
      if padcount >= 0:
        pad = []
        for i in range(padcount):
          pad = pad + [0]
        hash_list = hash_list + pad
      else:
        #for test data we'll trim if max_length greater than max_length from train data
        hash_list = hash_list[:max_length]

      return hash_list
        
    #the other entries are padded out with 0 to reach same length, if a train entry has longer length it is trimmed
    mdf_train[suffixcolumn] = \
    mdf_train[suffixcolumn].transform(_pad_hash)
    mdf_test[suffixcolumn] = \
    mdf_test[suffixcolumn].transform(_pad_hash)
    
    if max_length > 1:

      hashcolumns = []
      for i in range(max_length):

        hash_column = column + '_' + suffix + '_' + str(i)

        hashcolumns += [hash_column]

        #check for column header overlap
        suffixoverlap_results = \
        self.__df_check_suffixoverlap(mdf_train, hash_column, suffixoverlap_results, postprocess_dict['printstatus'])

        #now populate the column with i'th entry from hashed list
        mdf_train[hash_column] = mdf_train[suffixcolumn].transform(lambda x: x[i])
        mdf_test[hash_column] = mdf_test[suffixcolumn].transform(lambda x: x[i])

      #remove support column
      del mdf_train[suffixcolumn]
      del mdf_test[suffixcolumn]
      
    else:
      hashcolumns = [suffixcolumn]
      
      #now populate the column with i'th entry from hashed list
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(lambda x: x[0])
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(lambda x: x[0])

    #returned data type is conditional on the size of encoding space
    for hashcolumn in hashcolumns:

      max_encoding = vocab_size - 1
      if max_encoding <= 255:
        mdf_train[hashcolumn] = mdf_train[hashcolumn].astype(np.uint8)
        mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_train[hashcolumn] = mdf_train[hashcolumn].astype(np.uint16)
        mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint16)
      else:
        mdf_train[hashcolumn] = mdf_train[hashcolumn].astype(np.uint32)
        mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint32)
    
    hashnormalization_dict = {}
    if len(hashcolumns) > 0:
      hashnormalization_dict = {hashcolumns[0] : {'hashcolumns' : hashcolumns, \
                                                  'col_count' : max_length, \
                                                  'vocab_size_hash' : vocab_size, \
                                                  'heuristic_multiplier' : heuristic_multiplier, \
                                                  'heuristic_cap' : heuristic_cap, \
                                                  'max_length' : max_length, \
                                                  'excluded_characters' : excluded_characters, \
                                                  'space' : space, \
                                                  'salt' : salt, \
                                                  'max_column_count' : max_column_count, \
                                                  'hash_alg' : hash_alg, \
                                                  'suffix' : suffix, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}

    column_dict_list = []

    for hc in hashcolumns:

      if hc != hashcolumns[0]:
        hashnormalization_dict = {}
      
      column_dict = { hc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : hashnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : hashcolumns, \
                           'categorylist' : hashcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return mdf_train, mdf_test, column_dict_list

  def _process_hs10(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    """
    #applies the "hashing trick" to encode categoric sets
    #returning a set of columns binary encoded corresponding to integers returned from hash
    #this is intended for sets with very high cardinality
    #note that the same activation set may be returned for different entries
    #works by hashing each entry with hashlib md5 hashing algorithm
    #which is converted to integer and taken remainder from a division by vocab_size
    #where vocab_size is passed parameter intended to align with vocabulary size defaulting to 128
    #note that if vocab_size is not large enough some of words may be returned with encoding overlap
    #returns set of columns with suffix appenders '_hs10_#' where # is integer
    #uppercase conversion if desired is performed externally by the UPCS transform
    #applies a heuristic to
    #set a vocab_size based on number unique entries times heuristic_multiplier parameter which defaults to 2
    #also accepts heuristic_cap parameter where if unique * heuristic_muyltipler > heuristic_cap
    #then vocab_size = heuristic_cap
    #or user can manually specify a vocab_size instead of relying on heuristic
    """
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    if 'vocab_size' in params:
      vocab_size = params['vocab_size']
    else:
      vocab_size = False
      
    if 'heuristic_multiplier' in params:
      heuristic_multiplier = params['heuristic_multiplier']
    else:
      heuristic_multiplier = 2
      
    if 'heuristic_cap' in params:
      heuristic_cap = params['heuristic_cap']
    else:
      heuristic_cap = 1024

    #salt can be passed as arbitrary string to ensure privacy of encoding basis
    if 'salt' in params:
      salt = params['salt']
    else:
      salt = ''

    #a list of strings that are scrubbed from entries e.g. punctuations
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = []
      
    #accepts either 'hash' or 'md5', 
    #where hash is quicker since uses native python function instead of hashlib 
    #(md5 was the original basis, hash is new default)
    if 'hash_alg' in params:
      hash_alg = params['hash_alg']
    else:
      hash_alg = 'hash'

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix

    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
    #convert column to string, note this means that missing data converted to 'nan'
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(str)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)

    #now scrub special characters
    for scrub_punctuation in excluded_characters:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)
    
    if vocab_size is False:
      #calculate the vocab_size based on heuristic_multiplier and heuristic_cap
      vocab_size = int(mdf_train[suffixcolumn].nunique() * heuristic_multiplier)
      if vocab_size > heuristic_cap:
        vocab_size = int(heuristic_cap)
      
    if hash_alg == 'md5':
      from hashlib import md5
    
    def _md5_hash(entry):
      """
      applies hashing to the list of words
      this conversion to ingtegers is known as "the hashing trick"
      requires importing from hashlib import md5 if hash_alg = "md5"
      here n is the range of integers for vocabulary
      similar comment to _process_hash that md5 was partly inspired by a line in a keras function
      """
      if hash_alg == 'md5':
        return int(md5((salt + entry).encode()).hexdigest(), 16) % (vocab_size)
      else:
        return hash(salt + entry) % (vocab_size)

    #now apply hashing to convert to integers based on vocab_size
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(_md5_hash)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_md5_hash)

    binary_column_count = int(np.ceil(np.log2(vocab_size)))
    
    #convert integer encoding to binary
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].transform(bin)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(bin)
    
    #convert format to string of digits
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].str[2:]
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str[2:]
    
    #pad out zeros
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].str.zfill(binary_column_count)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.zfill(binary_column_count)
    
    hashcolumns = []
    for i in range(binary_column_count):

      hash_column = column + '_' + suffix + '_' + str(i)
      
      hashcolumns += [hash_column]
      
      #check for column header overlap
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, hash_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      #now populate the column with i'th entry from hashed list
      mdf_train[hash_column] = mdf_train[suffixcolumn].str[i].astype(np.int8)
      mdf_test[hash_column] = mdf_test[suffixcolumn].str[i].astype(np.int8)
    
    #remove support column
    del mdf_train[suffixcolumn]
    del mdf_test[suffixcolumn]

    hashnormalization_dict = {}
    if len(hashcolumns) > 0:
      hashnormalization_dict = {hashcolumns[0] : {'hashcolumns' : hashcolumns, \
                                                  'col_count' : binary_column_count, \
                                                  'vocab_size' : vocab_size, \
                                                  'heuristic_multiplier' : heuristic_multiplier, \
                                                  'heuristic_cap' : heuristic_cap, \
                                                  'salt' : salt, \
                                                  'excluded_characters' : excluded_characters, \
                                                  'hash_alg' : hash_alg, \
                                                  'suffix' : suffix, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}
    
    column_dict_list = []

    for hc in hashcolumns:

      if hc != hashcolumns[0]:
        hashnormalization_dict = {}
      
      column_dict = { hc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : hashnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : hashcolumns, \
                           'categorylist' : hashcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return mdf_train, mdf_test, column_dict_list

  def _process_srch(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #process_srch(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note that search parameter can include lists of search terms embedded in the list
    #which embedded lists will be aggregated to a single activation
    #for example if we want single activation for female names could pass search = [['Ms.', 'Miss', 'Mrs']] etc
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    """

    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #for this transform, suffixcolumn is not returned in final set
    #applying so that we can allow defaultinfill support without overwriting the input column
    
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
      
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)
    
    newcolumns = []
    search_dict = {}
    for searchitem in search:
      search_dict.update({column + '_' + suffix + '_' + str(searchitem) : str(searchitem)})
      
    for newcolumn in search_dict:
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train = \
      self.__autowhere(mdf_train, newcolumn, mdf_train[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')

      mdf_test = \
      self.__autowhere(mdf_test, newcolumn, mdf_test[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')
    
    newcolumns = list(search_dict)
    
    #now we'll address any aggregations fo search terms
    #from search parameter passed with embedded list of search terms
          
    #then after populating activations, we'll put this below
    #inverse_search_dict has key of search term and value of column for activations
    inverse_search_dict = {value:key for key,value in search_dict.items()}
    newcolumns_before_aggregation = newcolumns.copy()
    
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      if aggregated_dict_key in inverse_search_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
        
        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

          mdf_train = \
          self.__autowhere(mdf_train, aggregated_dict_key_column, mdf_train[target_for_aggregation_column] == 1, 1, specified='replacement')
          mdf_test = \
          self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')
          
          del mdf_train[target_for_aggregation_column]
          del mdf_test[target_for_aggregation_column]
          
          newcolumns.remove(target_for_aggregation_column)
    
    for newcolumn in newcolumns:

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
      
    #remove temporary support column
    del mdf_train[suffixcolumn]
    del mdf_test[suffixcolumn]

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'search_dict' : search_dict, \
                                                'inverse_search_dict' : inverse_search_dict, \
                                                'srch_newcolumns_srch'   : newcolumns, \
                                                'newcolumns_before_aggregation' : newcolumns_before_aggregation, \
                                                'search' : search, \
                                                'search_preflattening' : search_preflattening, \
                                                'aggregated_dict' : aggregated_dict, \
                                                'suffix' : suffix, \
                                                'inplace' : inplace, \
                                                'defaultinfill_dict' : defaultinfill_dict, \
                                                'case' : case}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_src2(self, mdf_train, mdf_test, column, category, \
                        treecategory, postprocess_dict, params = {}):
    """
    #process_src2(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #assumes that unique values of test set are same or subset of train set
    #for more efficient application in postmunge
    """

    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #for this transform, suffixcolumn is not returned in final set
    #applying so that we can allow defaultinfill support without overwriting the input column
    
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
      
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #convert to uppercase string when case sensitivity not desired based on case parameter
    if case is False:
      #convert column to uppercase string except for nan infill points
      mdf_train = \
      self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == mdf_train[suffixcolumn], mdf_train[suffixcolumn].astype(str).str.upper(), specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str).str.upper(), specified='replacement')
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[suffixcolumn].unique())

    unique_list = list(map(str, unique_list))
    
#     maxlength = max(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)
          
    #when case sensitivity not desired convert search terms to uppercase strings
    if case is False:
      for i in range(len(search)):
        search[i] = str(search[i]).upper()
      #similarly convert aggregated_dict keys and value lists to uppercase
      aggregated_dict_preconvert = self.__autocopy(aggregated_dict)
      aggregated_dict = {}
      for key, value in aggregated_dict_preconvert.items():
        for i in range(len(value)):
          value[i] = str(value[i]).upper()
        aggregated_dict.update({str(key).upper() : value})

    #we'll populate overlap_dict as
    #{search_string : [list of associate categories with that overlap found]}
    
    overlap_dict = {}
    
    for search_string in search:
      
      overlap_dict.update({search_string : []})
    
    for search_string in search:
      
      len_search_string = len(search_string)
    
      for unique in unique_list:
        
        len_unique = len(unique)
        
        if len_unique >= len_search_string:
          
          nbr_iterations = len_unique - len_search_string
          
          for i in range(nbr_iterations + 1):
            
            extract = unique[i:(len_search_string+i)]
            
            if extract in search:
              
              overlap_dict[extract].append(unique)
    
#     #now for mdf_test
    
#     unique_list_test = list(mdf_test[column].unique())

#     unique_list_test = list(map(str, unique_list_test))

#     test_overlap_dict = {}
    
#     for search_string in search:
      
#       test_overlap_dict.update({search_string : []})
    

#     train_keys = list(overlap_dict)

#     train_keys.sort(key = len, reverse=True)

#     for dict_key in train_keys:

#       for unique_test in unique_list_test:

#         len_key = len(dict_key)

#         if len(unique_test) >= len_key:

#           nbr_iterations4 = len(unique_test) - len_key

#           for l in range(nbr_iterations4 + 1):

#             extract4 = unique_test[l:(len_key+l)]

#             if extract4 == dict_key:

#               test_overlap_dict[dict_key].append(unique_test)
    
    newcolumns = []

    for dict_key in overlap_dict:
      
      if len(overlap_dict[dict_key]) > 0:

        newcolumn = column + '_' + suffix + '_' + dict_key

        mdf_train, suffixoverlap_results = \
        self.__df_copy_train(mdf_train, suffixcolumn, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
        
        mdf_test[newcolumn] = mdf_test[suffixcolumn].copy()

        mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(overlap_dict[dict_key])
#         mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
    
    #now in case there are any aggregated activations, inspired by approach in srch
    inverse_search_dict = dict(zip(search, newcolumns))
    newcolumns_before_aggregation = newcolumns.copy()
    
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      if aggregated_dict_key in inverse_search_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
        
        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

          mdf_train = \
          self.__autowhere(mdf_train, aggregated_dict_key_column, mdf_train[target_for_aggregation_column] == 1, 1, specified='replacement')
          mdf_test = \
          self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')
          
          del mdf_train[target_for_aggregation_column]
          del mdf_test[target_for_aggregation_column]
          
          newcolumns.remove(target_for_aggregation_column)
        
    for newcolumn in newcolumns:
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
      
    #remove temporary support column
    del mdf_train[suffixcolumn]
    del mdf_test[suffixcolumn]

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'overlap_dict' : overlap_dict, \
                                                'src2_newcolumns_src2'   : newcolumns, \
                                                'newcolumns_before_aggregation' : newcolumns_before_aggregation, \
                                                'search' : search, \
                                                'inverse_search_dict' : inverse_search_dict, \
                                                'aggregated_dict' : aggregated_dict, \
                                                'case' : case, \
                                                'suffix' : suffix, \
                                                'inplace' : inplace, \
                                                'defaultinfill_dict' : defaultinfill_dict,
                                                'search_preflattening' : search_preflattening}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list
  
  def _process_src3(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #process_src3(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #where srch is preferred for unbounded range of unique values
    
    #and src2 preferred when have bounded range of unique values for both train & test
    
    #and speculation is that src3 may be preferred when have a bounded
    #range of unique values but still want capacity to handle values in 
    #test set not found in train set
    """

    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #for this transform, suffixcolumn is not returned in final set
    #applying so that we can allow defaultinfill support without overwriting the input column
    
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
      
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #convert to uppercase string when case sensitivity not desired based on case parameter
    if case is False:
      #convert column to uppercase string except for nan infill points
      mdf_train = \
      self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == mdf_train[suffixcolumn], mdf_train[suffixcolumn].astype(str).str.upper(), specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str).str.upper(), specified='replacement')
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[suffixcolumn].unique())

    unique_list = list(map(str, unique_list))
    
#     maxlength = max(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)

    #when case sensitivity not desired convert search terms to uppercase strings
    if case is False:
      for i in range(len(search)):
        search[i] = str(search[i]).upper()
      
      #similarly convert aggregated_dict keys and value lists to uppercase
      aggregated_dict_preconvert = self.__autocopy(aggregated_dict)
      aggregated_dict = {}
      for key, value in aggregated_dict_preconvert.items():
        for i in range(len(value)):
          value[i] = str(value[i]).upper()
        aggregated_dict.update({str(key).upper() : value})

    #we'll populate overlap_dict as
    #{search_string : [list of associate categories with that overlap found]}
    
    overlap_dict = {}
    
    for search_string in search:
      
      overlap_dict.update({search_string : []})
    
    for search_string in search:
      
      len_search_string = len(search_string)
    
      for unique in unique_list:
        
        len_unique = len(unique)
        
        if len_unique >= len_search_string:
          
          nbr_iterations = len_unique - len_search_string
          
          for i in range(nbr_iterations + 1):
            
            extract = unique[i:(len_search_string+i)]
            
            if extract in search:
              
              overlap_dict[extract].append(unique)
           
    #now for mdf_test
    
    unique_list_test = list(mdf_test[suffixcolumn].unique())

    unique_list_test = list(map(str, unique_list_test))

    test_overlap_dict = {}
    
    for search_string in search:
      
      test_overlap_dict.update({search_string : []})
    
    train_keys = list(overlap_dict)

    train_keys.sort(key = len, reverse=True)

    for dict_key in train_keys:

      for unique_test in unique_list_test:

        len_key = len(dict_key)

        if len(unique_test) >= len_key:

          nbr_iterations4 = len(unique_test) - len_key

          for l in range(nbr_iterations4 + 1):

            extract4 = unique_test[l:(len_key+l)]

            if extract4 == dict_key:

              test_overlap_dict[dict_key].append(unique_test)
    
    newcolumns = []

    for dict_key in overlap_dict:
      
      if len(overlap_dict[dict_key]) > 0:

        newcolumn = column + '_' + suffix + '_' + dict_key

        mdf_train, suffixoverlap_results = \
        self.__df_copy_train(mdf_train, suffixcolumn, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
        
        mdf_test[newcolumn] = mdf_test[suffixcolumn].copy()

        mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
        mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
    #now in case there are any aggregated activations, inspired by approach in srch
    inverse_search_dict = dict(zip(search, newcolumns))
    newcolumns_before_aggregation = newcolumns.copy()
    
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      if aggregated_dict_key in inverse_search_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
        
        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

          mdf_train = \
          self.__autowhere(mdf_train, aggregated_dict_key_column, mdf_train[target_for_aggregation_column] == 1, 1, specified='replacement')
          mdf_test = \
          self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')
          
          del mdf_train[target_for_aggregation_column]
          del mdf_test[target_for_aggregation_column]
          
          newcolumns.remove(target_for_aggregation_column)
        
    #remove temporary support column
    del mdf_train[suffixcolumn]
    del mdf_test[suffixcolumn]

    textnormalization_dict = {}
    if len(newcolumns) > 0:
      textnormalization_dict = {newcolumns[0] : {'overlap_dict' : overlap_dict, \
                                                'srch_newcolumns_src3'   : newcolumns, \
                                                'aggregated_dict' : aggregated_dict, \
                                                'inverse_search_dict' : inverse_search_dict, \
                                                'case' : case, \
                                                'suffix' : suffix, \
                                                'inplace': inplace, \
                                                'defaultinfill_dict' : defaultinfill_dict, \
                                                'search' : search}}
    
    column_dict_list = []

    for tc in newcolumns:

      if tc != newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_src4(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #process_src4(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    
    #src4 builds on the srch by converting to an ordinal activation
    #with 0 reserved for no activations
    #note that if an entry was activated for multiple search terms
    #the order of entries in search parameter will dictate the final encoding
    #(e.g. entries at end of list are prioritized over beginning)
    """

    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    #for this transform, suffixcolumn is not returned in final set
    #applying so that we can allow defaultinfill support without overwriting the input column
    
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    if mdf_train[suffixcolumn].dtype.name == 'category':
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype('object')
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object')
      
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)
    
    newcolumns = []
    search_dict = {}
    for searchitem in search:
      search_dict.update({column + '_' + suffix + '_' + str(searchitem) : str(searchitem)})
      
    for newcolumn in search_dict:
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, newcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train = \
      self.__autowhere(mdf_train, newcolumn, mdf_train[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')

      mdf_test = \
      self.__autowhere(mdf_test, newcolumn, mdf_test[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')
    
    newcolumns = list(search_dict)

#     for newcolumn in newcolumns:

#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
#       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
      
    #ok now let's convert to ordinal for src4
    ordl_dict1 = {}
    ordl_dict2 = {}
    
    #reserve zero for no activations
    i = 1
    for newcolumn in newcolumns:
      ordl_dict1.update({i : newcolumn})
      ordl_dict2.update({newcolumn : i})
      i += 1
      
    mdf_train.loc[:, suffixcolumn] = 0
    mdf_test.loc[:, suffixcolumn] = 0
    
    for newcolumn in newcolumns:

      mdf_train = \
      self.__autowhere(mdf_train, suffixcolumn, mdf_train[newcolumn] == 1, ordl_dict2[newcolumn],  specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, suffixcolumn, mdf_test[newcolumn] == 1, ordl_dict2[newcolumn],  specified='replacement')
      
      del mdf_train[newcolumn]
      del mdf_test[newcolumn]
      
    #now we'll address any aggregations fo search terms
    #from search parameter passed with embedded list of search terms
          
    #then after populating activations, we'll put this below
    #inverse_search_dict has key of search term and value of column for activations
    inverse_search_dict = {value:key for key,value in search_dict.items()}
#     newcolumns_before_aggregation = newcolumns.copy()
      
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      if aggregated_dict_key in inverse_search_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
        aggregated_dict_key_encoding = ordl_dict2[aggregated_dict_key_column]
        
        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
          target_for_aggregation_encoding = ordl_dict2[target_for_aggregation_column]

          mdf_train = \
          self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == target_for_aggregation_encoding, aggregated_dict_key_encoding, specified='replacement')
          mdf_test = \
          self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == target_for_aggregation_encoding, aggregated_dict_key_encoding, specified='replacement')

    #we'll base the integer type on number of ordinal entries
    max_encoding = len(ordl_dict1)
    if max_encoding <= 255:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint8)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint8)
    elif max_encoding <= 65535:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint16)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint16)
    else:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint32)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint32)
    
    column_dict_list = []
    
    #newcolumns are based on the original srch transform
    #src4_newcolumns are after consolidating to ordinal encoding (single entry)
    src4_newcolumns = [suffixcolumn]

    textnormalization_dict = {}
    if len(src4_newcolumns) > 0:
      textnormalization_dict = {src4_newcolumns[0] : {'search_dict' : search_dict, \
                                                      'inverse_search_dict' : inverse_search_dict, \
                                                      'srch_newcolumns_src4' : newcolumns, \
                                                      'src4_newcolumns' : src4_newcolumns, \
                                                      'search' : search, \
                                                      'search_preflattening' : search_preflattening, \
                                                      'aggregated_dict' : aggregated_dict, \
                                                      'case' : case, \
                                                      'ordl_dict1' : ordl_dict1, \
                                                      'activations_list' : list(ordl_dict1), \
                                                      'defaultinfill_dict' : defaultinfill_dict, \
                                                      'suffix' : suffix, \
                                                      'inplace': inplace, \
                                                      'ordl_dict2' : ordl_dict2}}

    for tc in src4_newcolumns:

      if tc != src4_newcolumns[0]:
        textnormalization_dict = {}
      
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : src4_newcolumns, \
                           'categorylist' : src4_newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list
  
  def _process_aggt(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    #process_aggt(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #and aggregates differently spelled duplicates into single representation
    #based on user passed parameter 'aggregate'
    #which is a list of lists, where sublists are the aggregation groups
    #and the final representation will be the final item in list
    #note also supports passing aggregate as a single list of terms without embedded lists
    """

    suffixoverlap_results = {}
    
    if 'aggregate' in params:
      aggregate = params['aggregate']
    else:
      aggregate = [[]]

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    df, suffixoverlap_results = \
    self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    for sublist in aggregate:
      
      if not isinstance(sublist, list):
        
        sublist = aggregate
      
        length_sublist = len(sublist)

        for i in range(length_sublist-1):

          df = \
          self.__autowhere(df, suffixcolumn, df[suffixcolumn] == sublist[i], sublist[-1], specified='replacement')
          
        break
      
      else:
        
        length_sublist = len(sublist)

        for i in range(length_sublist-1):

          df = \
          self.__autowhere(df, suffixcolumn, df[suffixcolumn] == sublist[i], sublist[-1], specified='replacement')

    normalization_dict = {suffixcolumn : {'aggregate' : aggregate, 'suffix' : suffix, 'defaultinfill_dict' : defaultinfill_dict}}
    
    nmbrcolumns = [suffixcolumn]
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        normalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _process_strn(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    #process_strn(df, column, category, postprocess_dict)
    #parses string entries and if any strings present returns longest string
    #i.e. character subsets excluding numerical entries
    #note that since this transform doesn't return numerically enocded data
    #we are leaving infill in place as np.nan
    """

    suffixoverlap_results = {}

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    unique_list = list(df[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]
                  
                  has_number = False
                  
                  for j in range(len(extract)):
                    
                    if self.__is_number(extract[j]):
                      
                      has_number = True

  #                 extract_already_in_overlap_dict = False

                  if has_number is False:

                    overlap_dict.update({unique : extract})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
  
                  has_number = False
                  
                  for j in range(len(extract)):
                    
                    if self.__is_number(extract[j]):
                      
                      has_number = True

  #                 extract_already_in_overlap_dict = False

                  if has_number is False:
      
                    in_dict = True

                    overlap_dict.update({unique : extract})
                  
              if in_dict is False:

                overlap_dict.update({unique : np.nan})

    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    df[suffixcolumn] = df[column].copy()

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    df[suffixcolumn] = df[suffixcolumn].astype(str)
    df[suffixcolumn] = df[suffixcolumn].replace(overlap_dict)
    
#     #a few more metrics collected for driftreport
#     #get maximum value of training column
#     maximum = df[column + '_nmrc'].max()
#     #get minimum value of training column
#     minimum = df[column + '_nmrc'].min()
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'overlap_dict' : overlap_dict, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _process_strg(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #str function
    #accepts input of integer categoric sets, such as from an ordinal transform
    #and converts to strings for purposes of categoric recognition in some downstream libaries
    #(eg some libraries will treat integer label sets as targets for regression instead of classificaiton)
    #does not perform infill, just converts entries to string
    '''

    suffixoverlap_results = {}

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    strg_column = column + '_' + suffix
    
    df, suffixoverlap_results = \
    self.__df_copy_train(df, column, strg_column, suffixoverlap_results, postprocess_dict['printstatus'])

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, strg_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    df[strg_column] = df[strg_column].astype(str)

    column_dict_list = []

    column_dict = {strg_column : {'category' : treecategory, \
                                 'origcategory' : category, \
                                 'normalization_dict' : {strg_column:{'suffix' : suffix, 'defaultinfill_dict' : defaultinfill_dict}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [strg_column], \
                                 'categorylist' : [strg_column], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_nmrc(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    #process_nmrc(df, column, category, postprocess_dict)
    #parses string entries and if any numbers present returns numbers
    #entries without numbers present subject to infill
    #accepts parameters 
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    """

    suffixoverlap_results = {}
    
    if 'convention' in params:
      #accepts numbers/commas/spaces
      convention = params['convention']
    else:
      convention = 'numbers'
      
    if 'suffix' in params:
      #accepts string for suffix appender
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    nmrc_column = column + '_' + suffix
    
    df, suffixoverlap_results = \
    self.__df_copy_train(df, column, nmrc_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    unique_list = list(df[nmrc_column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
                  
                  if convention == 'numbers':
                  
                    if self.__is_number(extract):

                      overlap_dict.update({unique : float(extract)})
              
                  elif convention == 'commas':
                  
                    if self.__is_number_comma(extract):

                      overlap_dict.update({unique : float(extract.replace(',',''))})
                      
                  elif convention == 'spaces':
                  
                    if self.__is_number_EU(extract):

                      overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})
                      
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.__is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : float(extract)})

              if in_dict is False:

                overlap_dict.update({unique : np.nan})

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, nmrc_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    df[nmrc_column] = df[nmrc_column].astype(str)
    df[nmrc_column] = df[nmrc_column].replace(overlap_dict)

    df[nmrc_column] = pd.to_numeric(df[nmrc_column], errors='coerce')
    
    #get mean of training data
    mean = df[nmrc_column].mean()
    if mean != mean:
      mean = 0
      
    #replace missing data with training set mean as default infill
    df[nmrc_column] = df[nmrc_column].fillna(mean)
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = df[nmrc_column].max()
    #get minimum value of training column
    minimum = df[nmrc_column].min()
    
    #create list of columns
    nmbrcolumns = [nmrc_column]

    #populate data structures
    nmbrnormalization_dict = {nmrc_column : {'overlap_dict' : overlap_dict, \
                                            'mean' : mean, \
                                            'maximum' : maximum, \
                                            'minimum' : minimum, \
                                            'convention' : convention, \
                                            'defaultinfill_dict' : defaultinfill_dict,
                                            'suffix' : suffix }}
    
    column_dict_list = []
    
    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return df, column_dict_list

  def _process_nmr4(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #extract numeric partitions from categoric entries, test treated differently than train
    #accepts parameters
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    #test_same_as_train as True/False
    #where True copiues overlap_dict from train for test, False parses test entries not found in train
    """

    suffixoverlap_results = {}
    
    if 'convention' in params:
      #accepts numbers/commas/spaces
      convention = params['convention']
    else:
      convention = 'numbers'
      
    if 'suffix' in params:
      #accepts string for suffix appender
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'test_same_as_train' in params:
      #accepts boolean
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = True
      
    nmrc_column = column + '_' + suffix
    
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, nmrc_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[nmrc_column] = mdf_test[column].copy()
    
    #begin parsing train set
    
    unique_list = list(mdf_train[nmrc_column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
                  
                  if convention == 'numbers':
                  
                    if self.__is_number(extract):

                      overlap_dict.update({unique : float(extract)})
              
                  elif convention == 'commas':
                  
                    if self.__is_number_comma(extract):

                      overlap_dict.update({unique : float(extract.replace(',',''))})
                      
                  elif convention == 'spaces':
                  
                    if self.__is_number_EU(extract):

                      overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})
                      
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
  
                  if self.__is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : float(extract)})

              if in_dict is False:

                overlap_dict.update({unique : np.nan})

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, nmrc_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
                
    mdf_train[nmrc_column] = mdf_train[nmrc_column].astype(str)
    mdf_train[nmrc_column] = mdf_train[nmrc_column].replace(overlap_dict)
    
    #now test set
    test_unique_list = list(mdf_test[nmrc_column].unique())
    test_unique_list = list(map(str, test_unique_list))
    extra_test_unique = list(set(test_unique_list) - set(unique_list))

    test_overlap_dict = self.__autocopy(overlap_dict)
    
    if test_same_as_train is True:
      
      for test_unique in extra_test_unique:
        test_overlap_dict.update({str(test_unique) : np.nan})
      
    elif test_same_as_train is False:
      
      testmaxlength = max(len(x) for x in unique_list)

      overlap_lengths = list(range(testmaxlength, 0, -1))

  #     overlap_dict = {}

      for overlap_length in overlap_lengths:

        for unique in extra_test_unique:

          if unique not in test_overlap_dict:

            len_unique = len(unique)

            if len_unique >= overlap_length:

              if overlap_length > 1:

                nbr_iterations = len_unique - overlap_length

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if convention == 'numbers':
                    
                      if self.__is_number(extract):

                        test_overlap_dict.update({unique : float(extract)})
                  
                    elif convention == 'commas':
                    
                      if self.__is_number_comma(extract):

                        test_overlap_dict.update({unique : float(extract.replace(',',''))})
                        
                    elif convention == 'spaces':
                    
                      if self.__is_number_EU(extract):

                        test_overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})

              #else if overlap_length == 1    
              else:

                nbr_iterations = len_unique - overlap_length

                in_dict = False

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if self.__is_number(extract):

                      in_dict = True

                      test_overlap_dict.update({unique : float(extract)})
                    
                if in_dict is False:

                  test_overlap_dict.update({unique : np.nan})

    #apply defaultinfill to test data based on processdict entry
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, nmrc_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #great now that test_overlap_dict is populated
    mdf_test[nmrc_column] = mdf_test[nmrc_column].astype(str)
    mdf_test[nmrc_column] = mdf_test[nmrc_column].replace(test_overlap_dict)

    mdf_train[nmrc_column] = pd.to_numeric(mdf_train[nmrc_column], errors='coerce')
    mdf_test[nmrc_column] = pd.to_numeric(mdf_test[nmrc_column], errors='coerce')

    #get mean of training data
    mean = mdf_train[nmrc_column].mean()
    if mean != mean:
      mean = 0

    #replace missing data with training set mean as default infill
    mdf_train[nmrc_column] = mdf_train[nmrc_column].fillna(mean)
    mdf_test[nmrc_column] = mdf_test[nmrc_column].fillna(mean)
    
    #a few more metrics collected for driftreport
    maximum = mdf_train[nmrc_column].max()
    minimum = mdf_train[nmrc_column].min()
    
    #create list of columns
    nmbrcolumns = [nmrc_column]
    
    #populate data structures
    nmbrnormalization_dict = {nmrc_column : {'overlap_dict' : overlap_dict, \
                                            'mean' : mean, \
                                            'maximum' : maximum, \
                                            'minimum' : minimum, \
                                            'unique_list' : unique_list, \
                                            'maxlength' : maxlength, \
                                            'convention' : convention, \
                                            'suffix' : suffix, \
                                            'defaultinfill_dict' : defaultinfill_dict,
                                            'test_same_as_train' : test_same_as_train}}
    
    column_dict_list = []
    
    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return mdf_train, mdf_test, column_dict_list

  def _custom_train_ordl(self, df, column, normalization_dict):
    """
    #a rewrite of ordl transform in the custom train convention
    #this should benefit latency
    #return comparable form of output
    #and accept comaprable parameters
    #primary difference is cleaner code and trimmed a little fat
    #and returned missing data representation now defaults to 0

    #please note that due to its use in Binary dimensionality reduction,
    #the ordl normalization_dict has a few reserved strings
    """

    #ordered_overide is boolean to indicate if order of integer encoding basis will 
    #defer to cases when a column is a pandas categorical ordered set
    if 'ordered_overide' in normalization_dict:
      ordered_overide = normalization_dict['ordered_overide']
    else:
      ordered_overide = True
      normalization_dict.update({'ordered_overide' : ordered_overide})
      
    #frequency_sort changes the sorting of encodings from alphabetical to frequency of entries in train set
    #note that frequency_sort=True is default for ord3 and frequency_sort=False is default for ordl
    if 'frequency_sort' in normalization_dict:
      frequency_sort = normalization_dict['frequency_sort']
    else:
      frequency_sort = True
      normalization_dict.update({'frequency_sort' : frequency_sort})

    #all_activations is to base the full set of activations on user specification instead of training set
    if 'all_activations' in normalization_dict:
      #accepts False or a list of activation targets
      all_activations = normalization_dict['all_activations']
    else:
      all_activations = False
      normalization_dict.update({'all_activations' : all_activations})

    #add_activations is to include additional columns for entry activations even when not found in train set
    if 'add_activations' in normalization_dict:
      #accepts False or a list of added activation targets
      add_activations = normalization_dict['add_activations']
    else:
      add_activations = False
      normalization_dict.update({'add_activations' : add_activations})

    #less_activations is to remove entry activaiton columns even when entry found in train set
    if 'less_activations' in normalization_dict:
      #accepts False or a list of removed activation targets
      less_activations = normalization_dict['less_activations']
    else:
      less_activations = False
      normalization_dict.update({'less_activations' : less_activations})

    #consolidated_activations is to consolidate entries to a single common activation
    if 'consolidated_activations' in normalization_dict:
      #accepts False or a list of consolidated activation targets or a list of lists
      consolidated_activations = normalization_dict['consolidated_activations']
    else:
      consolidated_activations = False
      normalization_dict.update({'consolidated_activations' : consolidated_activations})
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    #str_convert defaults to True to avoid edge case with bytes type entries
    #accepts one of {True, False, 'conditional_on_bytes'}
    #where conditional_on_bytes sets as False unless bytes entries are present in train data then sets to True
    if 'str_convert' in normalization_dict:
      str_convert = normalization_dict['str_convert']
    else:
      str_convert = True
      # str_convert = 'conditional_on_bytes'
      normalization_dict.update({'str_convert' : str_convert})

    #null_activation is to have a distinct activation for missing data
    #which defaults to the 0 integer
    #note that when deactivated, missing data is grouped into whichever else entry is for the zero bucket
    #also accepts as 'Binary', which is used in Binary dimensionality reduction to return from inversion all zeros
    #for cases where test activation sets don't match any found in train activation sets
    #note that in Binary scenario we don't accept NaN entries in input
    if 'null_activation' in normalization_dict:
      null_activation = normalization_dict['null_activation']
    else:
      null_activation = True
      normalization_dict.update({'null_activation' : null_activation})
      
    #_____
    
    #for every derivation related to the set labels_train, we'll remove missing_marker and add once prior to assembling binaryencoding_dict
    #which helps accomodate a few peculiarities related to python sets with NaN inclusion
    missing_marker = None
    if null_activation == 'Binary':
      missing_marker = '0' * len(str(df[column].iat[0]))
    normalization_dict.update({'missing_marker' : missing_marker})

    #conditional_on_bytes str_convert scenario resets to True if train set bytes entries present, otherwise resets to False
    if str_convert == 'conditional_on_bytes' \
    and (df[column] == df[column].astype(bytes)).sum() > 0:
      str_convert = True
      normalization_dict.update({'str_convert' : str_convert})
    elif str_convert == 'conditional_on_bytes':
      str_convert = False
      normalization_dict.update({'str_convert' : str_convert})
    
    #labels_train will be adjusted through derivation and serves as basis for binarization encoding
    labels_train = set()
    
    ordered = False

    #ordered_overide is not compatible with activation parameters
    if ordered_overide:
      if df[column].dtype.name == 'category' and df[column].cat.ordered:
        ordered = True
        labels_train_ordered = list(df[column].cat.categories)
        #by convention NaN is reserved for use with missing data
        labels_train_ordered = [x for x in labels_train_ordered if x==x]
        if str_convert is True:
          labels_train_ordered = [str(x) for x in labels_train_ordered]
        
    #frequency_sort derives a sorting order based on frequency of entries found in set
    if ordered is False and frequency_sort is True:
      ordered = True
      
      tempstring = 'asdf'
      if tempstring in df:
        while tempstring in df:
          tempstring += 'z'
      
      if str_convert is True:
        labels_train_ordered = pd.DataFrame(df[column].astype(str).value_counts())
      else:
        labels_train_ordered = pd.DataFrame(df[column].value_counts())
      #note this sorting has an edge case for bytes type entries when str_convert passed as False
      labels_train_ordered = labels_train_ordered.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
      labels_train_ordered = list(labels_train_ordered.index)
      #by convention NaN is reserved for use with missing data
      labels_train_ordered = [x for x in labels_train_ordered if x==x]
      if str_convert is True:
        labels_train_ordered = [str(x) for x in labels_train_ordered]
    
    #_____
            
    if df[column].dtype.name == 'category':
      labels_train = set(df[column].cat.categories)
      labels_train = {x for x in labels_train if x==x}

    if labels_train == set():  
      labels_train = set(df[column].unique())
      labels_train = {x for x in labels_train if x==x}

    #if str_convert parameter activated replace numerical with string equivalent (for common encoding between e.g. 2=='2')
    if str_convert is True:
      #note this set excludes missing_marker
      labels_train = {str(x) for x in labels_train}
      #only convert non-NaN entries in target column
#         df.loc[df[column] == df[column], (column)] = df[column].astype(str)
      df = \
      self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')

    #now we have a few activation set related parameters, applied by adjusting labels_train
    #we'll have convention that in cases where activation parameters are assigned, will overide ordered_overide (for alphabetic sorting)

    if all_activations is not False or less_activations is not False:
      #labels_train_orig is a support record that won't be returned
      labels_train_orig = labels_train.copy()

    if all_activations is not False:
      all_activations = {x for x in set(all_activations) if x==x}
      labels_train = all_activations

    if add_activations is not False:
      add_activations = {x for x in set(add_activations) if x==x}
      labels_train = labels_train | add_activations

    if less_activations is not False:
      less_activations = {x for x in set(less_activations) if x==x}
      labels_train = labels_train - less_activations

    #______

    #now we'll take account for any activation consolidations from consolidated_activations parameter

    #as part of this implementation, we next want to derive
    #a version of labels_train excluding consolidations (labels_train)
    #a list of consolidations associated with each returned_consolidation mapped to the returned_consolidation (consolidation_translate_dict)
    #and an inverse_consolidation_translate_dict mapping consolidated entries to their activations
    #which we'll then apply with a replace operation

    labels_train_before_consolidation = labels_train.copy()
    consolidation_translate_dict = {}
    inverse_consolidation_translate_dict = {}

    if consolidated_activations is not False:

      #if user passes a single tier list instead of list of lists we'll embed in a list
      if isinstance(consolidated_activations, list) and len(consolidated_activations) > 0:
        if not isinstance(consolidated_activations[0], list):
          consolidated_activations = [consolidated_activations]
          normalization_dict.update({'consolidated_activations' : consolidated_activations})

      for consolidation_list in consolidated_activations:

        #here is where we add any consolidation targets that weren't present in labels_train
        if str_convert is True:
          consolidation_list = [str(x) for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)
        else:
          #by convention missing data marker not elligible for inclusion in consolidation_list due to NaN/set peculiarities
          #consolidations with NaN can be accomodated by assignnan to treat desired entries as missing data
          consolidation_list = [x for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)

        #no prepare a version of labels_train excluding consolidations (labels_train)

        #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
        returned_consolidation = consolidation_list[0]

        #now remove consolidated entries from labels_train
        #and map a list of consolidations associated with each returned_consolidation to the returned_consolidation (consolidation_translate_dict)
        for consolidation_list in consolidated_activations:

          if len(consolidation_list) > 1:

            #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
            returned_consolidation = consolidation_list[0]

            labels_train = labels_train - set(consolidation_list[1:])

            consolidation_translate_dict.update({returned_consolidation : consolidation_list[1:]})

      #now populate an inverse_consolidation_translate_dict mapping consolidated entries to their activations
      for key,value in consolidation_translate_dict.items():
        for consolidation_list_entry in value:
          inverse_consolidation_translate_dict.update({consolidation_list_entry : key})

      #we can then apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)

    del consolidation_translate_dict
    normalization_dict.update({'inverse_consolidation_translate_dict' : inverse_consolidation_translate_dict})
    del inverse_consolidation_translate_dict

    #____

    #there are a few activation parameter scenarios where we may want to replace train set entries with missing data marker
    if all_activations is not False or less_activations is not False:
      extra_entries = labels_train_orig - labels_train
      extra_entries = list({x for x in extra_entries if x==x})
      if len(extra_entries) > 0:
        plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
        df[column] = df[column].astype('object').replace(plug_dict)

      del labels_train_orig

    #____

    #now prepare our ordinal encoding
    
    #if there is a particular order to encodings we'll sort labels_train on basis of labels_train_ordered
    if ordered is True:
      #this converts labels_train to a sorted list
      labels_train = self.__list_sorting(labels_train_ordered, labels_train)
    
    elif ordered is False:
      #convert labels_train to list 
      #and add the missing data marker to first position which will result in all zero binarized representation
      labels_train = list(labels_train)
      labels_train = sorted(labels_train, key=str)
      
    #add our missing_marker, note adding as first position will result in 0 representation even in ordered scenario
    if null_activation is True:
      labels_train = [missing_marker] + labels_train
    elif null_activation == 'Binary':
      if missing_marker not in labels_train:
        labels_train = [missing_marker] + labels_train
      
    #get length of the list, then zip a dictionary from list and range(length)
    #the range values will be our ordinal points to replace the categories
    listlength = len(labels_train)
    ordinal_dict = dict(zip(labels_train, range(listlength)))
    
    if null_activation is False:
      ordinal_dict.update({np.nan : 0})
    
    normalization_dict.update({'ordinal_dict' : ordinal_dict})
    
    #replace the categories in train set via ordinal trasnformation
    df[column] = df[column].astype('object').replace(ordinal_dict)
    
    #driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for key in ordinal_dict:
      sumcalc = (df[column] == ordinal_dict[key]).sum() 
      ratio = sumcalc / df[column].shape[0]
      ordl_activations_dict.update({key:ratio})
    normalization_dict.update({'ordl_activations_dict' : ordl_activations_dict})
    
    return df, normalization_dict

  def _process_maxb(self, mdf_train, mdf_test, column, category, \
                   treecategory, postprocess_dict, params = {}):
    '''
    #process_maxb
    #function to translate an ord3 ordinal encoded categoric set
    #to a reduced number of activations based on a maxbincount parameter
    #where maxbincount can be passed as integer
    #for setting the maximum number of activations
    #alternately, user can set parameter minentrycount
    #as integer of the minimum number of entries in train set to include actiations
    #finally, minentryratio can also be passed as a float between 0-1
    #to designate the minimum ratio of entries in an activation to register
    #consolidated activations will be aggregated into the top activation
    #eg if maxbincount is 4, 0-3 will be retained and reaminder aggregated into 4
    #for missing values, uses adjacent cell infill as default
    #we'll set default values as False signalling not applied
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'maxbincount' in params:
      maxbincount = params['maxbincount']
    else:
      maxbincount = False
      
    #initialize parameters
    if 'minentrycount' in params:
      minentrycount = params['minentrycount']
    else:
      minentrycount = False
      
    #initialize parameters
    if 'minentryratio' in params:
      minentryratio = params['minentryratio']
    else:
      minentryratio = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #non integers are subject to infill
    mdf_train = \
    self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == mdf_train[suffixcolumn].round(), alternative = np.nan, specified='alternative')
    mdf_test = \
    self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn].round(), alternative = np.nan, specified='alternative')
    
    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #get maximum train set activation which for ord3 will be least frequent entry
    maxactivation = int(mdf_train[suffixcolumn].max())
    
    #first we'll inspect maxbincount
    bincount_maxactivation = maxactivation
    if maxbincount is not False:
      
      #current max activation
      if maxactivation > maxbincount:
        
        bincount_maxactivation = maxbincount

        mdf_train = \
        self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] < bincount_maxactivation, alternative = bincount_maxactivation, specified='alternative')
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] < bincount_maxactivation, alternative = bincount_maxactivation, specified='alternative')
    
    #then inspect minentrycount
    count_maxactivation = maxactivation
    if minentrycount is not False:
      
      if minentrycount >= 1:

        entry_counts = {}
        for i in range(maxactivation + 1):

          count = mdf_train[mdf_train[suffixcolumn] == i].shape[0]
          entry_counts.update({i : count})
          
        for i in entry_counts:
          
          if entry_counts[i] < minentrycount:
            
            count_maxactivation = i
            break
            
        if count_maxactivation < maxactivation:

          mdf_train = \
          self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] < count_maxactivation, alternative = count_maxactivation, specified='alternative')
          mdf_test = \
          self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] < count_maxactivation, alternative = count_maxactivation, specified='alternative')
      
    #else if minentrycount passed as a ratio
    ratio_maxactivation = maxactivation
    if minentryratio is not False:

      if minentryratio > 0. and minentryratio < 1.:

        train_row_count = mdf_train.shape[0]

        entry_ratios = {}
        for i in range(maxactivation + 1):

          ratio = mdf_train[mdf_train[suffixcolumn] == i].shape[0] / train_row_count
          entry_ratios.update({i : ratio})

        for i in entry_ratios:

          if entry_ratios[i] < minentryratio:

            ratio_maxactivation = i
            break

        if ratio_maxactivation < maxactivation:

          mdf_train = \
          self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] < ratio_maxactivation, alternative = ratio_maxactivation, specified='alternative')
          mdf_test = \
          self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] < ratio_maxactivation, alternative = ratio_maxactivation, specified='alternative')

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    #grab some driftreport metrics
    new_maxactivation = maxactivation
    if bincount_maxactivation < new_maxactivation:
      new_maxactivation = bincount_maxactivation
    if count_maxactivation < new_maxactivation:
      new_maxactivation = count_maxactivation
    if ratio_maxactivation < new_maxactivation:
      new_maxactivation = ratio_maxactivation
    
    consolidation_count = 0
    if new_maxactivation < maxactivation:
      consolidation_count = mdf_train[mdf_train[suffixcolumn] == new_maxactivation].shape[0]
      
    #set integer type based on encoding depth
    if new_maxactivation <= 255:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint8)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint8)
    elif new_maxactivation <= 65535:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint16)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint16)
    else:
      mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.uint32)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint32)

    normalization_dict = {suffixcolumn : {'bincount_maxactivation' : bincount_maxactivation, \
                                          'count_maxactivation' : count_maxactivation, \
                                          'ratio_maxactivation' : ratio_maxactivation, \
                                          'new_maxactivation' : new_maxactivation, \
                                          'orig_maxactivation' : maxactivation, \
                                          'consolidation_count' : consolidation_count, \
                                          'maxbincount' : maxbincount, \
                                          'minentrycount' : minentrycount, \
                                          'minentryratio' : minentryratio, \
                                          'suffix' : suffix, \
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'inplace' : inplace}}
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        normalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_ucct(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_ucct(mdf_train, mdf_test, column, category)
    #preprocess column with categories into unique class count sets
    #normalized by total row count
    #e.g. for each class in train set, 
    #counts instances and divides by total train set row count
    #(so values will fall in range 0-1)
    #test sets recive comparable encoding
    '''

    suffixoverlap_results = {}

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    #create new column for trasnformation
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    #convert column to object
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype('object')
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object')

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    mdf_train = \
    self.__autowhere(mdf_train, suffixcolumn, mdf_train[suffixcolumn] == mdf_train[suffixcolumn], mdf_train[suffixcolumn].astype(str), specified='replacement')
    mdf_test = \
    self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str), specified='replacement')
    
    labels_train = set(mdf_train[suffixcolumn].unique())
    origlen = len(labels_train)
    nanincluded = False
    labels_train = {x for x in labels_train if x==x}
    if len(labels_train) < origlen:
      nanincluded = True
  
    labels_test = set(mdf_test[suffixcolumn].unique())
    labels_test = {x for x in labels_test if x==x}
    
    #____
    
    #assemble the ordinal_dict
    #with key of class and value of normalized unique class count
    ordinal_dict = {}
    rowcount = mdf_train.shape[0]
    
    for item in labels_train:
      item_count = mdf_train[mdf_train[suffixcolumn] == item].shape[0]
      ordinal_dict.update({item: item_count / rowcount})

    ordinal_nan_value = 0
    if nanincluded:
      item_count = mdf_train[mdf_train[suffixcolumn].isna()].shape[0]
      ordinal_nan_value = item_count / rowcount
    
    #replace the cateogries in train set via ordinal trasnformation
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype('object').replace(ordinal_dict)
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].fillna(ordinal_nan_value)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object').replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object').replace(ordinal_dict)
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(ordinal_nan_value)
    
    categorylist = [suffixcolumn]  

    normalization_dict = {}
    if len(categorylist) > 0:
      normalization_dict = {categorylist[0] : {'ordinal_dict' : ordinal_dict, \
                                              'ordinal_nan_value' : ordinal_nan_value, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict}}
        
    column_dict_list = []
    
    for tc in categorylist:

      if tc != categorylist[0]:
        normalization_dict = {}
    
      column_dict = {tc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _custom_train_1010(self, df, column, normalization_dict):
    """
    #a rewrite of 1010 transform in the custom train convention
    #since is default categoric deserves a little more attention
    #this should benefit latency
    #return comparable form of output
    #and accept comaprable parameters
    #primary difference is cleaner code and trimmed a little fat
    #and returned missing data represenation now defaults to all 0's

    #please note that due to its use in Binary dimensionality reduction, 
    #the 1010 normalization_dict has a few reserved strings
    """

    #_____
    #First we'll access parameters from normalization_dict

    #all_activations is to base the full set of activations on user specification instead of training set
    if 'all_activations' in normalization_dict:
      #accepts False or a list of activation targets
      all_activations = normalization_dict['all_activations']
    else:
      all_activations = False
      normalization_dict.update({'all_activations' : all_activations})

    #add_activations is to include additional columns for entry activations even when not found in train set
    if 'add_activations' in normalization_dict:
      #accepts False or a list of added activation targets
      add_activations = normalization_dict['add_activations']
    else:
      add_activations = False
      normalization_dict.update({'add_activations' : add_activations})

    #less_activations is to remove entry activaiton columns even when entry found in train set
    if 'less_activations' in normalization_dict:
      #accepts False or a list of removed activation targets
      less_activations = normalization_dict['less_activations']
    else:
      less_activations = False
      normalization_dict.update({'less_activations' : less_activations})

    #consolidated_activations is to consolidate entries to a single common activation
    if 'consolidated_activations' in normalization_dict:
      #accepts False or a list of consolidated activation targets or a list of lists
      consolidated_activations = normalization_dict['consolidated_activations']
    else:
      consolidated_activations = False
      normalization_dict.update({'consolidated_activations' : consolidated_activations})
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    #str_convert defaults to True to align with other categoric encodings
    #even though 1010 does not have the edge case for bytes entries
    #accepts one of {True, False, 'conditional_on_bytes'}
    #where conditional_on_bytes sets as False unless bytes entries are present in train data then sets to True
    if 'str_convert' in normalization_dict:
      str_convert = normalization_dict['str_convert']
    else:
      str_convert = True
      normalization_dict.update({'str_convert' : str_convert})
      
    #null_activation is to have a distinct activation for missing data
    #which defaults to the 0 integer
    #note that when deactivated, missing data is grouped into whichever else entry is for the zero bucket
    #also accepts as 'Binary', which is used in Binary dimensionality reduction to return from inversion all zeros
    #for cases where test activation sets don't match any found in train activation sets
    #note that in Binary scenario we don't accept NaN entries in input
    if 'null_activation' in normalization_dict:
      null_activation = normalization_dict['null_activation']
    else:
      null_activation = True
      normalization_dict.update({'null_activation' : null_activation})
      
    #max_zero results in an encoding basis whereby higher frequency entries
    #have a higher prevelance of 0 entries in their encoding
    #the expectation is this may benefit applications with quantum circuits where 0 is lower energy state
    #this is similar philosophy to when they designed typewriters they put higher frequency letters further apart
    if 'max_zero' in normalization_dict:
      max_zero = normalization_dict['max_zero']
    else:
      max_zero = False
      normalization_dict.update({'max_zero' : max_zero})
      
    #______
    
    #for every derivation related to the set labels_train, we'll remove missing_marker and add once prior to assembling binaryencoding_dict
    #which helps accomodate a few peculiarities related to python sets with NaN inclusion
    missing_marker = None

    if null_activation == 'Binary':
      missing_marker = '0' * len(str(df[column].iat[0]))

    normalization_dict.update({'missing_marker' : missing_marker})

    #conditional_on_bytes str_convert scenario resets to True if train set bytes entries present, otherwise resets to False
    if str_convert == 'conditional_on_bytes' \
    and (df[column] == df[column].astype(bytes)).sum() > 0:
      str_convert = True
      normalization_dict.update({'str_convert' : str_convert})
    elif str_convert == 'conditional_on_bytes':
      str_convert = False
      normalization_dict.update({'str_convert' : str_convert})
    
    #labels_train will be adjusted through derivation and serves as basis for binarization encoding
    labels_train = set()
    
    #pandas category dtype may have already specified a set of valid entries
    if df[column].dtype.name == 'category':
      labels_train = set(df[column].cat.categories)
      labels_train = {x for x in labels_train if x==x}
    
    #setting to object allows mixed data types for .replace operations and removes complexity of pandas category dtype
    df[column] = df[column].astype('object')
    
    #if str_convert elected (for common encoding between e.g. 2=='2')
    if str_convert is True:
      df = \
      self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')

      #if we already had accessed from category dtype convert those to string 
      if labels_train != set():
        labels_train = set([str(x) for x in list(labels_train)])

    #extract categories for column labels if we didn't already for category dtype
    #note that .unique() extracts the labels as a numpy array which we convert to set
    if labels_train == set():
      labels_train = set(df[column].unique())
      labels_train = {x for x in labels_train if x==x}
      
    #in the max_zero case we'll sort unique entries by frequency
    if max_zero is True:

      tempstring = 'asdf'
      if tempstring in df:
        while tempstring in df:
          tempstring += 'z'
          
      labels_train_ordered = pd.DataFrame(df[column].value_counts())

      #note this sorting has an edge case for bytes type entries when str_convert passed as False
      labels_train_ordered = labels_train_ordered.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
      labels_train_ordered = list(labels_train_ordered.index)
      #by convention NaN is reserved for use with missing data
      labels_train_ordered = [x for x in labels_train_ordered if x==x]
    
    #______
    
    #now we have a few activation set related parameters, applied by adjusting labels_train
    #we'll have convention that in cases where activation parameters are assigned, will overide ordered_overide (for alphabetic sorting)

    if all_activations is not False or less_activations is not False:
      #labels_train_orig is a support record that won't be returned
      labels_train_orig = labels_train.copy()
    
    if all_activations is not False:
      all_activations = {x for x in set(all_activations) if x==x}
      labels_train = all_activations

    if add_activations is not False:
      add_activations = {x for x in set(add_activations) if x==x}
      labels_train = labels_train | add_activations

    if less_activations is not False:
      less_activations = {x for x in set(less_activations) if x==x}
      labels_train = labels_train - less_activations
    
    #______
    
    #now we'll take account for any activation consolidations from consolidated_activations parameter
    
    #as part of this implementation, we next want to derive
    #a version of labels_train excluding consolidations (labels_train)
    #a list of consolidations associated with each returned_consolidation mapped to the returned_consolidation (consolidation_translate_dict)
    #and an inverse_consolidation_translate_dict mapping consolidated entries to their activations
    #which we'll then apply with a replace operation

    labels_train_before_consolidation = labels_train.copy()
    consolidation_translate_dict = {}
    inverse_consolidation_translate_dict = {}

    if consolidated_activations is not False:
    
      #if user passes a single tier list instead of list of lists we'll embed in a list
      if isinstance(consolidated_activations, list) and len(consolidated_activations) > 0:
        if not isinstance(consolidated_activations[0], list):
          consolidated_activations = [consolidated_activations]
          normalization_dict.update({'consolidated_activations' : consolidated_activations})
          
      for consolidation_list in consolidated_activations:

        #here is where we add any consolidation targets that weren't present in labels_train
        #by convention missing data marker not elligible for inclusion in consolidation_list due to NaN/set peculiarities
        #consolidations with NaN can be accomodated by assignnan to treat desired entries as missing data
        if str_convert is True:
          consolidation_list = [str(x) for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)
        else:
          consolidation_list = [x for x in consolidation_list if x==x]
          labels_train = labels_train | set(consolidation_list)
      
        #no prepare a version of labels_train excluding consolidations (labels_train)

        #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
        returned_consolidation = consolidation_list[0]
      
        #now remove consolidated entries from labels_train
        #and map a list of consolidations associated with each returned_consolidation to the returned_consolidation (consolidation_translate_dict)
        for consolidation_list in consolidated_activations:
          
          if len(consolidation_list) > 1:

            #we'll take the first entry in list as the returned activation (relevant to normalization_dict)
            returned_consolidation = consolidation_list[0]
            
            labels_train = labels_train - set(consolidation_list[1:])
            
            consolidation_translate_dict.update({returned_consolidation : consolidation_list[1:]})
    
      #now populate an inverse_consolidation_translate_dict mapping consolidated entries to their activations
      for key,value in consolidation_translate_dict.items():
        for consolidation_list_entry in value:
          inverse_consolidation_translate_dict.update({consolidation_list_entry : key})
      
      #we can then apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)
      
    del consolidation_translate_dict
    normalization_dict.update({'inverse_consolidation_translate_dict' : inverse_consolidation_translate_dict})
    del inverse_consolidation_translate_dict
      
    #____
    
    #there are a few activation parameter scenarios where we may want to replace train set entries with missing data marker
    if all_activations is not False or less_activations is not False:
      extra_entries = labels_train_orig - labels_train
      extra_entries = list({x for x in extra_entries if x==x})
      if len(extra_entries) > 0:
        plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
        df[column] = df[column].astype('object').replace(plug_dict)

      del labels_train_orig
    
    #____
    
    #now prepare our binarization
    
    #convert labels_train to list 
    labels_train = list(labels_train)
    if max_zero is False:
      #and sort alphabetically
      labels_train = sorted(labels_train, key=str)
    elif max_zero is True:
      #else sort by frequency for max_zero case
      labels_train = self.__list_sorting(labels_train_ordered, labels_train)
    
    #add our missing_marker, note adding as first position will result in all 0 representation
    if null_activation is True:
      labels_train = [missing_marker] + labels_train
    elif null_activation == 'Binary':
      if missing_marker not in labels_train:
        labels_train = [missing_marker] + labels_train
    
    #get length of the list of activation targets
    listlength = len(labels_train)
    
    #calculate number of columns we'll need
    #using numpy since already imported, this could also be done with math library
    binary_column_count = int(np.ceil(np.log2(listlength)))
    
    normalization_dict.update({'binary_column_count' : binary_column_count})
    
    #initialize dictionary to store encodings
    binary_encoding_dict = {}
    
    if max_zero is False:
      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")

        #store the encoding in a dictionary
        binary_encoding_dict.update({labels_train[i] : encoding})
        
    elif max_zero is True:
      #this approach results in sorting the binary encodings by zero prevelance from high to low
      encodings = []
      for i in range(int(2**binary_column_count)):
        encodings.append( format(i, f"0{binary_column_count}b") )
        
      #one_count_dict has form {encoding : count of '1' in encoding}
      one_count_dict = \
      dict(zip(encodings, [pd.Series([int(x) for x in list(y)]).sum() for y in encodings]))
      
      encodings_sorted = \
      list(dict(sorted(one_count_dict.items(), key=lambda item: item[1])))
      
      #now populate binary_encoding_dict
      for i in range(listlength):
        binary_encoding_dict.update({labels_train[i] : encodings_sorted[i]})
      
    if null_activation is False:
      
      binary_encoding_dict.update({np.nan : format(0, f"0{binary_column_count}b")})
      
    normalization_dict.update({'binary_encoding_dict' : binary_encoding_dict})
    
    #____
    
    #driftreport metric _1010_activations_dict mapping the activation target to ratio of entries in train set
    _1010_activations_dict = {}
    for key in binary_encoding_dict:
      sumcalc = (df[column] == key).sum() 
      ratio = sumcalc / df[column].shape[0]
      _1010_activations_dict.update({key:ratio})
      
    normalization_dict.update({'_1010_activations_dict' : _1010_activations_dict})
    
    #____
    
    #now replace the entries in column with their binarization representation
    #note this representation is a string of 0's and 1's that will next be seperated into seperate columns
    df[column] = df[column].astype('object').replace(binary_encoding_dict)
    
    #now let's create a list of columns to store each entry of the binary encoding
    #note suffix overlap detection will take place later in the wrapper function
    _1010_columnlist = []
    
    for i in range(binary_column_count):
      _1010_columnlist.append(column + '_' + str(i))
      
    normalization_dict.update({'_1010_columnlist' : _1010_columnlist})
    
    #now let's store the encoding
    i=0
    for _1010_column in _1010_columnlist:

      if len(_1010_columnlist) > 1:
        df[_1010_column] = df[column].str.slice(i,i+1).astype(np.int8)
        i+=1
      else:
        df[_1010_column] = df[column].astype(np.int8)
      
    #now delete the support column
    del df[column]

    #prepare inverse_binary_encoding_dict for test set and inversion
    if null_activation is not False:
      inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items()}
    elif null_activation is False:
      inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items() if key==key}
    normalization_dict.update({'inverse_binary_encoding_dict' : inverse_binary_encoding_dict})

    return df, normalization_dict

  def _process_bshr(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to traditional business hours in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'start' in params:
      start = params['start']
    else:
      start = 9
      
    if 'end' in params:
      end = params['end']
    else:
      end = 17

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

    #convert improperly formatted values to datetime in new column
    df[suffixcolumn] = pd.to_datetime(df[column], errors = 'coerce', format='mixed')

    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[suffixcolumn] = df[suffixcolumn].dt.hour
    df[suffixcolumn] = df[suffixcolumn].between(start, end)

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill, recommend to otherwise apply naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    #reduce memory footprint
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)
    
    #create list of columns
    datecolumns = [suffixcolumn]

    #grab some driftreport metrics
    activationratio = df[suffixcolumn].sum() / df[suffixcolumn].shape[0]

    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'activationratio' : activationratio, 
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_wkdy(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to weekdays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #convert improperly formatted values to datetime in new column
    df[suffixcolumn] = pd.to_datetime(df[column], errors = 'coerce', format='mixed')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[suffixcolumn] = pd.DatetimeIndex(df[suffixcolumn]).dayofweek
    
    df[suffixcolumn] = df[suffixcolumn].between(0,4)

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill, recommend to otherwise apply naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    #reduce memory footprint
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)
    
    #create list of columns
    datecolumns = [suffixcolumn]

    #grab some driftreport metrics
    activationratio = df[suffixcolumn].sum() / df[suffixcolumn].shape[0]

    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'activationratio' : activationratio, 
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}      

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'deletecolumn' : False, \
                           'suffixoverlap_results' : suffixoverlap_results}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_hldy(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to US Federal Holidays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'holiday_list' in params:
      holiday_list = params['holiday_list']
    else:
      holiday_list = []

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if len(holiday_list) > 0:
    
      #reformat holiday_list
      holiday_list = pd.to_datetime(pd.DataFrame(holiday_list)[0], errors = 'coerce', format='mixed')

      #reform holiday_list again
      timestamp_list = []

      for row in range(holiday_list.shape[0]):
        timestamp = pd.Timestamp(holiday_list[row])
        timestamp_list += [timestamp]
      timestamp_list
      
    else:
      timestamp_list = []
      
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #convert improperly formatted values to datetime in new column
    df[suffixcolumn] = pd.to_datetime(df[column], errors = 'coerce', format='mixed')
    
    df[suffixcolumn] = df[suffixcolumn].dt.date
    
    df[suffixcolumn] = pd.to_datetime(df[suffixcolumn], errors = 'coerce', format='mixed')
    
    from pandas.tseries.holiday import USFederalHolidayCalendar

    #grab list of holidays from import
    holidays = USFederalHolidayCalendar().holidays().tolist()

    holidays += timestamp_list
    
    #activate boolean identifier for holidays
    df[suffixcolumn] = df[suffixcolumn].isin(holidays)

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill, recommend to otherwise apply naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    #reduce memory footprint
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)
    
    #create list of columns
    datecolumns = [suffixcolumn]

    #grab some driftreport metrics
    activationratio = df[suffixcolumn].sum() / df[suffixcolumn].shape[0]

    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'activationratio' : activationratio, 
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list
  
  def _process_wkds(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a categorical column 
    #corresponding to weekdays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    #defdault infill is eight days a week
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert improperly formatted values to datetime in new column
    df[suffixcolumn] = pd.to_datetime(df[suffixcolumn], errors = 'coerce', format='mixed')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[suffixcolumn] = pd.DatetimeIndex(df[suffixcolumn]).dayofweek
    
#     df[column+'_wkdy'] = df[column+'_wkdy'].between(0,4)

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    #when defaultinfill passed as naninfill reverts to default infill of eight days a week
    df[suffixcolumn] = df[suffixcolumn].fillna(7)
    
    #reduce memory footprint
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)
    
    #create list of columns
    datecolumns = [suffixcolumn]

    #grab some driftreport metrics
    numberofrows = df[suffixcolumn].shape[0]
    mon_ratio = df[df[suffixcolumn] == 0].shape[0] / numberofrows
    tue_ratio = df[df[suffixcolumn] == 1].shape[0] / numberofrows
    wed_ratio = df[df[suffixcolumn] == 2].shape[0] / numberofrows
    thr_ratio = df[df[suffixcolumn] == 3].shape[0] / numberofrows
    fri_ratio = df[df[suffixcolumn] == 4].shape[0] / numberofrows
    sat_ratio = df[df[suffixcolumn] == 5].shape[0] / numberofrows
    sun_ratio = df[df[suffixcolumn] == 6].shape[0] / numberofrows
    infill_ratio = df[df[suffixcolumn] == 7].shape[0] / numberofrows
  
    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'mon_ratio' : mon_ratio, \
                                          'tue_ratio' : tue_ratio, \
                                          'wed_ratio' : wed_ratio, \
                                          'thr_ratio' : thr_ratio, \
                                          'fri_ratio' : fri_ratio, \
                                          'sat_ratio' : sat_ratio, \
                                          'sun_ratio' : sun_ratio, \
                                          'infill_ratio' : infill_ratio, \
                                          'suffix' : suffix, \
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list
  
  def _process_mnts(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a categorical column 
    #corresponding to months in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    #default infill is 0
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert improperly formatted values to datetime in new column
    df[suffixcolumn] = pd.to_datetime(df[suffixcolumn], errors = 'coerce', format='mixed')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[suffixcolumn] = pd.DatetimeIndex(df[suffixcolumn]).month

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
#     df[column+'_wkdy'] = df[column+'_wkdy'].between(0,4)

    #we'll use convention for default infill of eight days a week
    #jan-dec is 1-12, 0 is default infill
    df[suffixcolumn] = df[suffixcolumn].fillna(0)
    
    #reduce memory footprint
    df[suffixcolumn] = df[suffixcolumn].astype(np.int8)
    
    #create list of columns
    datecolumns = [suffixcolumn]

    #grab some driftreport metrics
    numberofrows = df[suffixcolumn].shape[0]
    infill_ratio = df[df[suffixcolumn] == 0].shape[0] / numberofrows
    jan_ratio = df[df[suffixcolumn] == 1].shape[0] / numberofrows
    feb_ratio = df[df[suffixcolumn] == 2].shape[0] / numberofrows
    mar_ratio = df[df[suffixcolumn] == 3].shape[0] / numberofrows
    apr_ratio = df[df[suffixcolumn] == 4].shape[0] / numberofrows
    may_ratio = df[df[suffixcolumn] == 5].shape[0] / numberofrows
    jun_ratio = df[df[suffixcolumn] == 6].shape[0] / numberofrows
    jul_ratio = df[df[suffixcolumn] == 7].shape[0] / numberofrows
    aug_ratio = df[df[suffixcolumn] == 8].shape[0] / numberofrows
    sep_ratio = df[df[suffixcolumn] == 9].shape[0] / numberofrows
    oct_ratio = df[df[suffixcolumn] == 10].shape[0] / numberofrows
    nov_ratio = df[df[suffixcolumn] == 11].shape[0] / numberofrows
    dec_ratio = df[df[suffixcolumn] == 12].shape[0] / numberofrows
  
    #create normalization dictionary
    normalization_dict = {suffixcolumn : {'infill_ratio' : infill_ratio, \
                                          'jan_ratio' : jan_ratio, \
                                          'feb_ratio' : feb_ratio, \
                                          'mar_ratio' : mar_ratio, \
                                          'apr_ratio' : apr_ratio, \
                                          'may_ratio' : may_ratio, \
                                          'jun_ratio' : jun_ratio, \
                                          'jul_ratio' : jul_ratio, \
                                          'aug_ratio' : aug_ratio, \
                                          'sep_ratio' : sep_ratio, \
                                          'oct_ratio' : oct_ratio, \
                                          'nov_ratio' : nov_ratio, \
                                          'dec_ratio' : dec_ratio, \
                                          'suffix' : suffix, \
                                          'defaultinfill_dict' : defaultinfill_dict,
                                          'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_tmzn(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that defaults as a passthrough
    #and when a 'timezone' parameter is received, 
    #converts timezone entries to UTC and then to the designated time zone
    #such as may be useful when a set of timestamps includes entries from multiple time zones
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #timezone can be passed as False for passthrough or timezone designation 
    #(where timezone designation is consistent with form accepted by Pandas tz_convert)
    if 'timezone' in params:
      timezone = params['timezone']
    else:
      timezone = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
      
    if timezone is not False:

      df[suffixcolumn] = pd.to_datetime(df[suffixcolumn], errors = 'coerce', utc=True, format='mixed')
      
      df[suffixcolumn] = df[suffixcolumn].dt.tz_convert(timezone)

      #apply defaultinfill based on processdict entry
      #(this will default to naninfill)
      df, defaultinfill_dict = \
      self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    datecolumns = [suffixcolumn]
    
    normalization_dict = {suffixcolumn : {'timezone' : timezone, \
                                          'suffix' : suffix, \
                                          'defaultinfill' : 'naninfill',
                                          'inplace' : inplace}}

    for dc in datecolumns:

      if dc != datecolumns[0]:
        normalization_dict = {}

      column_dict = { dc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_tmsc(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #time data segregated by time scale
    #with sin or cos applied to address periodicity
    #such as may be useful to return both by seperate transformation categories
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #note that some scales can be returned combined by passing 
    #monthday/dayhourminute/hourminutesecond/minutesecond
    #accepts parameter 'suffix' for returned column header suffix
    #accets parameter 'function' to distinguish between sin/cos
    """

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'scale' in params:
      #accepts year/month/day/hour/minute/second
      scale = params['scale']
    else:
      scale = 'monthday'
      
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'function' in params:
      #accepts sin/cos
      function = params['function']
    else:
      function = 'sin'
    
    time_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, time_column, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[time_column] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, time_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : time_column}, inplace = True)
      mdf_test.rename(columns = {column : time_column}, inplace = True)
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_train[time_column] = pd.to_datetime(mdf_train[time_column], errors = 'coerce', format='mixed')
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce', format='mixed')
    
    #access time scale from one of year/month/day/hour/minute/second
    #monthday/dayhourminute/hourminutesecond/minutesecond
    if scale == 'year':
      mdf_train[time_column] = mdf_train[time_column].dt.year
      mdf_test[time_column] = mdf_test[time_column].dt.year
      
      #we'll scale periodicity by decade
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 10
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 10
      
    elif scale == 'month':
      mdf_train[time_column] = mdf_train[time_column].dt.month
      mdf_test[time_column] = mdf_test[time_column].dt.month
      
      #we'll scale periodicity by year
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 12
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 12
      
    elif scale == 'day':
      mdf_train[time_column] = mdf_train[time_column].dt.day
      mdf_test[time_column] = mdf_test[time_column].dt.day
      
      #we'll scale periodicity by week
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 7
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 7
      
    elif scale == 'hour':
      mdf_train[time_column] = mdf_train[time_column].dt.hour
      mdf_test[time_column] = mdf_test[time_column].dt.hour
      
      #we'll scale periodicity by day
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 24
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 24
      
    elif scale == 'minute':
      mdf_train[time_column] = mdf_train[time_column].dt.minute
      mdf_test[time_column] = mdf_test[time_column].dt.minute
      
      #we'll scale periodicity by hour
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
      
    elif scale == 'second':
      mdf_train[time_column] = mdf_train[time_column].dt.second
      mdf_test[time_column] = mdf_test[time_column].dt.second
      
      #we'll scale periodicity by minute
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
    
    elif scale == 'monthday':
      tempcolumn1 = 1
      tempcolumn2 = 2
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, [tempcolumn1, tempcolumn2], suffixoverlap_results, postprocess_dict['printstatus'])
      
      #temp1 is for number of days in month, temp2 is to handle leap year support
      mdf_train[tempcolumn1] = mdf_train[time_column].copy()
      mdf_train[tempcolumn2] = mdf_train[time_column].copy()
      
      mdf_train[tempcolumn1] = mdf_train[tempcolumn1].dt.month
      mdf_train[tempcolumn2] = mdf_train[tempcolumn2].dt.is_leap_year

      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn2, mdf_train[tempcolumn2], 29, 28)

      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn1, mdf_train[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, specified='replacement')
      
      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn1, mdf_train[tempcolumn1].isin([4,6,9,11]), 30, specified='replacement')

      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn1, mdf_train[tempcolumn1].isin([2]), mdf_train[tempcolumn2], specified='replacement')
      
      #do same for test set
      mdf_test[tempcolumn1] = mdf_test[time_column].copy()
      mdf_test[tempcolumn2] = mdf_test[time_column].copy()
      
      mdf_test[tempcolumn1] = mdf_test[tempcolumn1].dt.month
      mdf_test[tempcolumn2] = mdf_test[tempcolumn2].dt.is_leap_year

      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn2, mdf_test[tempcolumn2], 29, 28)

      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, specified='replacement')

      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([4,6,9,11]), 30, specified='replacement')

      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([2]), mdf_test[tempcolumn2], specified='replacement')
      
      #combine month and day, scale for trigonomic transform, periodicity by year
      mdf_train[time_column] = (mdf_train[time_column].dt.month + mdf_train[time_column].dt.day / \
      mdf_train[tempcolumn1]) * 2 * np.pi / 12
      
      mdf_test[time_column] = (mdf_test[time_column].dt.month + mdf_test[time_column].dt.day / \
      mdf_test[tempcolumn1]) * 2 * np.pi / 12
      
      #delete the support columns 
      del mdf_train[tempcolumn1]
      del mdf_test[tempcolumn1]

      del mdf_train[tempcolumn2]
      del mdf_test[tempcolumn2]
      
    elif scale == 'dayhourminute':
      #we'll scale periodicity by week
      mdf_train[time_column] = (mdf_train[time_column].dt.day + mdf_train[time_column].dt.hour / 24 + mdf_train[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7
      mdf_test[time_column] = (mdf_test[time_column].dt.day + mdf_test[time_column].dt.hour / 24 + mdf_test[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7

    elif scale == 'hourminutesecond':
      #we'll scale periodicity by day
      mdf_train[time_column] = (mdf_train[time_column].dt.hour + mdf_train[time_column].dt.minute / 60 + mdf_train[time_column].dt.second / 60 / 60) * 2 * np.pi / 24
      mdf_test[time_column] = (mdf_test[time_column].dt.hour + mdf_test[time_column].dt.minute / 60 + mdf_test[time_column].dt.second / 60 / 60) * 2 * np.pi / 24

    elif scale == 'minutesecond':
      #we'll scale periodicity by hour
      mdf_train[time_column] = (mdf_train[time_column].dt.minute + mdf_train[time_column].dt.second / 60) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column].dt.minute + mdf_test[time_column].dt.second / 60) * 2 * np.pi / 60
      
    #grab a few drift metrics, we'll evaluate prior to trigometric transform
    timemean = mdf_train[time_column].mean()
    timemax = mdf_train[time_column].max()
    timemin = mdf_train[time_column].min()
    timestd = mdf_train[time_column].std()
    
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, time_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, time_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #we'll only return a column if meaningful training data present
    #because it is not uncommon for time series to only contain single time scale recording
    if mdf_train[time_column].nunique() > 1:

      #apply trigometric transform

      if function == 'sin':

        mdf_train[time_column] = np.sin(mdf_train[time_column])
        mdf_test[time_column] = np.sin(mdf_test[time_column])

      if function == 'cos':

        mdf_train[time_column] = np.cos(mdf_train[time_column])
        mdf_test[time_column] = np.cos(mdf_test[time_column])

      #populate data structures
      column_dict_list = []
      categorylist = [time_column]

      norm_dict = {}
      if len(categorylist) > 0:
        norm_dict = {categorylist[0] : {'scale'         : scale, \
                                        'suffix'        : suffix, \
                                        'function'      : function, \
                                        'timemean'      : timemean, \
                                        'timemax'       : timemax, \
                                        'timemin'       : timemin, \
                                        'timestd'       : timestd, \
                                        'defaultinfill_dict' : defaultinfill_dict,
                                        'inplace'       : inplace}}

      for tc in categorylist:

        if tc != categorylist[0]:
          norm_dict = {}

        column_dict = {tc : {'category' : treecategory, \
                             'origcategory' : category, \
                             'normalization_dict' : norm_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict)
        
    else:
      del mdf_train[time_column]
      del mdf_test[time_column]
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_time(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    """
    #z-score normalized time data segregated by a particular time scale
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #accepts parameter 'suffix' for returned column header suffix
    #accepts parameter 'normalization' to distinguish between zscore/minmax/unscaled
    """

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'scale' in params:
      #accepts year/month/day/hour/minute/second
      scale = params['scale']
    else:
      scale = 'year'
      
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'normalization' in params:
      #accepts zscore/minmax/unscaled
      normalization = params['normalization']
    else:
      normalization = 'zscore'
      
    time_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, time_column, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[time_column] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, time_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : time_column}, inplace = True)
      mdf_test.rename(columns = {column : time_column}, inplace = True)
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_train[time_column] = pd.to_datetime(mdf_train[time_column], errors = 'coerce', format='mixed')
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce', format='mixed')
    
    #access time scale from one of year/month/day/hour/minute/second
    if scale == 'year':
      mdf_train[time_column] = mdf_train[time_column].dt.year
      mdf_test[time_column] = mdf_test[time_column].dt.year
    elif scale == 'month':
      mdf_train[time_column] = mdf_train[time_column].dt.month
      mdf_test[time_column] = mdf_test[time_column].dt.month
    elif scale == 'day':
      mdf_train[time_column] = mdf_train[time_column].dt.day
      mdf_test[time_column] = mdf_test[time_column].dt.day
    elif scale == 'hour':
      mdf_train[time_column] = mdf_train[time_column].dt.hour
      mdf_test[time_column] = mdf_test[time_column].dt.hour
    elif scale == 'minute':
      mdf_train[time_column] = mdf_train[time_column].dt.minute
      mdf_test[time_column] = mdf_test[time_column].dt.minute
    elif scale == 'second':
      mdf_train[time_column] = mdf_train[time_column].dt.second
      mdf_test[time_column] = mdf_test[time_column].dt.second

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, time_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, time_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
    #grab a few drift metrics
    timemean = mdf_train[time_column].mean()
    timemax = mdf_train[time_column].max()
    timemin = mdf_train[time_column].min()
    timestd = mdf_train[time_column].std()

    maxminusmin = timemax - timemin
      
    #formula for scaling is (x - scaler) / divisor
    #normalizaiton is either zscore/minmax/unscaled
    if normalization == 'zscore':
      scaler = timemean
      divisor = timestd
    if normalization == 'minmax':
      scaler = timemin
      divisor = maxminusmin
    if normalization == 'unscaled':
      scaler = 0
      divisor = 1
    
    if divisor == 0 or divisor != divisor:
      divisor = 1
      
    if scaler != scaler:
      scaler = 0
      
    #apply normalization
    if normalization != 'unscaled':
      mdf_train[time_column] = (mdf_train[time_column] - scaler) / divisor
      mdf_test[time_column] = (mdf_test[time_column] - scaler) / divisor
      
    #we'll only return a column if meaningful training data present
    #because it is not uncommon for time series to only contain single time scale recording
    if mdf_train[time_column].nunique() > 1:
      
      #populate data structures
      column_dict_list = []
      categorylist = [time_column]

      norm_dict = {}
      if len(categorylist) > 0:
        norm_dict = {categorylist[0] : {'scale'         : scale, \
                                        'suffix'        : suffix, \
                                        'normalization' : normalization, \
                                        'scaler'        : scaler, \
                                        'divisor'       : divisor, \
                                        'timemean'      : timemean, \
                                        'timemax'       : timemax, \
                                        'timemin'       : timemin, \
                                        'timestd'       : timestd, \
                                        'maxminusmin'   : maxminusmin, \
                                        'defaultinfill_dict' : defaultinfill_dict,
                                        'inplace'       : inplace}}

      for tc in categorylist:

        if tc != categorylist[0]:
          norm_dict = {}

        column_dict = {tc : {'category' : treecategory, \
                             'origcategory' : category, \
                             'normalization_dict' : norm_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict)
        
    else:

      del mdf_train[time_column]
      del mdf_test[time_column]
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def _process_qttf(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    """
    Applies quantile transform
    Makes use of sklearn.preprocessing.quantile_transform
    From Scikit Learn library
    
    The thought is that we default to returning output distribution of normal instead of uniform 
    Because in general we expect surrounding variables will closer adhere (by degree) to normal vs. uniform
    And thus better approximate i.i.d. in aggregate
    This will differ from the scikit default of returning output_distribution as uniform
    
    default infill will be mean imputation performed after fitting but before applying quantile transform
    
    Accepts parameters (defaults) {accepted values}
    n_quantiles (1000) {integer}
    output_distribution ('normal') {'normal', 'uniform'} => differs from scikit default of 'uniform'
    ignore_implicit_zeros (False) {boolean}
    subsample (1e5) {integer}
    random_state (automunge randomseed) {integer}
    
    suffix (treecategory) {string}
    
    copy not supported
    fit parameters not supported
    inplace not supported since copying input more than once

    qttf returned as False when fit operation not performed
    """

    suffixoverlap_results = {}
    
    if 'n_quantiles' in params:
      n_quantiles = params['n_quantiles']
    else:
      n_quantiles = 1000
      
    if 'output_distribution' in params:
      output_distribution = params['output_distribution']
    else:
      output_distribution = 'normal'
      
    if 'ignore_implicit_zeros' in params:
      ignore_implicit_zeros = params['ignore_implicit_zeros']
    else:
      ignore_implicit_zeros = False
      
    if 'subsample' in params:
      subsample = params['subsample']
    else:
      subsample = int(1e5)
      
    if 'random_state' in params:
      random_state = params['random_state']
    else:
      random_state = postprocess_dict['randomseed']
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    #copy source column into new column
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

    mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #grab a few drift stats prior to trasnform
    input_max = mdf_train[suffixcolumn].max()
    input_min = mdf_train[suffixcolumn].min()
    input_stdev = mdf_train[suffixcolumn].std()
    input_mean = mdf_train[suffixcolumn].mean()

    if input_mean != input_mean:
      qttf = False
      mdf_train.loc[:, suffixcolumn] = 0
      mdf_test.loc[:, suffixcolumn] = 0
    
    else:
      from sklearn.preprocessing import QuantileTransformer

      #initialize quantile transformer
      qttf = QuantileTransformer(n_quantiles=n_quantiles, 
                                output_distribution=output_distribution,
                                ignore_implicit_zeros=ignore_implicit_zeros,
                                subsample=subsample,
                                random_state=random_state)
      
      #fit trasnformer to train set
      qttf.fit(pd.DataFrame(mdf_train[suffixcolumn]))
  
      #now reset the suffixcolumn since fitting process may replace nan with method value
      mdf_train[suffixcolumn] = mdf_train[column].copy()
      mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    
    #now default imputation with mean
    mean_impute = input_mean
    
    if mean_impute != mean_impute:
      mean_impute = 0
      
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #apply transform
    if qttf is not False:
      mdf_train[suffixcolumn] = pd.DataFrame(qttf.transform(pd.DataFrame(mdf_train[suffixcolumn])), index=mdf_train.index)
      mdf_test[suffixcolumn] = pd.DataFrame(qttf.transform(pd.DataFrame(mdf_test[suffixcolumn])), index=mdf_test.index)

    #output of a list of the created column names
    nmbrcolumns = [suffixcolumn]
    
    #initilize column_dict_list
    column_dict_list = []
    
    normalization_dict = {}
    if len(nmbrcolumns) > 0:
      normalization_dict = {nmbrcolumns[0] : {'input_max' : input_max,
                                              'input_min' : input_min,
                                              'input_stdev' : input_stdev,
                                              'input_mean' : input_mean,
                                              'qttf' : qttf,
                                              'mean_impute' : mean_impute,
                                              'n_quantiles' : n_quantiles,
                                              'output_distribution' : output_distribution,
                                              'ignore_implicit_zeros' : ignore_implicit_zeros,
                                              'subsample' : subsample,
                                              'random_state' : random_state,
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'suffix' : suffix}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        normalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return mdf_train, mdf_test, column_dict_list

  def _process_bxcx(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    Applies Box-Cox transform to an all-positive numerical set.
    
    We have a few scenarios where transform won't be applied signaled by setting bxcx_lmbda = False
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #convert non-positive values to nan
    mdf_train.loc[mdf_train[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    
    #get the mean value to apply to infill
    mean = mdf_train[suffixcolumn].mean()
    
    bxcx_lmbda = None
    test_bxcx_lmbda = None
    
    #edge case
    if mean != mean or mean <= 0:
      mean = 0
      bxcx_lmbda = False
    
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #edge case to avoid stats.boxcox error
    if mdf_train[suffixcolumn].nunique() == 1:
      #we'll use convention that if training data is set to 0 then so will all subsequent data
      bxcx_lmbda = False
      
    if mdf_test[suffixcolumn].nunique() == 1:
      test_bxcx_lmbda = False
      
    if bxcx_lmbda is not False:
      
      mdf_train[suffixcolumn], bxcx_lmbda = stats.boxcox(mdf_train[suffixcolumn])
      
      if test_bxcx_lmbda is not False:
        mdf_test[suffixcolumn] = stats.boxcox(mdf_test[suffixcolumn], lmbda = bxcx_lmbda)
      else:
        mdf_test.loc[:, suffixcolumn] = 0
      
    elif bxcx_lmbda is False:
      
      mdf_train.loc[:, suffixcolumn] = 0
      mdf_test.loc[:, suffixcolumn] = 0
      
    #this is to address an error when bxcx transform produces overflow
    #I'm not sure of cause, showed up in the housing set)
    max_train = mdf_train[suffixcolumn].max()
    max_test = mdf_test[suffixcolumn].max()
    
    if max_train > (2 ** 31 - 1):
      mdf_train.loc[:, suffixcolumn] = 0
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "overflow condition found in boxcox transform to train set, column set to 0: ", suffixcolumn)
      self.__autoprint('warning', '')
      
    if max_test > (2 ** 31 - 1):
      mdf_test.loc[:, suffixcolumn] = 0
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "overflow condition found in boxcox transform to test set, column set to 0: ", suffixcolumn)
      self.__autoprint('warning', '')
    
    #output of a list of the created column names
    nmbrcolumns = [suffixcolumn]
    
    #initilize column_dict_list
    column_dict_list = []
    
    normalization_dict = {}
    if len(nmbrcolumns) > 0:
      #test_bxcx_lmbda is False when mdf_test[suffixcolumn].nunique() == 1
      normalization_dict = {nmbrcolumns[0] : {'mean' : mean, \
                                              'bxcx_lmbda' : bxcx_lmbda, \
                                              'test_bxcx_lmbda' : test_bxcx_lmbda, \
                                              'max_train' : max_train, \
                                              'max_test' : max_test, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        normalization_dict = {}
      
      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return mdf_train, mdf_test, column_dict_list

  def _process_log0(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_log0(mdf_train, mdf_test, column, category)
    #function to apply logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base 10)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].replace(zeroreplace)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[suffixcolumn] = np.log10(mdf_train[suffixcolumn])
    mdf_test[suffixcolumn] = np.log10(mdf_test[suffixcolumn])
    
    #get mean of train set
    meanlog = mdf_train[suffixcolumn].mean()
    
    if meanlog != meanlog:
      meanlog = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

#     #replace missing data with 0
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].fillna(0)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'meanlog' : meanlog, 
                                              'inplace' : inplace, 
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_logn(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_logn(mdf_train, mdf_test, column, category)
    #function to apply natural logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base e)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].replace(zeroreplace)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[suffixcolumn] = np.log(mdf_train[suffixcolumn])
    mdf_test[suffixcolumn] = np.log(mdf_test[suffixcolumn])
    
    #get mean of train set
    meanlog = mdf_train[suffixcolumn].mean()
    
    if meanlog != meanlog:
      meanlog = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

#     #replace missing data with 0
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].fillna(0)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'meanlog' : meanlog, 
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace, 
                                              'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list
  
  def _process_sqrt(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_sqrt(mdf_train, mdf_test, column, category)
    #function to apply square root transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a square root transform
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].replace(zeroreplace)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[suffixcolumn] < 0, (suffixcolumn)] = np.nan
    mdf_test.loc[mdf_test[suffixcolumn] < 0, (suffixcolumn)] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[suffixcolumn] = np.sqrt(mdf_train[suffixcolumn])
    mdf_test[suffixcolumn] = np.sqrt(mdf_test[suffixcolumn])
    
    #get mean of train set
    meansqrt = mdf_train[suffixcolumn].mean()
    
    if meansqrt != meansqrt:
      meansqrt = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

#     #replace missing data with 0
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].fillna(0)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

#     #change data type for memory savings
#     mdf_train[suffixcolumn] = mdf_train[suffixcolumn].astype(np.float32)
#     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'meansqrt' : meansqrt, 
                                              'inplace' : inplace,
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list
  
  def _process_addd(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_addd(.)
    #function to apply addition transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'add' for amount of addition, otherwise defaults to adding 1
    #applies an addition transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'add' in params:
      add = params['add']
    else:
      add = 1

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #apply addition
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] + add
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] + add
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, \
                                              'add' : add, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_sbtr(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_sbtr(.)
    #function to apply subtraction transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'subtract' for amount of subtraction, otherwise defaults to subtracting 1
    #applies a subtraction transform
    #replaces non-numeric entries with set mean after subtraction
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'subtract' in params:
      subtract = params['subtract']
    else:
      subtract = 1

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #apply subtraction
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] - subtract
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - subtract
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, \
                                              'subtract' : subtract, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_mltp(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_mltp(.)
    #function to apply multiplication transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'multiply' for amount of addition, otherwise defaults to multiplying 2
    #applies an multiplication transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'multiply' in params:
      multiply = params['multiply']
    else:
      multiply = 2

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #apply multiplication
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] * multiply
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] * multiply
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, \
                                              'multiply' : multiply, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_divd(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_divd(.)
    #function to apply division transform
    #accepts parameter 'divide' for amount of division, otherwise defaults to dividing by 2
    #applies a division transform
    #replaces non-numeric entries with set mean after division
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'divide' in params:
      divide = params['divide']
    else:
      divide = 2

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
      
    #special case override to avoid div by 0
    if divide == 0:
      divide = 1
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #apply division
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] / divide
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / divide
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]


    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, \
                                              'divide' : divide, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_rais(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_rais(.)
    #function to apply raise to a power transform
    #accepts parameter 'raiser' for amount of power, otherwise defaults to square (raise by 2)
    #applies an raise transform
    #replaces non-numeric entries with set mean after raise
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'raiser' in params:
      raiser = params['raiser']
    else:
      raiser = 2

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

    #apply addition
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn] ** raiser
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn] ** raiser
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, \
                                              'raiser' : raiser, \
                                              'suffix' : suffix, \
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'inplace' : inplace}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list
  
  def _process_absl(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #process_absl(.)
    #function to apply absolute transform
    #does not accept paraemters
    #applies an absolute transform
    #replaces non-numeric entries with set mean after transform
    #returns same dataframes with new column of name suffixcolumn
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[suffixcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : suffixcolumn}, inplace = True)
      mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[suffixcolumn] = pd.to_numeric(mdf_train[suffixcolumn], errors='coerce')
    mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
    #apply addition
    mdf_train[suffixcolumn] = mdf_train[suffixcolumn].abs()
    mdf_test[suffixcolumn] = mdf_test[suffixcolumn].abs()
    
    #get mean of train set
    mean = mdf_train[suffixcolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'mean' : mean, 
                                              'inplace' : inplace,
                                              'defaultinfill_dict' : defaultinfill_dict,
                                              'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_pwrs(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating bins corresponding to powers
    #of ten in one hot encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #postiive values encoded under column 'column' + '_10^#' where # is power of 10
    #0 and negative values considered infill with no activations
    
    #if all values are infill no columns returned
    
    #accepts boolean 'negvalues' parameter, defaults False, True activates encoding for values <0
    '''

    suffixoverlap_results = {}
    
    if 'negvalues' in params:
      negvalues = params['negvalues']
    else:
      negvalues = False
      
    if 'zeroset' in params:
      zeroset = params['zeroset']
    else:
      zeroset = False
      
    #cap can be passed as a specific value prior to binning, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as a specific value prior to binning, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    suffixcolumn = column + '_' + suffix

    tempcolumn = 1
    tempcolumn_zero = suffixcolumn + '_zero'
    negtempcolumn = -1

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, tempcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[tempcolumn] = mdf_test[column].copy()
    
    #convert all values to either numeric or NaN
    mdf_train[tempcolumn] = pd.to_numeric(mdf_train[tempcolumn], errors='coerce')
    mdf_test[tempcolumn] = pd.to_numeric(mdf_test[tempcolumn], errors='coerce')
    
    maximum = mdf_train[tempcolumn].max()
    minimum = mdf_train[tempcolumn].min()
    if cap is not False:
      if cap > maximum:
        cap = False
    if floor is not False:
      if floor < minimum:
        floor = False
        
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[tempcolumn] > cap, (tempcolumn)] = cap
      mdf_test.loc[mdf_test[tempcolumn] > cap, (tempcolumn)] = cap
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[tempcolumn] < floor, (tempcolumn)] = floor
      mdf_test.loc[mdf_test[tempcolumn] < floor, (tempcolumn)] = floor
    
    #convert all values <= 0 to Nan
    mdf_train = \
    self.__autowhere(mdf_train, tempcolumn, mdf_train[tempcolumn] <= 0, np.nan, specified='replacement')
    mdf_test = \
    self.__autowhere(mdf_test, tempcolumn, mdf_test[tempcolumn] <= 0, np.nan, specified='replacement')
    
    #now log trasnform positive values in column column 
    mdf_train[tempcolumn] = np.floor(np.log10(mdf_train[tempcolumn].astype(float)))
    mdf_test[tempcolumn] = np.floor(np.log10(mdf_test[tempcolumn].astype(float)))

    train_pos_dict = {}
    newposunique_list = []
    posunique = mdf_train[tempcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = suffixcolumn + '_10^' + str(int(unique))
      train_pos_dict.update({unique : newunique})
      newposunique_list.append(newunique)
      
    test_pos_dict = {}
    posunique = mdf_test[tempcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = suffixcolumn + '_10^' + str(int(unique))
      if newunique in newposunique_list and newunique == newunique:
        test_pos_dict.update({unique : newunique})
      else:
        test_pos_dict.update({unique : np.nan})
    
    mdf_train[tempcolumn] = mdf_train[tempcolumn].replace(train_pos_dict)
    mdf_test[tempcolumn] = mdf_test[tempcolumn].replace(test_pos_dict)
      
    #now if the neg columns included do same      
    if negvalues is True:
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, negtempcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[negtempcolumn] = mdf_test[column].copy()
      
      #convert all values to either numeric or NaN
      mdf_train[negtempcolumn] = pd.to_numeric(mdf_train[negtempcolumn], errors='coerce')
      mdf_test[negtempcolumn] = pd.to_numeric(mdf_test[negtempcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_train.loc[mdf_train[negtempcolumn] > cap, (negtempcolumn)] = cap
        mdf_test.loc[mdf_test[negtempcolumn] > cap, (negtempcolumn)] = cap
      if floor is not False:
        #replace values in test < floor with floor
        mdf_train.loc[mdf_train[negtempcolumn] < floor, (negtempcolumn)] = floor
        mdf_test.loc[mdf_test[negtempcolumn] < floor, (negtempcolumn)] = floor

      #convert all values in negtempcolumn >= 0 to Nan
      mdf_train = \
      self.__autowhere(mdf_train, negtempcolumn, mdf_train[negtempcolumn] >= 0, np.nan, specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, negtempcolumn, mdf_test[negtempcolumn] >= 0, np.nan, specified='replacement')
          
      #take abs value of negtempcolumn
      mdf_train[negtempcolumn] = mdf_train[negtempcolumn].abs()
      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()

      #now log trasnform positive values in column column 
      mdf_train[negtempcolumn] = np.floor(np.log10(mdf_train[negtempcolumn].astype(float)))
      mdf_test[negtempcolumn] = np.floor(np.log10(mdf_test[negtempcolumn].astype(float)))
    
      #log transform column

      train_neg_dict = {}
      newunique_list = []
      negunique = mdf_train[negtempcolumn].unique()
      for unique in negunique:
        if unique != unique:
          newunique = np.nan
        else:
          #this is update for difference between pwr2 and pwrs
          if negvalues:
            newunique = suffixcolumn + '_-10^' + str(int(unique))
          else:
            newunique = np.nan
        train_neg_dict.update({unique : newunique})
        newunique_list.append(newunique)

      test_neg_dict = {}
      negunique = mdf_test[negtempcolumn].unique()
      for unique in negunique:
        if unique != unique:
          newunique = np.nan
        else:
          #this is update for difference between pwr2 and pwrs
          if negvalues:
            newunique = suffixcolumn + '_-10^' + str(int(unique))
          else:
            newunique = np.nan
        if newunique in newunique_list and newunique == newunique:
          test_neg_dict.update({unique : newunique})
        else:
          test_neg_dict.update({unique : np.nan})

      mdf_train[negtempcolumn] = mdf_train[negtempcolumn].replace(train_neg_dict)
      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
    
    #now if the zero column included is a little simpler
    if zeroset is True:
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, tempcolumn_zero, suffixoverlap_results, postprocess_dict['printstatus'])
      mdf_test[tempcolumn_zero] = mdf_test[column].copy()
      
      #convert all values to either numeric or NaN
      mdf_train[tempcolumn_zero] = pd.to_numeric(mdf_train[tempcolumn_zero], errors='coerce')
      mdf_test[tempcolumn_zero] = pd.to_numeric(mdf_test[tempcolumn_zero], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_train.loc[mdf_train[tempcolumn_zero] > cap, (tempcolumn_zero)] = cap
        mdf_test.loc[mdf_test[tempcolumn_zero] > cap, (tempcolumn_zero)] = cap
      if floor is not False:
        #replace values in test < floor with floor
        mdf_train.loc[mdf_train[tempcolumn_zero] < floor, (tempcolumn_zero)] = floor
        mdf_test.loc[mdf_test[tempcolumn_zero] < floor, (tempcolumn_zero)] = floor
      
      #convert to 1 activations for zero and 0 elsewhere
      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn_zero, mdf_train[tempcolumn_zero]==0, 1, 0)
      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn_zero, mdf_test[tempcolumn_zero]==0, 1, 0)

    #combine the positive and negative columns if applicable
    if negvalues is True:
      mdf_train = \
      self.__autowhere(mdf_train, tempcolumn, mdf_train[negtempcolumn] == mdf_train[negtempcolumn], mdf_train[negtempcolumn], specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn, mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[negtempcolumn], specified='replacement')

    #one hot encoding support function, using scenario zero which will derive column headers based on entries
    df_train_cat = self.__onehot_support(mdf_train, tempcolumn, scenario=0)
    df_test_cat = self.__onehot_support(mdf_test, tempcolumn, scenario=1, activations_list=list(df_train_cat))
    
    labels_train = list(df_train_cat)
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, list(df_train_cat), suffixoverlap_results, postprocess_dict['printstatus'])
    
    #concatinate the sparse set with the rest of our training data
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    #replace original column from training data
    if negvalues is True:
      del mdf_train[negtempcolumn]    
      del mdf_test[negtempcolumn]
    
    del mdf_train[tempcolumn]    
    del mdf_test[tempcolumn]
    
    #create output of a list of the created column names
    labels_train = list(df_train_cat)
    if zeroset is True:
      labels_train = [tempcolumn_zero] + labels_train

    powercolumns = labels_train
  
    #change data type for memory savings
    for powercolumn in powercolumns:
      mdf_train[powercolumn] = mdf_train[powercolumn].astype(np.int8)
      mdf_test[powercolumn] = mdf_test[powercolumn].astype(np.int8)
    
    normalizationdictvalues = labels_train
    normalizationdictkeys = powercolumns
    
    normalizationdictkeys.sort()
    normalizationdictvalues.sort()
    
    powerlabelsdict = dict(zip(normalizationdictkeys, normalizationdictvalues))
    
    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []

    powernormalization_dict = {}
    if len(powercolumns) > 0:
      #pos_and_negative_list excludes the zero column
      powernormalization_dict = {powercolumns[0] : {'powerlabelsdict_pwrs' : powerlabelsdict, \
                                                    'labels_train' : labels_train, \
                                                    'negvalues' : negvalues, \
                                                    'zeroset' : zeroset, \
                                                    'cap' : cap, \
                                                    'floor' : floor, \
                                                    'suffix' : suffix, \
                                                    'pos_and_negative_list' : list(df_train_cat)}}

    #drift stats for activation ratios
    powernormalization_dict.update({'activation_ratios' : {}})
    
    for pc in powercolumns:

      #new parameter collected for driftreport
      tc_ratio = pc + '_ratio'
      tcratio = mdf_train[pc].sum() / mdf_train[pc].shape[0]

      powernormalization_dict['activation_ratios'].update({tc_ratio : tcratio})
    
    for pc in powercolumns:

      if pc != powercolumns[0]:
        powernormalization_dict = {}
    
      column_dict = {pc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : powernormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : powercolumns, \
                           'categorylist' : powercolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}
        
      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_pwor(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating bins coresponding to powers
    #of ten in ordinal encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #infill has 0, other designations are based on the data
    
    #negative values based on negvalues parameter, makes comparable to por2
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'negvalues' in params:
      negvalues = params['negvalues']
    else:
      negvalues = False
      
    if 'zeroset' in params:
      zeroset = params['zeroset']
    else:
      zeroset = False
      
    #cap can be passed as a specific value prior to binning, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as a specific value prior to binning, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    pworcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, pworcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_test[pworcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, pworcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : pworcolumn}, inplace = True)
      mdf_test.rename(columns = {column : pworcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[pworcolumn] = pd.to_numeric(mdf_train[pworcolumn], errors='coerce')
    mdf_test[pworcolumn] = pd.to_numeric(mdf_test[pworcolumn], errors='coerce')
    
    maximum = mdf_train[pworcolumn].max()
    minimum = mdf_train[pworcolumn].min()
    if cap is not False:
      if cap > maximum:
        cap = False
    if floor is not False:
      if floor < minimum:
        floor = False
        
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[pworcolumn] > cap, (pworcolumn)] = cap
      mdf_test.loc[mdf_test[pworcolumn] > cap, (pworcolumn)] = cap
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[pworcolumn] < floor, (pworcolumn)] = floor
      mdf_test.loc[mdf_test[pworcolumn] < floor, (pworcolumn)] = floor
    
    #copy set for negative values
    if negvalues is True:
      negtempcolumn = -1

      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, pworcolumn, negtempcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[negtempcolumn] = mdf_test[pworcolumn].copy()
    
      #convert all values >= 0 to Nan
      mdf_train = \
      self.__autowhere(mdf_train, negtempcolumn, mdf_train[negtempcolumn] >= 0, np.nan, specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, negtempcolumn, mdf_test[negtempcolumn] >= 0, np.nan, specified='replacement')

      #take abs value of negtempcolumn
      mdf_train[negtempcolumn] = mdf_train[negtempcolumn].abs()
      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
      
    if zeroset is True:
      zerotempcolumn = 0
      
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, pworcolumn, zerotempcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[zerotempcolumn] = mdf_test[pworcolumn].copy()
      
      #convert all values != 0 to Nan
      mdf_train = \
      self.__autowhere(mdf_train, zerotempcolumn, mdf_train[zerotempcolumn] != 0, np.nan, column + '_zero')
      mdf_test = \
      self.__autowhere(mdf_test, zerotempcolumn, mdf_test[zerotempcolumn] != 0, np.nan, column + '_zero')
    
    #convert all values <= 0 in pworcolumn to Nan
    mdf_train = \
    self.__autowhere(mdf_train, pworcolumn, mdf_train[pworcolumn] <= 0, np.nan, specified='replacement')
    mdf_test = \
    self.__autowhere(mdf_test, pworcolumn, mdf_test[pworcolumn] <= 0, np.nan, specified='replacement')

    mdf_train[pworcolumn] = np.floor(np.log10(mdf_train[pworcolumn].astype(float)))
    mdf_test[pworcolumn] = np.floor(np.log10(mdf_test[pworcolumn].astype(float)))
    
    #now do same for column
    train_pos_dict = {}
    newposunique_list = []
    posunique = mdf_train[pworcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      train_pos_dict.update({unique : newunique})
      newposunique_list.append(newunique)
      
    test_pos_dict = {}
    posunique = mdf_test[pworcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      if newunique in newposunique_list and newunique == newunique:
        test_pos_dict.update({unique : newunique})
      else:
        test_pos_dict.update({unique : np.nan})
    
    mdf_train[pworcolumn] = mdf_train[pworcolumn].replace(train_pos_dict)
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_pos_dict)
    
    #do same for negtempcolumn
    if negvalues is True:
      mdf_train[negtempcolumn] = np.floor(np.log10(mdf_train[negtempcolumn].astype(float)))
      mdf_test[negtempcolumn] = np.floor(np.log10(mdf_test[negtempcolumn].astype(float)))

      train_neg_dict = {}
      newunique_list = []
      negunique = mdf_train[negtempcolumn].unique()
      for unique in negunique:
        if unique != unique:
          newunique = np.nan
        else:
          #this is update for difference between pwr2 and pwrs
          if negvalues:
            newunique = column + '_-10^' + str(int(unique))
          else:
            newunique = np.nan
        train_neg_dict.update({unique : newunique})
        newunique_list.append(newunique)

      test_neg_dict = {}
      negunique = mdf_test[negtempcolumn].unique()
      for unique in negunique:
        if unique != unique:
          newunique = np.nan
        else:
          #this is update for difference between pwr2 and pwrs
          if negvalues:
            newunique = column + '_-10^' + str(int(unique))
          else:
            newunique = np.nan
        if newunique in newunique_list and newunique == newunique:
          test_neg_dict.update({unique : newunique})
        else:
          test_neg_dict.update({unique : np.nan})

      mdf_train[negtempcolumn] = mdf_train[negtempcolumn].replace(train_neg_dict)
      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)

    #combine the columns
    if negvalues is True:
      mdf_train = \
      self.__autowhere(mdf_train, pworcolumn, mdf_train[negtempcolumn] == mdf_train[negtempcolumn], mdf_train[negtempcolumn], specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, pworcolumn, mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[negtempcolumn], specified='replacement')
      
    if zeroset is True:
      mdf_train = \
      self.__autowhere(mdf_train, pworcolumn, mdf_train[zerotempcolumn] == mdf_train[zerotempcolumn], mdf_train[zerotempcolumn], specified='replacement')
      mdf_test = \
      self.__autowhere(mdf_test, pworcolumn, mdf_test[zerotempcolumn] == mdf_test[zerotempcolumn], mdf_test[zerotempcolumn], specified='replacement')

    train_unique = mdf_train[pworcolumn].unique()
    test_unique = mdf_test[pworcolumn].unique()
  
    #Get missing entries in test set that are present in training set
    missing_cols = set( list(train_unique) ) - set( list(test_unique) )
    
    extra_cols = set( list(test_unique) ) - set( list(train_unique) )
    
    train_replace_dict = {}
    train_len = len(train_unique)
    nan_marker = False
    for i in range(train_len):
      if train_unique[i] != train_unique[i]:
        train_replace_dict.update({train_unique[i] : 0})
        nan_marker = True
      else:
        train_replace_dict.update({train_unique[i] : i+1})
    if nan_marker is False:
      train_replace_dict.update({np.nan : 0})
      
    test_replace_dict = {}
    for testunique in test_unique:
      if testunique in train_unique:
        test_replace_dict.update({testunique : train_replace_dict[testunique]})
      else:
        test_replace_dict.update({testunique : 0})
    
#     pworcolumn = column + '_por2'
#     mdf_train[pworcolumn] = mdf_train[column].copy()
#     mdf_test[pworcolumn] = mdf_test[column].copy()
    
    mdf_train[pworcolumn] = mdf_train[pworcolumn].replace(train_replace_dict)
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_replace_dict)
    
    #delete support columns
    if negvalues is True:
      del mdf_train[negtempcolumn]    
      del mdf_test[negtempcolumn]    
    if zeroset is True:
      del mdf_train[zerotempcolumn]    
      del mdf_test[zerotempcolumn] 
      
    bn_count = len(train_replace_dict)
    inverse_train_replace_dict = {value:key for key,value in train_replace_dict.items()}
    activations_list = list(inverse_train_replace_dict)

    max_encoding = max(list(inverse_train_replace_dict))
    if max_encoding <= 255:
      mdf_train[pworcolumn] = mdf_train[pworcolumn].astype(np.uint8)
      mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint8)
    elif max_encoding <= 65535:
      mdf_train[pworcolumn] = mdf_train[pworcolumn].astype(np.uint16)
      mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint16)
    else:
      mdf_train[pworcolumn] = mdf_train[pworcolumn].astype(np.uint32)
      mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint32)
   
    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []
    
    powercolumns = [pworcolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[pworcolumn].unique():
      sumcalc = (mdf_train[pworcolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[pworcolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})
    
    powernormalization_dict = {}
    if len(powercolumns) > 0:
      powernormalization_dict = {powercolumns[0] : {'train_replace_dict' : train_replace_dict, \
                                                    'inverse_train_replace_dict' : inverse_train_replace_dict, \
                                                    'activations_list' : activations_list, \
                                                    'test_replace_dict' : test_replace_dict, \
                                                    'ordl_activations_dict' : ordl_activations_dict, \
                                                    'negvalues' : negvalues, \
                                                    'zeroset' : zeroset, \
                                                    'cap' : cap, \
                                                    'floor' : floor, \
                                                    'suffix' : suffix, \
                                                    'inplace' : inplace}}

    for pc in powercolumns:

      if pc != powercolumns[0]:
        powernormalization_dict = {}
    
      column_dict = {pc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : powernormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : powercolumns, \
                           'categorylist' : powercolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}
        
      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_bins(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins will be intended for a raw set that is not normalized
    #bint will be intended for a previously normalized set
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 6 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    '''

    suffixoverlap_results = {}
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 6
    
    #if data is known to be z-score normalized we'll reduce the computational overhead
    if 'normalizedinput' in params:
      normalizedinput = params['normalizedinput']
    else:
      normalizedinput = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    binscolumn = column + '_' + suffix
    
    if bincount > 0:

      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if normalizedinput is False:

        #get mean of training data
        mean = mdf_train[binscolumn].mean()

        if mean != mean:
          mean = 0

        #get standard deviation of training data
        std = mdf_train[binscolumn].std()

        #special case, if standard deviation is 0 we'll set it to 1 to avoid division by 0
        if std == 0:
          std = 1

        if std != std:
          std = 1
      
      else:
        mean = 0
        std = 1
      
      #apply defaultinfill based on processdict entry
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_train[binscolumn] = (mdf_train[binscolumn] - mean) / std
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #derive cuts based on bincount

      bincuts = []

      mincut = - (bincount - 2) / 2
      bincuts.append(-float('inf'))

      for i in range(bincount - 1):
        bincuts.append(mincut)
        mincut += 1

      bincuts.append(float('inf'))

      binlabels = list(range(bincount))
      binlabels = list(map(str, binlabels))

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bins'
      mdf_train[binscolumn] = \
      pd.cut( mdf_train[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      #returned column headers
      textcolumns = []
      for binlabel in binlabels:
        textcolumns.append(binscolumn + '_' + binlabel)

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])

      #process bins as a categorical set
      df_train_cat = \
      self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_train_cat
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
    
      #store some values in the nmbr_dict{} for use later in ML infill methods
      column_dict_list = []

      nmbrnormalization_dict = {}
      if len(textcolumns) > 0:
        nmbrnormalization_dict = {textcolumns[0] : {'bincuts' : bincuts, \
                                                  'binlabels' : binlabels, \
                                                  'binscolumns' : textcolumns, \
                                                  'bincount' : bincount, \
                                                  'binsmean' : mean, \
                                                  'binsstd' : std, \
                                                  'normalizedinput' : normalizedinput, \
                                                  'suffix' : suffix, \
                                                  'defaultinfill_dict' : defaultinfill_dict}}

      #drift stats for activation ratios
      nmbrnormalization_dict.update({'activation_ratios' : {}})
      
      for nc in textcolumns:

        #new parameter collected for driftreport
        tc_ratio = nc + '_ratio'
        tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

        nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})
      
      for nc in textcolumns:

        if nc != textcolumns[0]:
          nmbrnormalization_dict = {}

        column_dict = { nc : {'category' : treecategory, \
                              'origcategory' : category, \
                              'normalization_dict' : nmbrnormalization_dict, \
                              'origcolumn' : column, \
                              'inputcolumn' : column, \
                              'columnslist' : textcolumns, \
                              'categorylist' : textcolumns, \
                              'infillmodel' : False, \
                              'infillcomplete' : False, \
                              'suffixoverlap_results' : suffixoverlap_results, \
                              'deletecolumn' : False}}

        column_dict_list.append(column_dict)
          
    else:
      
      column_dict_list = []

    return mdf_train, mdf_test, column_dict_list

  def _process_bsor(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    
    #bsor is comparable to bins but returns ordinal encoded column
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 6
      
    #if data is known to be z-score normalized we'll reduce the computational overhead
    if 'normalizedinput' in params:
      normalizedinput = params['normalizedinput']
    else:
      normalizedinput = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    binscolumn = column + '_' + suffix
    
    if bincount > 0:

      if inplace is not True:

        #copy source column into new column
        mdf_train, suffixoverlap_results = \
        self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

        mdf_test[binscolumn] = mdf_test[column].copy()

      else:

        suffixoverlap_results = \
        self.__df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

        mdf_train.rename(columns = {column : binscolumn}, inplace = True)
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)
      
      #convert all values to either numeric or NaN
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if normalizedinput is False:

        #get mean of training data
        mean = mdf_train[binscolumn].mean()
        if mean != mean:
          mean = 0

        #get standard deviation of training data
        std = mdf_train[binscolumn].std()
        if std == 0:
          std = 1
        if std != std:
          std = 1
          
      else:
        
        mean = 0
        std = 1

      #apply defaultinfill based on processdict entry
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_train[binscolumn] = (mdf_train[binscolumn] - mean) / std
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std
      
      #derive cuts based on bincount

      bincuts = []

      mincut = - (bincount - 2) / 2
      bincuts.append(-float('inf'))

      for i in range(bincount - 1):
        bincuts.append(mincut)
        mincut += 1

      bincuts.append(float('inf'))

      binlabels = list(range(bincount))

  #     binscolumn = column + '_bsor'
      mdf_train[binscolumn] = \
      pd.cut( mdf_train[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      ordinal_dict = {}
      for binlabel in binlabels:
        ordinal_dict.update({str(binlabel) : binlabel})

      #new driftreport metric ordl_activations_dict
      ordl_activations_dict = {}
      for key in ordinal_dict:
        sumcalc = (mdf_train[binscolumn] == ordinal_dict[key]).sum() 
        ratio = sumcalc / mdf_train[binscolumn].shape[0]
        ordl_activations_dict.update({key:ratio})

      inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
      activations_list = list(inverse_ordinal_dict)

      #data type is conditional based on encoding space
      max_encoding = len(ordinal_dict) - 1
      if max_encoding <= 255:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint8)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint16)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint32)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

      #create list of columns
      nmbrcolumns = [binscolumn]

      #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

      #store some values in the nmbr_dict{} for use later in ML infill methods
      column_dict_list = []

      nmbrnormalization_dict = {}
      if len(nmbrcolumns) > 0:
        nmbrnormalization_dict = {nmbrcolumns[0] : {'ordinal_dict' : ordinal_dict, \
                                                    'inverse_ordinal_dict' : inverse_ordinal_dict, \
                                                    'activations_list' : activations_list, \
                                                    'ordl_activations_dict' : ordl_activations_dict, \
                                                    'binsmean' : mean, \
                                                    'binsstd' : std, \
                                                    'normalizedinput' : normalizedinput, \
                                                    'bincount' : bincount, \
                                                    'bincuts' : bincuts, \
                                                    'binlabels' : binlabels, \
                                                    'suffix' : suffix, \
                                                    'defaultinfill_dict' : defaultinfill_dict,
                                                    'inplace' : inplace}}

      for nc in nmbrcolumns:

        if nc != nmbrcolumns[0]:
          nmbrnormalization_dict = {}

        column_dict = { nc : {'category' : treecategory, \
                              'origcategory' : category, \
                              'normalization_dict' : nmbrnormalization_dict, \
                              'origcolumn' : column, \
                              'inputcolumn' : column, \
                              'columnslist' : nmbrcolumns, \
                              'categorylist' : nmbrcolumns, \
                              'infillmodel' : False, \
                              'infillcomplete' : False, \
                              'suffixoverlap_results' : suffixoverlap_results, \
                              'deletecolumn' : False}}

        column_dict_list.append(column_dict)

    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list
  
  def _process_bnwd(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in one-hot encoded set
    
    #deletes columns without activations
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    suffixoverlap_results = {}
    
    if 'width' in params:
      bn_width = params['width']
    else:
      bn_width = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
    if bn_delta == 0:
      bn_delta = 1
    bn_count = int(np.ceil(bn_delta / bn_width))
    
    bins_id = []
    for i in range(bn_count):
      bins_id.append(str(bn_width) + '_' + str(i))
      
    bins_cuts = [-float('inf'), float('inf')]
    for i in range(bn_count-1):
      bins_cuts.insert(-1,(bn_min + (i+1) * bn_width))
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport  will return columns in alphabetical order
    textcolumns.sort()
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #process bins as a categorical set
    df_train_cat = \
    self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
    df_test_cat = \
    self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
  
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    del df_train_cat
    del df_test_cat
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(textcolumns) > 0:
      nmbrnormalization_dict = {textcolumns[0] : {'suffix' : suffix, \
                                                'binsmean' : mean, \
                                                'bn_min' : bn_min, \
                                                'bn_max' : bn_max, \
                                                'bn_delta' : bn_delta, \
                                                'bn_count' : bn_count, \
                                                'bins_id' : bins_id, \
                                                'bins_cuts' : bins_cuts, \
                                                'bn_width_bnwd' : bn_width, \
                                                'textcolumns' : textcolumns, \
                                                'defaultinfill_dict' : defaultinfill_dict}}

    #drift stats for activation ratios
    nmbrnormalization_dict.update({'activation_ratios' : {}})
    
    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})

    for nc in textcolumns:

      if nc != textcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)
       
    return mdf_train, mdf_test, column_dict_list

  def _process_bnwo(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'width' in params:
      bn_width = params['width']
    else:
      bn_width = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
    if bn_delta == 0:
      bn_delta = 1
    bn_count = int(np.ceil(bn_delta / bn_width))
    
    bins_id = []
    for i in range(bn_count):
      bins_id.append(i)
      
    bins_cuts = [-float('inf'), float('inf')]
    for i in range(bn_count-1):
      bins_cuts.insert(-1,(bn_min + (i+1) * bn_width))
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    
    #returned data type is conditional on the size of encoding space
    max_encoding = bn_count - 1
    if max_encoding <= 255:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint8)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
    elif max_encoding <= 65535:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint16)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
    else:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint32)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    activations_list = list(ordl_activations_dict)

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(nmbrcolumns) > 0:
      nmbrnormalization_dict = {nmbrcolumns[0] : {'suffix' : suffix, \
                                                  'binsmean' : mean, \
                                                  'bn_min' : bn_min, \
                                                  'bn_max' : bn_max, \
                                                  'bn_delta' : bn_delta, \
                                                  'bn_count' : bn_count, \
                                                  'bins_id' : bins_id, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bn_width' : bn_width, \
                                                  'activations_list' : activations_list, \
                                                  'ordl_activations_dict' : ordl_activations_dict, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list

  def _process_bnep(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    suffixoverlap_results = {}
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 5
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    #copy original column
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

#     if bn_delta > 0 and bn_min == bn_min:

    if bn_delta > 0 and bn_min == bn_min:

      #grab the intervals using qcut based on equal population in train set
      intervalset = pd.qcut(mdf_train[binscolumn], bincount, duplicates='drop').unique()

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operatoin
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(str(i))

      bins_cuts = cutintervals

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwd'
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      foundinset = mdf_train[binscolumn].unique()

      textcolumns = []
      for i in foundinset:
        if i == i:
          textcolumns.append(binscolumn + '_' + str(i))

      #postprocess_textsupport  will return columns in alphabetical order
      textcolumns.sort()
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])

      #process bins as a categorical set
      df_train_cat = \
      self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_train_cat
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
      
    else:
      mdf_train.loc[:, binscolumn] = 0
      mdf_test.loc[:, binscolumn] = 0
      
      textcolumns = [binscolumn]
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(textcolumns) > 0:
      nmbrnormalization_dict = {textcolumns[0] : {'suffix' : suffix, \
                                                  'binsmean' : mean, \
                                                  'bn_min' : bn_min, \
                                                  'bn_max' : bn_max, \
                                                  'bn_delta' : bn_delta, \
                                                  'bn_count' : bn_count, \
                                                  'bins_id' : bins_id, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bincount_bnep' : bincount, \
                                                  'textcolumns' : textcolumns}}

    #drift stats for activation ratios
    nmbrnormalization_dict.update({'activation_ratios' : {}})
    
    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})

    for nc in textcolumns:

      if nc != textcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)
       
    return mdf_train, mdf_test, column_dict_list

  def _process_bneo(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in ordinal encoded set
    
    #default infill is adjacent cell infill
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 5
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

    if bn_delta > 0 and bn_min == bn_min:

      #grab the intervals using qcut based on equal population in train set
      intervalset = pd.qcut(mdf_train[binscolumn], bincount, duplicates='drop').unique()

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operation
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(i)

      bins_cuts = cutintervals

      #create bins based on prepared increments
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      #apply defaultinfill based on processdict entry
      #(this will default to adjinfill)
      mdf_train, defaultinfill_dict = \
      self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
      
      #returned data type is conditional on the size of encoding space
      max_encoding = bn_count - 1
      if max_encoding <= 255:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint8)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint16)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint32)
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)
      
    else:
      
      mdf_train.loc[:, binscolumn] = 0
      mdf_test.loc[:, binscolumn] = 0
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False
      defaultinfill_dict = {'defaultinfill' : 'adjinfill'}

    #create list of columns
    nmbrcolumns = [binscolumn]

    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(nmbrcolumns) > 0:
      nmbrnormalization_dict = {nmbrcolumns[0] : {'suffix' : suffix, \
                                                  'binsmean' : mean, \
                                                  'bn_min' : bn_min, \
                                                  'bn_max' : bn_max, \
                                                  'bn_delta' : bn_delta, \
                                                  'bn_count' : bn_count, \
                                                  'bins_id' : bins_id, \
                                                  'activations_list' : bins_id, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bincount' : bincount, \
                                                  'ordl_activations_dict' : ordl_activations_dict, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list
  
  def _process_tlbn(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 9
    #and returning in one-hot encoded set

    #as alternate to specifying bincount for equal population bins
    #can also pass parameter buckets as list of bucket boundaries
    #leaving out -/+ inf in first and last bins as will be added
    
    #how this differs from bnep in that the activated bins are replaced with
    #min-max scaling for source column values found in that bin, and then other values as -1
    
    #note that for the bottom bin order reversed to accomodate subsequent values out of range 
    #and still use -1 register
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    suffixoverlap_results = {}
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 9
    bincount_orig = bincount
      
    #if buckets is passed as list of boundaries, it overrides bincount
    #note that should leave out -/+ inf in first and last bins (will be added)
    if 'buckets' in params:
      buckets = params['buckets']
    else:
      buckets = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    binscolumn = column + '_' + suffix

    #copy original column
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

    if bn_delta > 0 and bn_min == bn_min:
      
      if buckets is False or not isinstance(buckets, list):      
        #grab the intervals using qcut based on equal population in train set
        intervalset = pd.qcut(mdf_train[binscolumn], bincount, duplicates='drop').unique()
      else:
        intervalset = []
        #to be sort of consistent with what is returned from qcut
        #with difference that we're allowing buckets outside of range found in set
        for i in range(len(buckets)):
          if i == 0:
            #buckets[i]-1 will be replaced with -inf below
            intervalset.append(pd.Interval(buckets[i]-1, buckets[i]))
          elif i != len(buckets)-1:
            intervalset.append(pd.Interval(buckets[i-1], buckets[i]))
          else:
            intervalset.append(pd.Interval(buckets[i-1], buckets[i]))
            #buckets[i]+1 will be replaced with +inf below
            intervalset.append(pd.Interval(buckets[i], buckets[i]+1))
        bincount = len(intervalset)

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operatoin
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(str(i))

      bins_cuts = cutintervals

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwd'
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      foundinset = mdf_train[binscolumn].unique()

      textcolumns = []
      # for i in foundinset:
      for i in range(bn_count):
        if i == i:
          textcolumns.append(binscolumn + '_' + str(i))

      #postprocess_textsupport  will return columns in alphabetical order
      textcolumns.sort()
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])

      #process bins as a categorical set
      df_train_cat = \
      self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_train_cat
      del df_test_cat
      
      #initialize binscolumn once more
      mdf_train[binscolumn] = mdf_train[column].copy()
      mdf_test[binscolumn] = mdf_test[column].copy()
      
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if len(textcolumns) > 1:

        #for i in range(bincount):
        for i in range(len(textcolumns)):

          tlbn_column = binscolumn + '_' + str(i)

          if i == 0:

            mdf_train = \
            self.__autowhere(mdf_train, tlbn_column, mdf_train[tlbn_column] == 1, 
                            (bins_cuts[i+1] - mdf_train[binscolumn]) / (bins_cuts[i+1] - bn_min), -1)
            mdf_test = \
            self.__autowhere(mdf_test, tlbn_column, mdf_test[tlbn_column] == 1, 
                            (bins_cuts[i+1] - mdf_test[binscolumn]) / (bins_cuts[i+1] - bn_min), -1)

          # elif i == bincount - 1:
          elif i == len(textcolumns) - 1:

            mdf_train = \
            self.__autowhere(mdf_train, tlbn_column, mdf_train[tlbn_column] == 1, 
                            (mdf_train[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), -1)
            mdf_test = \
            self.__autowhere(mdf_test, tlbn_column, mdf_test[tlbn_column] == 1, 
                            (mdf_test[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), -1)

          else:

            mdf_train = \
            self.__autowhere(mdf_train, tlbn_column, mdf_train[tlbn_column] == 1, 
                            (mdf_train[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), -1)
            mdf_test = \
            self.__autowhere(mdf_test, tlbn_column, mdf_test[tlbn_column] == 1, 
                            (mdf_test[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), -1)

#       #change data type for memory savings
#       for textcolumn in textcolumns:
#         mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
#         mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
      
    else:
      mdf_train.loc[:, binscolumn] = 0
      mdf_test.loc[:, binscolumn] = 0
      
      textcolumns = [binscolumn]
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #there's still an edge scenario for custom buckets where nan get's populated, 
    #so this is just a hack to clean up
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].fillna(-1)
      mdf_test[textcolumn] = mdf_test[textcolumn].fillna(-1)

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(textcolumns) > 0:
      nmbrnormalization_dict = {textcolumns[0] : {'binsmean' : mean, \
                                                  'bn_min' : bn_min, \
                                                  'bn_max' : bn_max, \
                                                  'bn_delta' : bn_delta, \
                                                  'bn_count' : bn_count, \
                                                  'bins_id' : bins_id, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bincount' : bincount, \
                                                  'bincount_tlbn' : bincount_orig, \
                                                  'buckets_tlbn' : buckets, \
                                                  'textcolumns' : textcolumns, \
                                                  'suffix' : suffix}}

    #drift stats for activation ratios
    nmbrnormalization_dict.update({'activation_ratios' : {}})

    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})

    for nc in textcolumns:

      if nc != textcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)
       
    return mdf_train, mdf_test, column_dict_list
  
  def _process_bkt1(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set

    #default infill is no acitvations in a row
    '''

    suffixoverlap_results = {}
    
    #buckets can be passed as list for direct values or as a set to signal they are percent values
    if 'buckets' in params:
      buckets = params['buckets']
      origbuckets = params['buckets']
    else:
      buckets = [0,1,2]
      origbuckets = [0,1,2]

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0
    
    trainmax = mdf_train[binscolumn].max()
    trainmin = mdf_train[binscolumn].min()
    
    #if buckets is a set instead of list that signals to convert percentages to values
    if type(buckets) == type({1,2}):
      buckets = sorted(list(buckets))
      
      if trainmax != trainmax:
        trainmax = 0
      if trainmin != trainmin:
        trainmin = 0
        
      buckets = [(trainmax - trainmin) * x + trainmin for x in buckets]

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
    bins_cuts.insert(0, -np.inf)
    bins_cuts.insert(len(bins_cuts), np.inf)
    
    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))
    
    #create bins based on increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport  will return columns in alphabetical order
    textcolumns.sort()
    
    #remove nan for cases where value did not fall within range
    textcolumns = [x for x in textcolumns if x[-3:] != 'nan']
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #process bins as a categorical set
    df_train_cat = \
    self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
    df_test_cat = \
    self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
  
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    del df_train_cat
    del df_test_cat
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #create list of columns
    nmbrcolumns = textcolumns

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(textcolumns) > 0:
      nmbrnormalization_dict = {textcolumns[0] : {'binsmean' : mean, \
                                                  'buckets_bkt1' : buckets, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bins_id' : bins_id, \
                                                  'textcolumns' : textcolumns, \
                                                  'trainmax' : trainmax, \
                                                  'trainmin' : trainmin, \
                                                  'suffix' : suffix, \
                                                  'origbuckets_bkt1' : origbuckets}}

    #drift stats for activation ratios
    nmbrnormalization_dict.update({'activation_ratios' : {}})

    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})

    for nc in textcolumns:

      if nc != textcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)
       
    return mdf_train, mdf_test, column_dict_list
  
  def _process_bkt2(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''

    suffixoverlap_results = {}
    
    #buckets can be passed as list for direct values or as a set to signal they are percent values
    if 'buckets' in params:
      buckets = params['buckets']
      origbuckets = params['buckets']
    else:
      buckets = [0,1,2]
      origbuckets = [0,1,2]

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0
      
    trainmax = mdf_train[binscolumn].max()
    trainmin = mdf_train[binscolumn].min()
    
    #if buckets is a set instead of list that signals to convert percentages to values
    if type(buckets) == type({1,2}):
      buckets = sorted(list(buckets))
      
      if trainmax != trainmax:
        trainmax = 0
      if trainmin != trainmin:
        trainmin = 0
        
      buckets = [(trainmax - trainmin) * x + trainmin for x in buckets]

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
#     bins_cuts.insert(0, -np.inf)
#     bins_cuts.insert(len(bins_cuts), np.inf)

    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))
    
    #create bins based on increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport  will return columns in alphabetical order
    textcolumns.sort()
    
    #remove nan for cases where value did not fall within range
    textcolumns = [x for x in textcolumns if x[-3:] != 'nan']
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #process bins as a categorical set
    df_train_cat = \
    self.__onehot_support(mdf_train, binscolumn, scenario=2, activations_list = textcolumns)
    df_test_cat = \
    self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
  
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    del df_train_cat
    del df_test_cat
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(textcolumns) > 0:
      nmbrnormalization_dict = {textcolumns[0] : {'binsmean' : mean, \
                                                  'buckets_bkt2' : buckets, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bins_id' : bins_id, \
                                                  'textcolumns' : textcolumns, \
                                                  'trainmax' : trainmax, \
                                                  'trainmin' : trainmin, \
                                                  'suffix' : suffix, \
                                                  'origbuckets_bkt2' : origbuckets}}

    #drift stats for activation ratios
    nmbrnormalization_dict.update({'activation_ratios' : {}})

    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict['activation_ratios'].update({tc_ratio : tcratio})

    for nc in textcolumns:

      if nc != textcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)
       
    return mdf_train, mdf_test, column_dict_list
  
  def _process_bkt3(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'buckets' in params:
      buckets = params['buckets']
    else:
      buckets = [0,1,2]

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0
      
    trainmax = mdf_train[binscolumn].max()
    trainmin = mdf_train[binscolumn].min()
    
    #if buckets is a set instead of list that signals to convert percentages to values
    if type(buckets) == type({1,2}):
      buckets = sorted(list(buckets))
      
      if trainmax != trainmax:
        trainmax = 0
      if trainmin != trainmin:
        trainmin = 0
        
      buckets = [(trainmax - trainmin) * x + trainmin for x in buckets]

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
    bins_cuts.insert(0, -np.inf)
    bins_cuts.insert(len(bins_cuts), np.inf)
    
    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))

    infill_activation = len(bins_cuts)-1
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    mdf_train[binscolumn] = mdf_train[binscolumn].astype(float)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)

    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #replace missing data with infill_activation
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(infill_activation)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)
    
    #returned data type is conditional on the size of encoding space
    max_encoding = len(bins_cuts) - 1
    if max_encoding <= 255:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint8)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
    elif max_encoding <= 65535:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint16)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
    else:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint32)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(nmbrcolumns) > 0:
      nmbrnormalization_dict = {nmbrcolumns[0] : {'binsmean' : mean, \
                                                  'buckets' : buckets, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bins_id' : bins_id, \
                                                  'activations_list' : bins_id, \
                                                  'infill_activation' : infill_activation, \
                                                  'ordl_activations_dict' : ordl_activations_dict, \
                                                  'trainmax' : trainmax, \
                                                  'trainmin' : trainmin, \
                                                  'suffix' : suffix, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list
  
  def _process_bkt4(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'buckets' in params:
      buckets = params['buckets']
    else:
      buckets = [0,1,2]

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    binscolumn = column + '_' + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
    
    trainmax = mdf_train[binscolumn].max()
    trainmin = mdf_train[binscolumn].min()
    
    #if buckets is a set instead of list that signals to convert percentages to values
    if type(buckets) == type({1,2}):
      buckets = sorted(list(buckets))
      
      if trainmax != trainmax:
        trainmax = 0
      if trainmin != trainmin:
        trainmin = 0
        
      buckets = [(trainmax - trainmin) * x + trainmin for x in buckets]
    
    #set all values that fall outside of bounded buckets to nan for replacement with mean
    mdf_train.loc[mdf_train[binscolumn] <= buckets[0], (binscolumn)] = np.nan
    mdf_test.loc[mdf_test[binscolumn] <= buckets[0], (binscolumn)] = np.nan
    
    mdf_train.loc[mdf_train[binscolumn] > buckets[-1], (binscolumn)] = np.nan
    mdf_test.loc[mdf_test[binscolumn] > buckets[-1], (binscolumn)] = np.nan

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0
      
    #edge case, if mean does nto fall within buckets range, we'll apply infill to top bucket
    #this edge case specific to bkt4
    #this assumes buckets was passed with sorted values
    if mean < buckets[0] or mean > buckets[-1]:
      mean = buckets[-1]

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
#     bins_cuts.insert(0, -np.inf)
#     bins_cuts.insert(len(bins_cuts), np.inf)

    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))

    infill_activation = len(bins_cuts)-1
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    
    mdf_train[binscolumn] = mdf_train[binscolumn].astype(float)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)

    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #replace missing data with infill_activation
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(infill_activation)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)

    #returned data type is conditional on the size of encoding space
    max_encoding = len(bins_cuts) - 1
    if max_encoding <= 255:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint8)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
    elif max_encoding <= 65535:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint16)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
    else:
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.uint32)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    nmbrnormalization_dict = {}
    if len(nmbrcolumns) > 0:
      nmbrnormalization_dict = {nmbrcolumns[0] : {'binsmean' : mean, \
                                                  'buckets' : buckets, \
                                                  'bins_cuts' : bins_cuts, \
                                                  'bins_id' : bins_id, \
                                                  'activations_list' : bins_id, \
                                                  'infill_activation' : infill_activation, \
                                                  'ordl_activations_dict' : ordl_activations_dict, \
                                                  'trainmax' : trainmax, \
                                                  'trainmin' : trainmin, \
                                                  'suffix' : suffix, \
                                                  'defaultinfill_dict' : defaultinfill_dict,
                                                  'inplace' : inplace}}

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list

  def _process_DPnb(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPnb(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data with z-score normalization to mean 0 and sigma 1
    #adds data sampled from normal distribution with mean 0 and sigma 0.06 by default
    #where noise only injected to a subset of data based on flip_prob defaulting to 0.03
    #the noise properties may be customized with parameters 'mu', 'sigma', 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default

    #the 'protected_feature' parameter allows for specification of a protected categoric feature
    #in which the noise scaling will be adjustd based on scale of this feature's segments
    #corresponding to unique entries in the protected feature
    #protected features are prepared after other features to accomodate parallelization
    #the conversion for each segment is merely a matter of multiplying by the ratio of standard deviations
    #between the segment and the full feature
    #e.g. adjusted_normal_samples = normal_samples * stdev_segment / stdev_feature
    #this will average out to the feature stdev accross segments
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'test_mu' in params:
      test_mu = params['test_mu']
    else:
      test_mu = mu
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      #default also populated in __random_parameters_params_append
      sigma = 0.06
      
    if 'test_sigma' in params:
      test_sigma = params['test_sigma']
    else:
      #default also populated in __random_parameters_params_append
      test_sigma = 0.03

    if 'rescale_sigmas' in params:
      rescale_sigmas = params['rescale_sigmas']
    else:
      rescale_sigmas = False
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = flip_prob
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      noisedistribution = 'normal'
      
    if 'test_noisedistribution' in params:
      test_noisedistribution = params['test_noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      test_noisedistribution = noisedistribution

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False
      
    if 'protected_feature' in params:
      protected_feature = params['protected_feature']

      if protected_feature == column \
      or column in postprocess_dict['column_dict'] \
      and protected_feature == postprocess_dict['column_dict'][column]['origcolumn'] \
      or protected_feature not in mdf_train:
        protected_feature = False

    else:
      protected_feature = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
    
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
    
    #note that in the random_generator == False scenario 
    #__random_parameters_params_append will have set to np.random.PCG64
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    sigma_list = False
    if isinstance(sigma, list):
      sigma_list = sigma.copy()
      sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_sigma_list = False
    if isinstance(test_sigma, list):
      test_sigma_list = test_sigma.copy()
      test_sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      sigma_list = False
      test_sigma_list = False
      flip_prob_list = False
      test_flip_prob_list = False

    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    mu_dist = False
    if isinstance(mu, type(stats.expon(1))):
      mu_dist = mu
      mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(mu, sampling_resource_dict, nprandom_dict, 'train')

    test_mu_dist = False
    if isinstance(test_mu, type(stats.expon(1))):
      test_mu_dist = test_mu
      test_mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_mu, sampling_resource_dict, nprandom_dict, 'test')

    sigma_dist = False
    if isinstance(sigma, type(stats.expon(1))):
      sigma_dist = sigma
      sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(sigma, sampling_resource_dict, nprandom_dict, 'train')
      if sigma < 0:
        sigma = 0

    test_sigma_dist = False
    if isinstance(test_sigma, type(stats.expon(1))):
      test_sigma_dist = test_sigma
      test_sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_sigma, sampling_resource_dict, nprandom_dict, 'test')
      if test_sigma < 0:
        test_sigma = 0

    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      mu_dist = False
      test_mu_dist = False
      sigma_dist = False
      test_sigma_dist = False
      flip_prob_dist = False
      test_flip_prob_dist = False
        
    #________

    #the sigma defaults are based on z-score data with stdev=1, for unscaled data (like with DPne) can rescale noise sigmas to align
    #rescale_sigmas defaults to True for DPne
    train_stdev =  pd.to_numeric(mdf_train[column], errors='coerce').std()
    if rescale_sigmas is True:
      if train_stdev == train_stdev and train_stdev > 0:
        sigma *= train_stdev
        test_sigma *= train_stdev

    DPnm_column = column + '_' + suffix
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, DPnm_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    binomial_activation_count = 0
    test_binomial_activation_count = 0
    
    #___
    #the protected_feature option when elected populates 
    #a dicitonary of segment standard deviations as attribute_stdev_dict
    #which is used for scaling basis to both train and test data
    
    attribute_stdev_dict = {}
    attributes = []
    feature_stdev = pd.to_numeric(mdf_train[column], errors='coerce').std()
    if protected_feature is not False:

      #if we are scaling noise based on segments from a protected feature
      #we will first populated a dictionary mapping unique entries of protectted feature to ratio of entries

      attributes = list(mdf_train[protected_feature].unique())

      #we'll evaluate and scale noise for each segment in this for loop
      for attribute in attributes:

        #this is for noise scaling
        if attribute == attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature]==attribute, column], errors='coerce').std()

        elif attribute != attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature].isna(), column], errors='coerce').std()

        if attribute_stdev != attribute_stdev:
          #if a segment did not have any variation, will set noise profile to zero
          #could also set to 1, but that wouldn't align with segment with small stdev
          attribute_stdev = 0

        #now record that value with key of attribute
        attribute_stdev_dict.update({attribute : attribute_stdev})
        
    #___
    
    if trainnoise is True and flip_prob > 0 and sigma > 0:
      
      #___
      
      #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
      if flip_prob == 1:
        binomial_samples = np.ones(mdf_train.shape[0]).astype(int)
      else:
        sampling_id = 'binomial_train'
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_train.shape[0]
        binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0]))
      binomial_activation_count = np.sum(binomial_samples)
      
      #___
      
      #now derive our distribution sampled noise for injection
      sampling_id = 'distribution_train'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
      if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
        normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
        normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
        normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

      if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
        normal_samples = abs(normal_samples)
      elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
        normal_samples = (-1) * abs(normal_samples)
        
      #pass binomial samples to column in dataframe
      mdf_train[DPnm_column] = pd.DataFrame(binomial_samples, index=mdf_train.index)
      
      #now multiply activations by the normal samples
      #combining Bernoulli and distribution samples
      #this is equivalent to multiplying the Bernoulli and Distribution samples since Bernoulli had 0's and 1's
      mdf_train.loc[mdf_train[DPnm_column] == 1, DPnm_column] = normal_samples
      
      #___
          
      if protected_feature is not False:
        
        if feature_stdev == feature_stdev and feature_stdev > 0:

          for attribute in attributes:

            if attribute == attribute:
              #this applies the noise scaling
              #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
              mdf_train.loc[mdf_train[protected_feature] == attribute, DPnm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev
            elif attribute != attribute:
              mdf_train.loc[mdf_train[protected_feature].isna(), DPnm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev

      #___

      #now inject noise
      if inplace is True:
        #this inplace support is a little different from convention
        #due to DPnm_column already being used to hold support data
        #has the same effect
        #note the suffix overlap already checked above
        mdf_train[column] = mdf_train[DPnm_column] + pd.to_numeric(mdf_train[column], errors='coerce')
        del mdf_train[DPnm_column]
        mdf_train.rename(columns = {column : DPnm_column}, inplace = True)
        
      elif inplace is False:
        mdf_train[DPnm_column] = mdf_train[DPnm_column] + pd.to_numeric(mdf_train[column], errors='coerce')

    #else train data is just pass-through
    else:
      if inplace is True:
        mdf_train.rename(columns = {column : DPnm_column}, inplace = True)
      elif inplace is False:
        mdf_train[DPnm_column] = pd.to_numeric(mdf_train[column], errors='coerce').copy()
    
    if testnoise is True and test_flip_prob > 0 and test_sigma > 0:
      
      #___
    
      #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
      if test_flip_prob == 1:
        binomial_samples = np.ones(mdf_test.shape[0]).astype(int)
      else:
        sampling_id = 'binomial_test'
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        binomial_samples = nprandom.binomial(n=1, p=test_flip_prob, size=(mdf_test.shape[0]))
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
      test_binomial_activation_count = np.sum(binomial_samples)
      
      #___
    
      #now derive our sampled noise for injection
      sampling_id = 'distribution_test'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += test_binomial_activation_count
      if test_noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
        normal_samples = nprandom.normal(loc=test_mu, scale=test_sigma, size=(test_binomial_activation_count))
      elif test_noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
        normal_samples = nprandom.laplace(loc=test_mu, scale=test_sigma, size=(test_binomial_activation_count))
      elif test_noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
        normal_samples = nprandom.uniform(low=(test_mu-test_sigma), high=(test_sigma-test_mu), size=(test_binomial_activation_count))

      if test_noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
        normal_samples = abs(normal_samples)
      elif test_noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
        normal_samples = (-1) * abs(normal_samples)
      
      #pass binomial samples to column in dataframe
      mdf_test[DPnm_column] = pd.DataFrame(binomial_samples, index=mdf_test.index)
    
      #now multiply activations by the normal samples
      #this combines the Bernoulli and distribution samples to single set
      #this is equivalent to multiplying the Bernoulli and Distribution samples since Bernoulli had 0's and 1's
      mdf_test.loc[mdf_test[DPnm_column] == 1, DPnm_column] = normal_samples
      
      #___
      
      if protected_feature is not False:

        if feature_stdev == feature_stdev and feature_stdev > 0:

          for attribute in attributes:
            
            if attribute == attribute:
              #this applies the noise scaling
              #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
              mdf_test.loc[mdf_test[protected_feature] == attribute, DPnm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev
            elif attribute != attribute:
              mdf_test.loc[mdf_test[protected_feature].isna(), DPnm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev
            
          #for test attributes in protected feature that weren't found in train, no need to scale          
      
      #___

      #now inject noise
      if inplace is True:
        #this inplace support is a little different from convention
        #due to DPnm_column already being used to hold support data
        #has the same effect
        #note the suffix overlap already checked above
        mdf_test[column] = mdf_test[DPnm_column] + pd.to_numeric(mdf_test[column], errors='coerce')
        del mdf_test[DPnm_column]
        mdf_test.rename(columns = {column : DPnm_column}, inplace = True)
        
      elif inplace is False:
        mdf_test[DPnm_column] = mdf_test[DPnm_column] + pd.to_numeric(mdf_test[column], errors='coerce')
    
    #else test data is just pass-through
    else:
      if inplace is True:
        mdf_test.rename(columns = {column : DPnm_column}, inplace = True)
      elif inplace is False:
        mdf_test[DPnm_column] = pd.to_numeric(mdf_test[column], errors='coerce').copy()
    
    #create list of columns
    nmbrcolumns = [DPnm_column]
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    nmbrnormalization_dict = {DPnm_column : {'mu' : mu, \
                                             'sigma' : sigma, \
                                             'flip_prob' : flip_prob, \
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'suffix' : suffix, \
                                             'inplace' : inplace, \
                                             'noisedistribution' : noisedistribution, \
                                             'test_mu' : test_mu, \
                                             'test_sigma' : test_sigma, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'test_noisedistribution' : test_noisedistribution, \
                                             'mu_dist' : mu_dist, \
                                             'sigma_dist' : sigma_dist, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_mu_dist' : test_mu_dist, \
                                             'test_sigma_dist' : test_sigma_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'rescale_sigmas' : rescale_sigmas, \
                                             'train_stdev' : train_stdev, \
                                             'binomial_activation_count' : binomial_activation_count, \
                                             'test_binomial_activation_count' : test_binomial_activation_count, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'sigma_list' : sigma_list, \
                                             'test_sigma_list' : test_sigma_list, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'retain_basis' : retain_basis, \
                                             'protected_feature' : protected_feature, \
                                             'attribute_stdev_dict' : attribute_stdev_dict, \
                                             'attributes' : attributes, \
                                             'feature_stdev' : feature_stdev, \
                                            }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_DPmm(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPmm(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data min-max scaled within range 0-1
    #adds data sampled from normal distribution with mean 0 and sigma 0.03 by default
    #the noise properties may be customized with parameters 'mu', 'sigma'
    #also accepts parameter 'flip_prob' for ratio of data that will be adjusted (defaults to 1.)
    #noise is scaled based on the received points to keep within range 0-1
    #(e.g. for received data point 0.1, noise is scaled so as not to fall below -0.1)
    #gaussian noise source is also capped to maintain the range -0.5 to 0.5 (rare outlier points)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'test_mu' in params:
      test_mu = params['test_mu']
    else:
      test_mu = mu
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      #default also populated in __random_parameters_params_append
      sigma = 0.03
      
    if 'test_sigma' in params:
      test_sigma = params['test_sigma']
    else:
      #default also populated in __random_parameters_params_append
      test_sigma = 0.02
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = flip_prob
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      noisedistribution = 'normal'
      
    if 'test_noisedistribution' in params:
      test_noisedistribution = params['test_noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      test_noisedistribution = noisedistribution
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True
      
    if 'noise_scaling_bias_offset' in params:
      noise_scaling_bias_offset = params['noise_scaling_bias_offset']
    else:
      noise_scaling_bias_offset = True
      
    if 'protected_feature' in params:
      protected_feature = params['protected_feature']

      if protected_feature == column \
      or column in postprocess_dict['column_dict'] \
      and protected_feature == postprocess_dict['column_dict'][column]['origcolumn'] \
      or protected_feature not in mdf_train:
        protected_feature = False

    else:
      protected_feature = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
      
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    sigma_list = False
    if isinstance(sigma, list):
      sigma_list = sigma.copy()
      sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_sigma_list = False
    if isinstance(test_sigma, list):
      test_sigma_list = test_sigma.copy()
      test_sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      sigma_list = False
      test_sigma_list = False
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    mu_dist = False
    if isinstance(mu, type(stats.expon(1))):
      mu_dist = mu
      mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(mu, sampling_resource_dict, nprandom_dict, 'train')

    test_mu_dist = False
    if isinstance(test_mu, type(stats.expon(1))):
      test_mu_dist = test_mu
      test_mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_mu, sampling_resource_dict, nprandom_dict, 'test')

    sigma_dist = False
    if isinstance(sigma, type(stats.expon(1))):
      sigma_dist = sigma
      sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(sigma, sampling_resource_dict, nprandom_dict, 'train')
      if sigma < 0:
        sigma = 0

    test_sigma_dist = False
    if isinstance(test_sigma, type(stats.expon(1))):
      test_sigma_dist = test_sigma
      test_sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_sigma, sampling_resource_dict, nprandom_dict, 'test')
      if test_sigma < 0:
        test_sigma = 0

    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      mu_dist = False
      test_mu_dist = False
      sigma_dist = False
      test_sigma_dist = False
      flip_prob_dist = False
      test_flip_prob_dist = False
        
    #________
      
    DPmm_column = column + '_' + suffix
    DPmm_column_temp1 = column + '_' + suffix + '_tmp1'
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, [DPmm_column, DPmm_column_temp1], suffixoverlap_results, postprocess_dict['printstatus'])
    
    def debiasmmnoise(df, DPmm_column, DPmm_column_temp1, mu, sigma, noisedistribution):
      #1) derives noise and scales
      #2) measure mean, re sample noise with that mean as an offset
      #3) meansure the resulting mean, linear interpolate between to get final mean
      #4) the resulting mean returned for the final sampling
      #note that debiasmmnoise uses default numpy sampling without entropy seeds via np.random
      
      def noisescalingevaluationsupport(df, DPmm_column, DPmm_column_temp1, mu, sigma, noisedistribution):
      
        #first we'll derive our sampled noise for injection
        if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
          normal_samples = np.random.normal(loc=mu, scale=sigma, size=(df.shape[0]))
        elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
          normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(df.shape[0]))
        elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
          normal_samples = np.random.uniform(low=(mu-sigma), high=(sigma-mu), size=(df.shape[0]))

        if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
          normal_samples = abs(normal_samples)
        elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
          normal_samples = (-1) * abs(normal_samples)

        df[DPmm_column] = pd.DataFrame(normal_samples, index=df.index)

        #cap outliers
        df = \
        self.__autowhere(df, DPmm_column, df[DPmm_column] < -0.5, -0.5, specified='replacement')
        df = \
        self.__autowhere(df, DPmm_column, df[DPmm_column] > 0.5, 0.5, specified='replacement')

        #support column to signal sign of noise, 0 is neg, 1 is pos
        df = \
        self.__autowhere(df, DPmm_column_temp1, df[DPmm_column] >= 0., 1, specified='replacement')

        #now scale noise, with scaled noise to maintain range 0-1
        #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
        #this does not include the injection by addition to df[column], final injection follows in a seperate function
        #this is just to evaluate for scaled noise bias from df[column] distribution properties
        df = \
        self.__autowhere(df, 
                        DPmm_column, 
                        df[column] < 0.5, 
  #                       df[column] + \
                        (1 - df[DPmm_column_temp1]) * (df[DPmm_column] * df[column] / 0.5) + \
                        (df[DPmm_column_temp1]) * (df[DPmm_column]), 
                        specified='replacement')

        df = \
        self.__autowhere(df, 
                        DPmm_column, 
                        df[column] >= 0.5, 
  #                       df[column] + \
                        (1 - df[DPmm_column_temp1]) * (df[DPmm_column]) + \
                        (df[DPmm_column_temp1]) * (df[DPmm_column] * (1 - df[column]) / 0.5), \
                        specified='replacement')
        
        return df
      
      #first noise sampling and scaling applied
      df = noisescalingevaluationsupport(df, DPmm_column, DPmm_column_temp1, mu, sigma, noisedistribution)

      #evaluate mean of resulting scaled noise
      meanofscaledfrom_mu = df[DPmm_column].mean()
      mu_iter1 = mu - meanofscaledfrom_mu
      
      #remove support columns
      del df[DPmm_column]
      del df[DPmm_column_temp1]    
      
      #now resample and scale again, this time using adjusted mu
      df = noisescalingevaluationsupport(df, DPmm_column, DPmm_column_temp1, mu_iter1, sigma, noisedistribution)
      
      #evaluate mean of resulting scaled noise
      meanofscaledfrom_mu_iter1 = df[DPmm_column].mean()
      
      #remove support columns
      del df[DPmm_column]
      del df[DPmm_column_temp1]    
      
      #so we'll linearly interpolate for a final bias offset, 
      #want to solve for target_mu that produces a scaled mean of mu
      #this may not give an optimal but should be an improvement
      #this is derived from:
      #(target_mu - mu) / (mu_iter1 - mu) = (mu - meanofscaledfrom_mu) / (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu)
      #where mu on right side of above is based on trying to derive a target_mu that produces a scaled noise mean of mu

      if (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu) * (mu_iter1 - mu) + mu != 0:
        target_mu = \
        (mu - meanofscaledfrom_mu) / (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu) \
        * (mu_iter1 - mu) + mu
      else:
        target_mu = mu
      
      # #validation printouts
      # df = noisescalingevaluationsupport(df, DPmm_column, DPmm_column_temp1, target_mu)
      # meanofscaledfrom_target_mu = df[DPmm_column].mean()
      # del df[DPmm_column]
      # del df[DPmm_column_temp1]   
      # print('mu = ', mu)
      # print('meanofscaledfrom_mu = ', meanofscaledfrom_mu)
      # print()
      # print('mu_iter1 = ', mu_iter1)
      # print('meanofscaledfrom_mu_iter1 = ', meanofscaledfrom_mu_iter1)
      # print()
      # print('target_mu = ', target_mu)
      # print('meanofscaledfrom_target_mu = ', meanofscaledfrom_target_mu)
      
      return target_mu
    
    def _injectmmnoise(df, column, DPmm_column, DPmm_column_temp1, mu, sigma, flip_prob, \
                       noisedistribution, sampling_resource_dict, nprandom_dict, traintest, \
                       protected_feature, attributes, attribute_stdev_dict, feature_stdev):
      #support function for noise injection
      
      #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
      if flip_prob == 1:
        binomial_samples = np.ones(df.shape[0]).astype(int)
      else:
        sampling_id = 'binomial_' + traintest
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
        binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(df.shape[0]))
      binomial_activation_count = np.sum(binomial_samples)
      
      #now derive our sampled noise for injection
      sampling_id = 'distribution_' + traintest
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
      if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
        normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
        normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
        normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

      if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
        normal_samples = abs(normal_samples)
      elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
        normal_samples = (-1) * abs(normal_samples)
      
      #pass binomial samples to column in dataframe
      df[DPmm_column] = pd.DataFrame(binomial_samples, index=df.index)
      
      #now multiply activations by the normal samples
      df.loc[df[DPmm_column] == 1, DPmm_column] = normal_samples
      
      #___
          
      if protected_feature is not False:
        
        if feature_stdev == feature_stdev and feature_stdev > 0:

          for attribute in attributes:
            
            if attribute == attribute:
              #this applies the noise scaling
              #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
              df.loc[df[protected_feature] == attribute, DPmm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev
            elif attribute != attribute:
              df.loc[df[protected_feature].isna(), DPmm_column] *= \
              attribute_stdev_dict[attribute] / feature_stdev

      #___

      #cap outliers
      df = \
      self.__autowhere(df, DPmm_column, df[DPmm_column] < -0.5, -0.5, specified='replacement')
      df = \
      self.__autowhere(df, DPmm_column, df[DPmm_column] > 0.5, 0.5, specified='replacement')

      #support column to signal sign of noise, 0 is neg, 1 is pos
      df = \
      self.__autowhere(df, DPmm_column_temp1, df[DPmm_column] >= 0., 1, specified='replacement')

      #now inject noise, with scaled noise to maintain range 0-1
      #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
      df = \
      self.__autowhere(df, 
                      DPmm_column, 
                      df[column] < 0.5, 
                      df[column] + \
                      (1 - df[DPmm_column_temp1]) * (df[DPmm_column] * df[column] / 0.5) + \
                      (df[DPmm_column_temp1]) * (df[DPmm_column]), 
                      specified='replacement')

      df = \
      self.__autowhere(df, 
                      DPmm_column, 
                      df[column] >= 0.5, 
                      df[column] + \
                      (1 - df[DPmm_column_temp1]) * (df[DPmm_column]) + \
                      (df[DPmm_column_temp1]) * (df[DPmm_column] * (1 - df[column]) / 0.5), \
                      specified='replacement')

      #adjacent cell infill (this is included as a precaution shouldn't have any effect since upstream normalization)
      df[DPmm_column] = df[DPmm_column].fillna(method='ffill')
      df[DPmm_column] = df[DPmm_column].fillna(method='bfill')

      #remove support column
      del df[DPmm_column_temp1]
      
      return df, binomial_activation_count, sampling_resource_dict
    
    mu_orig = mu
    test_mu_orig = test_mu
    if noise_scaling_bias_offset is True:
      
      if flip_prob > 0 and sigma > 0:
        mu = debiasmmnoise(mdf_train, DPmm_column, DPmm_column_temp1, mu, sigma, noisedistribution)
      
      #if test_mu was specified differently than mu, then a test set specific debiasing is performed
      if test_mu == mu_orig:
        test_mu = mu
      else:
        if test_flip_prob > 0 and test_sigma > 0:
          #the single row case is when no df_test was passed to automunge(.)
          #even though this is for test noise parameters, we're intentionally basing bias offset on train data properties
          if mdf_test.shape[0] > 1:
            test_mu = debiasmmnoise(mdf_train, DPmm_column, DPmm_column_temp1, test_mu, test_sigma, test_noisedistribution)
    
    #___
    #the protected_feature option when elected populates 
    #a dicitonary of segment standard deviations as attribute_stdev_dict
    #which is used for scaling basis to both train and test data
    
    attribute_stdev_dict = {}
    attributes = set()
    feature_stdev = pd.to_numeric(mdf_train[column], errors='coerce').std()

    if protected_feature is not False:

      #if we are scaling noise based on segments from a protected feature
      #we will first populated a dictionary mapping unique entries of protectted feature to ratio of entries

      attributes = set(mdf_train[protected_feature].unique())

      #we'll evaluate and scale noise for each segment in this for loop
      for attribute in attributes:

        #this is for noise scaling
        if attribute == attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature]==attribute, column], errors='coerce').std()

        elif attribute != attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature].isna(), column], errors='coerce').std()      

        if attribute_stdev != attribute_stdev:
          #if a segment did not have any variation, will set noise profile to zero
          #could also set to 1, but that wouldn't align with segment with small stdev
          attribute_stdev = 0

        #now record that value with key of attribute
        attribute_stdev_dict.update({attribute : attribute_stdev})
        
    #___
    
    binomial_activation_count = 0
    test_binomial_activation_count = 0
    
    if trainnoise is True and flip_prob > 0 and sigma > 0:
      mdf_train, binomial_activation_count, sampling_resource_dict = \
      _injectmmnoise(mdf_train, column, DPmm_column, DPmm_column_temp1, mu, sigma, flip_prob, \
                     noisedistribution, sampling_resource_dict, nprandom_dict, 'train', \
                     protected_feature, attributes, attribute_stdev_dict, feature_stdev)
    else:
      #else train data is pass-through
      mdf_train[DPmm_column] = mdf_train[column].copy()
    
    if testnoise is True and test_flip_prob > 0 and test_sigma > 0:
      mdf_test, test_binomial_activation_count, sampling_resource_dict = \
      _injectmmnoise(mdf_test, column, DPmm_column, DPmm_column_temp1, test_mu, test_sigma, test_flip_prob, \
                     test_noisedistribution, sampling_resource_dict, nprandom_dict, 'test', \
                     protected_feature, attributes, attribute_stdev_dict, feature_stdev)
    else:
      #else test data is pass-through
      mdf_test[DPmm_column] = mdf_test[column].copy()

    #create list of columns
    nmbrcolumns = [DPmm_column]
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    nmbrnormalization_dict = {DPmm_column : {'mu' : mu, \
                                             'mu_orig' : mu_orig, \
                                             'sigma' : sigma, \
                                             'flip_prob' : flip_prob, \
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'noise_scaling_bias_offset' : noise_scaling_bias_offset, \
                                             'suffix' : suffix, \
                                             'noisedistribution' : noisedistribution, \
                                             'test_mu' : test_mu, \
                                             'test_mu_orig' : test_mu_orig, \
                                             'test_sigma' : test_sigma, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'test_noisedistribution' : test_noisedistribution, \
                                             'mu_dist' : mu_dist, \
                                             'sigma_dist' : sigma_dist, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_mu_dist' : test_mu_dist, \
                                             'test_sigma_dist' : test_sigma_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'binomial_activation_count' : binomial_activation_count, \
                                             'test_binomial_activation_count' : test_binomial_activation_count, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'sigma_list' : sigma_list, \
                                             'test_sigma_list' : test_sigma_list, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'retain_basis' : retain_basis, \
                                             'protected_feature' : protected_feature, \
                                             'attribute_stdev_dict' : attribute_stdev_dict, \
                                             'attributes' : attributes, \
                                             'feature_stdev' : feature_stdev, \
                                            }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_DPrt(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    """
    #process_DPrt 
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #followed by a noise injection similar to DPmm based based on this set's retn range
    
    #replaces missing or improperly formatted data with mean of remaining values
    #(prior to noise injection)
    
    #returns same dataframes with new column of name column + '_DPrt'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """

    suffixoverlap_results = {}
    
    #accepts divisor parameters of 'minmax' or 'std', eg divisor for normalization equation
    #note that standard deviation doesn't have same properties for sign retention when all values > or < 0
    if 'divisor' in params:
      divisor = params['divisor']
    else:
      divisor = 'minmax'
    
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
      
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #stdev_cap can be passed as integer or float for setting cap based on number of standard deviations from mean
    #only performed with cap=False and floor=False, default of stdev_cap=False deactivates
    if 'stdev_cap' in params:
      stdev_cap = params['stdev_cap']
    else:
      stdev_cap = False
      
    #here are differential privacy parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'test_mu' in params:
      test_mu = params['test_mu']
    else:
      test_mu = mu
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      #default also populated in __random_parameters_params_append
      sigma = 0.03
      
    if 'test_sigma' in params:
      test_sigma = params['test_sigma']
    else:
      #default also populated in __random_parameters_params_append
      test_sigma = 0.02
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = flip_prob
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      noisedistribution = 'normal'
      
    if 'test_noisedistribution' in params:
      test_noisedistribution = params['test_noisedistribution']
    else:
      #can pass as: 
      #'normal', 'abs_normal', 'negabs_normal', 'laplace', 'abs_laplace', 'negabs_laplace', 
      #'uniform', 'abs_uniform', 'netabs_uniform'
      test_noisedistribution = noisedistribution
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True
      
    if 'noise_scaling_bias_offset' in params:
      noise_scaling_bias_offset = params['noise_scaling_bias_offset']
    else:
      noise_scaling_bias_offset = True
      
    if 'protected_feature' in params:
      protected_feature = params['protected_feature']

      if protected_feature == column \
      or column in postprocess_dict['column_dict'] \
      and protected_feature == postprocess_dict['column_dict'][column]['origcolumn'] \
      or protected_feature not in mdf_train:
        protected_feature = False

    else:
      protected_feature = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
      
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    sigma_list = False
    if isinstance(sigma, list):
      sigma_list = sigma.copy()
      sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_sigma_list = False
    if isinstance(test_sigma, list):
      test_sigma_list = test_sigma.copy()
      test_sigma, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      sigma_list = False
      test_sigma_list = False
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    mu_dist = False
    if isinstance(mu, type(stats.expon(1))):
      mu_dist = mu
      mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(mu, sampling_resource_dict, nprandom_dict, 'train')

    test_mu_dist = False
    if isinstance(test_mu, type(stats.expon(1))):
      test_mu_dist = test_mu
      test_mu, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_mu, sampling_resource_dict, nprandom_dict, 'test')

    sigma_dist = False
    if isinstance(sigma, type(stats.expon(1))):
      sigma_dist = sigma
      sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(sigma, sampling_resource_dict, nprandom_dict, 'train')
      if sigma < 0:
        sigma = 0

    test_sigma_dist = False
    if isinstance(test_sigma, type(stats.expon(1))):
      test_sigma_dist = test_sigma
      test_sigma, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_sigma, sampling_resource_dict, nprandom_dict, 'test')
      if test_sigma < 0:
        test_sigma = 0

    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      mu_dist = False
      test_mu_dist = False
      sigma_dist = False
      test_sigma_dist = False
      flip_prob_dist = False
      test_flip_prob_dist = False
        
    #________
      
    DPrt_column = column + '_' + suffix
    DPrt_column_temp1 = column + '_' + suffix + '_tmp1'
    DPrt_column_temp2 = column + '_' + suffix + '_tmp2'
    
    newcolumns = [DPrt_column, DPrt_column_temp1, DPrt_column_temp2]
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    #copy source column into new column
    mdf_train[DPrt_column] = mdf_train[column].copy()
    mdf_test[DPrt_column] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[DPrt_column] = pd.to_numeric(mdf_train[DPrt_column], errors='coerce')
    mdf_test[DPrt_column] = pd.to_numeric(mdf_test[DPrt_column], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[DPrt_column].std()
    
    # mad = mdf_train[DPrt_column].mad()
    mad = (mdf_train[DPrt_column] - mdf_train[DPrt_column].median(axis=0)).abs().median()
    
    #get maximum value of training column
    maximum = mdf_train[DPrt_column].max()
    
    #get minimum value of training column
    minimum = mdf_train[DPrt_column].min()
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
      
    if std != std or std == 0:
      std = 1
      
    if mad != mad or mad == 0:
      mad = 1
      
    #if cap/floor set based on number of standard deviations, measure statistics and set variables cap and/or floor
    if stdev_cap is not False and cap is False and floor is False:
      
      pre_capfloor_mean = mdf_train[DPrt_column].mean()
      pre_capfloor_stdev = std
      
      if pre_capfloor_mean != pre_capfloor_mean:
        pre_capfloor_mean = 0
      if pre_capfloor_stdev != pre_capfloor_stdev:
        pre_capfloor_stdev = 1

      cap = pre_capfloor_mean + stdev_cap * pre_capfloor_stdev
      floor = pre_capfloor_mean - stdev_cap * pre_capfloor_stdev
      
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[DPrt_column] > cap, (DPrt_column)] \
      = cap
      
      mdf_test.loc[mdf_test[DPrt_column] > cap, (DPrt_column)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[DPrt_column] < floor, (DPrt_column)] \
      = floor
      
      mdf_test.loc[mdf_test[DPrt_column] < floor, (DPrt_column)] \
      = floor
      
    #get mean of training data
    mean = mdf_train[DPrt_column].mean()
    if mean != mean:
      mean = 0
    
    #apply defaultinfill based on processdict entry
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, DPrt_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, DPrt_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)

    #edge case (only neccesary so scalingapproach is assigned)
    if maximum != maximum:
      maximum = 0
    if minimum != minimum:
      minimum = 0
      
    #divisor
    if divisor not in {'minmax', 'std', 'mad'}:
      # if postprocess_dict['printstatus'] != 'silent':
      self.__autoprint('warning', "Error: retn transform parameter 'divisor' only accepts entries of 'minmax' 'mad' or 'std'")
      self.__autoprint('warning', '')
    if divisor == 'minmax':
      divisor = maxminusmin
    elif divisor == 'mad':
      divisor = mad
    else:
      divisor = std
      
    if divisor == 0 or divisor != divisor:
      divisor = 1
    
    #driftreport metric scalingapproach returned as 'retn' or 'mnmx' or 'mxmn'
    #where mnmx is for cases where all values in train set are positive
    #mxmn is for cases where all values in train set are negative
    
    if maximum >= 0 and minimum <= 0:
      scalingapproach = 'retn'
    elif maximum >= 0 and minimum >= 0:
      scalingapproach = 'mnmx'
    elif maximum <= 0 and minimum <= 0:
      scalingapproach = 'mxmn'
    
    #initially we'll scale comparable to min-max, and then after noise convert to retain scaling
    #note that any multiplier or offset will be applied after conversion to retain
      
    #scale to min-max
    #perform min-max scaling to train and test sets using values from train
    mdf_train[DPrt_column] = (mdf_train[DPrt_column] - minimum) / \
                                  (maxminusmin)

    mdf_test[DPrt_column] = (mdf_test[DPrt_column] - minimum) / \
                                 (maxminusmin)
      
    #now apply noise injection
    
    def debiasrtnoise(df, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, mu, sigma, noisedistribution):
      #1) derives noise and scales
      #2) measure mean, re sample noise with that mean as an offset
      #3) meansure the resulting mean, linear interpolate between to get final mean
      #4) the resulting mean returned for the final sampling
      #note that debiasmmnoise uses default numpy sampling without entropy seeds via np.random
      
      def noisescalingevaluationsupport(df, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, mu, sigma, noisedistribution):
      
        #first we'll derive our sampled noise for injection
        if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
          normal_samples = np.random.normal(loc=mu, scale=sigma, size=(df.shape[0]))
        elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
          normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(df.shape[0]))
        elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
          normal_samples = np.random.uniform(low=(mu-sigma), high=(sigma-mu), size=(df.shape[0]))

        if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
          normal_samples = abs(normal_samples)
        elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
          normal_samples = (-1) * abs(normal_samples)

        df[DPrt_column_temp2] = pd.DataFrame(normal_samples, index=df.index)

        #cap outliers
        df = \
        self.__autowhere(df, DPrt_column_temp2, df[DPrt_column_temp2] < -0.5, -0.5, specified='replacement')
        df = \
        self.__autowhere(df, DPrt_column_temp2, df[DPrt_column_temp2] > 0.5, 0.5, specified='replacement')

        #support column to signal sign of noise, 0 is neg, 1 is pos
        df = \
        self.__autowhere(df, DPrt_column_temp1, df[DPrt_column_temp2] >= 0., 1, specified='replacement')

        #now scale noise, with scaled noise to maintain range 0-1
        #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
        #this does not include the injection by addition to df[column], final injection follows in a seperate function
        #this is just to evaluate for scaled noise bias from df[column] distribution properties
        df = \
        self.__autowhere(df, 
                        DPrt_column_temp2, 
                        df[DPrt_column] < 0.5, 
#                         df[DPrt_column] + \
                        (1 - df[DPrt_column_temp1]) * (df[DPrt_column_temp2] * df[DPrt_column] / 0.5) + \
                        (df[DPrt_column_temp1]) * (df[DPrt_column_temp2]), \
                        specified='replacement')

        df = \
        self.__autowhere(df, 
                        DPrt_column_temp2, 
                        df[DPrt_column] >= 0.5, 
#                         df[DPrt_column] + \
                        (1 - df[DPrt_column_temp1]) * (df[DPrt_column_temp2]) + \
                        (df[DPrt_column_temp1]) * (df[DPrt_column_temp2] * (1 - df[DPrt_column]) / 0.5), \
                        specified='replacement')
        
        return df
      
      #first noise sampling and scaling applied
      df = noisescalingevaluationsupport(df, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, mu, sigma, noisedistribution)

      #evaluate mean of resulting scaled noise
      meanofscaledfrom_mu = df[DPrt_column_temp2].mean()
      mu_iter1 = mu - meanofscaledfrom_mu
      
      #remove support columns
      del df[DPrt_column_temp1]
      del df[DPrt_column_temp2]    
      
      #now resample and scale again, this time using adjusted mu
      df = noisescalingevaluationsupport(df, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, mu_iter1, sigma, noisedistribution)
      
      #evaluate mean of resulting scaled noise
      meanofscaledfrom_mu_iter1 = df[DPrt_column_temp2].mean()
      
      #remove support columns
      del df[DPrt_column_temp1]
      del df[DPrt_column_temp2]    
      
      #so we'll linearly interpolate for a final bias offset, 
      #want to solve for target_mu that produces a scaled mean of mu
      #this may not give an optimal but should be an improvement
      #this is derived from:
      #(target_mu - mu) / (mu_iter1 - mu) = (mu - meanofscaledfrom_mu) / (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu)
      #where mu on right side of above is based on trying to derive a target_mu that produces a scaled noise mean of mu

      if (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu) * (mu_iter1 - mu) + mu != 0:
        target_mu = \
        (mu - meanofscaledfrom_mu) / (meanofscaledfrom_mu_iter1 - meanofscaledfrom_mu) \
        * (mu_iter1 - mu) + mu
      else:
        target_mu = mu
      
      # #validation printouts
      # df = noisescalingevaluationsupport(df, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, target_mu)
      # meanofscaledfrom_target_mu = df[DPrt_column_temp2].mean()
      # del df[DPrt_column_temp1]
      # del df[DPrt_column_temp2] 
      # print('mu = ', mu)
      # print('meanofscaledfrom_mu = ', meanofscaledfrom_mu)
      # print()
      # print('mu_iter1 = ', mu_iter1)
      # print('meanofscaledfrom_mu_iter1 = ', meanofscaledfrom_mu_iter1)
      # print()
      # print('target_mu = ', target_mu)
      # print('meanofscaledfrom_target_mu = ', meanofscaledfrom_target_mu)
      
      return target_mu
    
    def _injectrtnoise(df, column, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, \
                       mu, sigma, flip_prob, noisedistribution, sampling_resource_dict, \
                       nprandom_dict, traintest, protected_feature, attributes, attribute_stdev_dict, feature_stdev):
      #support function for DPrt noise injection
      
      #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
      if flip_prob == 1:
        binomial_samples = np.ones(df.shape[0]).astype(int)
      else:
        sampling_id = 'binomial_' + traintest
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
        binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(df.shape[0]))
      binomial_activation_count = np.sum(binomial_samples)

      #now derive our sampled noise for injection
      sampling_id = 'distribution_' + traintest
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
      if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
        normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
        normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
      elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
        normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

      if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
        normal_samples = abs(normal_samples)
      elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
        normal_samples = (-1) * abs(normal_samples)

      #pass binomial samples to column in dataframe
      df[DPrt_column_temp2] = pd.DataFrame(binomial_samples, index=df.index)
      
      #now multiply activations by the normal samples
      df.loc[df[DPrt_column_temp2] == 1, DPrt_column_temp2] = normal_samples
      
      #___
      
      if protected_feature is not False:

        if feature_stdev == feature_stdev and feature_stdev > 0:

          for attribute in attributes:
            
            if attribute == attribute:
              #this applies the noise scaling
              #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
              df.loc[df[protected_feature] == attribute, DPrt_column_temp2] *= \
              attribute_stdev_dict[attribute] / feature_stdev
            elif attribute != attribute:
              df.loc[df[protected_feature].isna(), DPrt_column_temp2] *= \
              attribute_stdev_dict[attribute] / feature_stdev
      
      #___

      #cap outliers
      df = \
      self.__autowhere(df, DPrt_column_temp2, df[DPrt_column_temp2] < -0.5, -0.5, specified='replacement')
      df = \
      self.__autowhere(df, DPrt_column_temp2, df[DPrt_column_temp2] > 0.5, 0.5, specified='replacement')

      #support column to signal sign of noise, 0 is neg, 1 is pos
      df = \
      self.__autowhere(df, DPrt_column_temp1, df[DPrt_column_temp2] >= 0., 1, specified='replacement')

      #now inject noise, with scaled noise to maintain range 0-1
      #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
      df = \
      self.__autowhere(df, 
                      DPrt_column, 
                      df[DPrt_column] < 0.5, 
                      df[DPrt_column] + \
                      (1 - df[DPrt_column_temp1]) * (df[DPrt_column_temp2] * df[DPrt_column] / 0.5) + \
                      (df[DPrt_column_temp1]) * (df[DPrt_column_temp2]), \
                      specified='replacement')

      df = \
      self.__autowhere(df, 
                      DPrt_column, 
                      df[DPrt_column] >= 0.5, 
                      df[DPrt_column] + \
                      (1 - df[DPrt_column_temp1]) * (df[DPrt_column_temp2]) + \
                      (df[DPrt_column_temp1]) * (df[DPrt_column_temp2] * (1 - df[DPrt_column]) / 0.5), \
                      specified='replacement')

      #remove support columns
      del df[DPrt_column_temp1]
      del df[DPrt_column_temp2]
        
      return df, binomial_activation_count, sampling_resource_dict
    
    mu_orig = mu
    test_mu_orig = test_mu
    if noise_scaling_bias_offset is True:
      
      if flip_prob > 0 and sigma > 0:
        mu = debiasrtnoise(mdf_train, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, mu, sigma, noisedistribution)
      
      #if test_mu was specified differently than mu, then a test set specific debiasing is performed
      if test_mu == mu_orig:
        test_mu = mu
      else:
        if test_flip_prob > 0 and test_sigma > 0:
          #the single row case is when no df_test was passed to automunge(.)
          #even though this is for test noise parameters, we're intentionally basing bias offset on train data properties
          if mdf_test.shape[0] > 1:
            test_mu = debiasrtnoise(mdf_train, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, test_mu, test_sigma, test_noisedistribution)

    binomial_activation_count = 0
    test_binomial_activation_count = 0
    
    #___
    #the protected_feature option when elected populates 
    #a dicitonary of segment standard deviations as attribute_stdev_dict
    #which is used for scaling basis to both train and test data
    
    attribute_stdev_dict = {}
    attributes = set()
    feature_stdev = pd.to_numeric(mdf_train[column], errors='coerce').std()
    
    if protected_feature is not False:

      #if we are scaling noise based on segments from a protected feature
      #we will first populated a dictionary mapping unique entries of protectted feature to ratio of entries

      attributes = set(mdf_train[protected_feature].unique())

      #we'll evaluate and scale noise for each segment in this for loop
      for attribute in attributes:

        #this is for noise scaling
        if attribute == attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature]==attribute, column], errors='coerce').std()

        elif attribute != attribute:
          attribute_stdev = \
          pd.to_numeric(mdf_train.loc[mdf_train[protected_feature].isna(), column], errors='coerce').std()
        
        if attribute_stdev != attribute_stdev:
          #if a segment did not have any variation, will set noise profile to zero
          #could also set to 1, but that wouldn't align with segment with small stdev
          attribute_stdev = 0

        #now record that value with key of attribute
        attribute_stdev_dict.update({attribute : attribute_stdev})
        
    #___
            
    if trainnoise is True and flip_prob > 0 and sigma > 0:
      mdf_train, binomial_activation_count, sampling_resource_dict = \
      _injectrtnoise(mdf_train, column, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, \
                     mu, sigma, flip_prob, noisedistribution, sampling_resource_dict, \
                     nprandom_dict, 'train', protected_feature, attributes, attribute_stdev_dict, feature_stdev)
    
    #for test data is just pass-through unless testnoise or traindata is activated
    if testnoise is True and test_flip_prob > 0 and test_sigma > 0:
      mdf_test, test_binomial_activation_count, sampling_resource_dict = \
      _injectrtnoise(mdf_test, column, DPrt_column, DPrt_column_temp1, DPrt_column_temp2, \
                     test_mu, test_sigma, test_flip_prob, test_noisedistribution, sampling_resource_dict, \
                     nprandom_dict, 'test', protected_feature, attributes, attribute_stdev_dict, feature_stdev)
      
    #now that we've injected noise, will convert the minmax representation to 'retain' normalization
    if scalingapproach == 'retn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] + (minimum / divisor) ) * multiplier + offset
      mdf_test[DPrt_column] = (mdf_test[DPrt_column] + (minimum / divisor) ) * multiplier + offset
    elif scalingapproach == 'mnmx':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column]) * multiplier + offset
      mdf_test[DPrt_column] = (mdf_test[DPrt_column]) * multiplier + offset
    elif scalingapproach == 'mxmn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] - 1) * multiplier + offset
      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - 1) * multiplier + offset
    
    #create list of columns
    nmbrcolumns = [DPrt_column]
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)
    
    nmbrnormalization_dict = {DPrt_column : {'mu' : mu, \
                                             'mu_orig' : mu_orig, \
                                             'sigma' : sigma, \
                                             'flip_prob' : flip_prob, \
                                             'noisedistribution' : noisedistribution, \
                                             'minimum' : minimum, \
                                             'maximum' : maximum, \
                                             'mean' : mean, \
                                             'std' : std, \
                                             'mad' : mad, \
                                             'scalingapproach' : scalingapproach, \
                                             'offset' : offset, \
                                             'multiplier': multiplier, \
                                             'cap' : cap, \
                                             'floor' : floor, \
                                             'divisor' : divisor, \
                                             'suffix' : suffix, \
                                             'defaultinfill_dict' : defaultinfill_dict,
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'noise_scaling_bias_offset' : noise_scaling_bias_offset, \
                                             'test_mu' : test_mu, \
                                             'test_mu_orig' : test_mu_orig, \
                                             'test_sigma' : test_sigma, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'test_noisedistribution' : test_noisedistribution, \
                                             'mu_dist' : mu_dist, \
                                             'sigma_dist' : sigma_dist, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_mu_dist' : test_mu_dist, \
                                             'test_sigma_dist' : test_sigma_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'binomial_activation_count' : binomial_activation_count, \
                                             'test_binomial_activation_count' : test_binomial_activation_count, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'sigma_list' : sigma_list, \
                                             'test_sigma_list' : test_sigma_list, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'retain_basis' : retain_basis, \
                                             'protected_feature' : protected_feature, \
                                             'attribute_stdev_dict' : attribute_stdev_dict, \
                                             'attributes' : attributes, \
                                             'feature_stdev' : feature_stdev, \
                                            }}
    
    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_DPbn(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPbn(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is bnry encoded data (i.e. boolean integers in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #unless testnoise is activated which applies test noise injection in both automunge and postmunge
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = 0.01
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
      
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      flip_prob_dist = False
      test_flip_prob_dist = False
      
    #___
      
    DPbn_column = column + '_' + suffix
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, DPbn_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
    #now inject noise
    if trainnoise is True and flip_prob > 0:
      #first we'll derive our sampled noise for injection
      sampling_id = 'binomial_train'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += mdf_train.shape[0]
      mdf_train[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0])), index=mdf_train.index)
      #now inject
      mdf_train[DPbn_column] = abs(mdf_train[column] - mdf_train[DPbn_column])
    elif trainnoise is False:
      mdf_train[DPbn_column] = mdf_train[column].copy()
    
    #for test data is just pass-through unless testnoise or traindata is activated
    if testnoise is False:
      mdf_test[DPbn_column] = mdf_test[column].copy()
    elif testnoise is True and test_flip_prob > 0:
      #first we'll derive our sampled noise for injection
      sampling_id = 'binomial_test'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
      mdf_test[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=test_flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)

      #now inject noise
      mdf_test[DPbn_column] = abs(mdf_test[column] - mdf_test[DPbn_column])
      
    #change data types to 8-bit (1 byte) integers for memory savings
    mdf_train[DPbn_column] = mdf_train[DPbn_column].astype(np.int8)
    mdf_test[DPbn_column] = mdf_test[DPbn_column].astype(np.int8)
    
    #create list of columns
    nmbrcolumns = [DPbn_column]
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    nmbrnormalization_dict = {DPbn_column : {'flip_prob' : flip_prob, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'suffix' : suffix, \
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'retain_basis' : retain_basis, \
                                            }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_DPod(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPod(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is ordinal encoded data (i.e. categoric by integer in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #when flip activated selects from the set of encodings per level random draw
    #(including potenitally the current encoding for no flip)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #unless testnoise is activated which applies test noise injection in both automunge and postmunge
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = 0.01
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    if 'weighted' in params:
      weighted = params['weighted']
    else:
      weighted = True
      
    if 'test_weighted' in params:
      test_weighted = params['test_weighted']
    else:
      test_weighted = weighted

    if 'upstream_hsh2' in params:
      upstream_hsh2 = params['upstream_hsh2']
    else:
      upstream_hsh2 = False
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
      
    if 'protected_feature' in params:
      protected_feature = params['protected_feature']

      if protected_feature == column \
      or column in postprocess_dict['column_dict'] \
      and protected_feature == postprocess_dict['column_dict'][column]['origcolumn'] \
      or protected_feature not in mdf_train:
        protected_feature = False

    else:
      protected_feature = False
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'passthrough' in params:
      passthrough = params['passthrough']
    else:
      passthrough = False
      
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma

    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')

    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      flip_prob_dist = False
      test_flip_prob_dist = False
      
    #___
      
    DPod_column = column + '_' + suffix
    DPod_tempcolumn1 = 1
    
    newcolumns = [DPod_tempcolumn1]
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, DPod_column, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[DPod_column] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, DPod_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      mdf_train.rename(columns = {column : DPod_column}, inplace = True)
      mdf_test.rename(columns = {column : DPod_column}, inplace = True)
      
    #___
    
    #we'll want to know the set of activations present in column, for automunge this is unique values
    ord_encodings = mdf_train[DPod_column].unique()

    if upstream_hsh2 is True:
      if column in postprocess_dict['column_dict'] \
      and column in postprocess_dict['column_dict'][column]['normalization_dict'] \
      and 'vocab_size' in postprocess_dict['column_dict'][column]['normalization_dict'][column]:
        vocab_size = postprocess_dict['column_dict'][column]['normalization_dict'][column]['vocab_size']
        ord_encodings = list(range(vocab_size))

    weights = []
    if weighted is True:
      for entry in ord_encodings:
        if entry == entry:
          weight = (mdf_train[mdf_train[DPod_column] == entry].shape[0]) / mdf_train.shape[0]
        elif entry != entry:
          weight = (mdf_train[mdf_train[DPod_column].isna()].shape[0]) / mdf_train.shape[0]
        weights.append(weight)
        
    #___
    #the protected_feature option when elected populates 
    #a dictionary of segment weightings as attribute_weightings_dict
    #which is used for scaling basis to both train and test data
    
    attribute_weightings_dict = {}
    attributes = []
    if protected_feature is not False and weighted is True:

      #if we are scaling noise based on segments from a protected feature
      #we will first populated a dictionary mapping unique entries of protectted feature to ratio of entries

      attributes = list(mdf_train[protected_feature].unique())
      
      for attribute in attributes:
        attribute_weights = []
        for entry in ord_encodings:

          if attribute == attribute and entry == entry:
            attribute_weight = (mdf_train.loc[(mdf_train[protected_feature] == attribute) & (mdf_train[DPod_column] == entry)].shape[0]) \
            / mdf_train.loc[(mdf_train[protected_feature] == attribute)].shape[0]
          elif attribute != attribute and entry == entry:
            attribute_weight = (mdf_train.loc[(mdf_train[protected_feature].isna()) & (mdf_train[DPod_column] == entry)].shape[0]) \
            / mdf_train.loc[mdf_train[protected_feature].isna()].shape[0]
          elif attribute == attribute and entry != entry:
            attribute_weight = (mdf_train.loc[(mdf_train[protected_feature] == attribute) & (mdf_train[DPod_column].isna())].shape[0]) \
            / mdf_train.loc[(mdf_train[protected_feature] == attribute)].shape[0]
          elif attribute != attribute and entry != entry:
            attribute_weight = (mdf_train.loc[(mdf_train[protected_feature].isna()) & (mdf_train[DPod_column].isna())].shape[0]) \
            / mdf_train.loc[(mdf_train[protected_feature].isna())].shape[0]
          
          attribute_weights.append(attribute_weight)
          
        attribute_weightings_dict.update({attribute : attribute_weights})

    #___

    binomial_activation_count = 0
    test_binomial_activation_count = 0

    if trainnoise is True and flip_prob > 0:  
      
      if flip_prob == 1:
        mdf_train[DPod_tempcolumn1] = np.ones(mdf_train.shape[0]).astype(int)
      else:
        #derive our sampled noise for injection
        sampling_id = 'binomial_train'
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_train.shape[0]
        mdf_train[DPod_tempcolumn1] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0])), index=mdf_train.index)
      binomial_activation_count = mdf_train[DPod_tempcolumn1].sum()
  
      sampling_id = 'choice_train'
      
      if protected_feature is False or weighted is False:
        
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_train.shape[0]
        
        if weighted is False:
          choice_samples = nprandom.choice(ord_encodings, size=(binomial_activation_count))
        elif weighted is True:
          choice_samples = nprandom.choice(ord_encodings, p=weights, size=(binomial_activation_count))

        #the samples were a shape based on number of binomial activations, now inject
        # if passthrough is False:
        #   mdf_train[DPod_column] = \
        #   mdf_train[DPod_column] * (1 - mdf_train[DPod_tempcolumn1]) + mdf_train[DPod_tempcolumn1] * mdf_train[DPod_tempcolumn2]
        # else:
        #this is equivalent to the multiplication operation and adds support for non-numeric entries
        mdf_train.loc[(mdf_train[DPod_tempcolumn1] == 1), DPod_column] = choice_samples
        
      elif protected_feature is not False:
        
        seeds_available = sampling_resource_dict[sampling_id + '_seeds']
        
        for attribute in attributes:
          
          temp_sampling_resource_dict = self.__autocopy(sampling_resource_dict)
          
          if attribute == attribute:
            attribute_count = mdf_train.loc[(mdf_train[protected_feature]==attribute) & (mdf_train[DPod_tempcolumn1] == 1)].shape[0]
          elif attribute != attribute:
            attribute_count = mdf_train.loc[(mdf_train[protected_feature].isna()) & (mdf_train[DPod_tempcolumn1] == 1)].shape[0]
          
          attribute_seeds = seeds_available
          
          if attribute_count < len(seeds_available):
            
            attribute_seeds = seeds_available[:attribute_count]
            
          temp_sampling_resource_dict.update({sampling_id + '_seeds' : attribute_seeds})
          
          nprandom = self.__get_nprandom(sampling_id, temp_sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += attribute_count
          
          choice_samples = nprandom.choice(ord_encodings, 
                                           p=attribute_weightings_dict[attribute], 
                                           size=(attribute_count))

          if attribute == attribute:

            #the samples were a shape based on number of binomial activations, now extract to full column
            mdf_train.loc[(mdf_train[protected_feature]==attribute) & (mdf_train[DPod_tempcolumn1] == 1), DPod_column] = choice_samples

          elif attribute != attribute:
            #the samples were a shape based on number of binomial activations, now inject
            mdf_train.loc[(mdf_train[protected_feature].isna()) & (mdf_train[DPod_tempcolumn1] == 1), DPod_column] = choice_samples

          #if we have enough seeds left for the next attribute we'll expend the current used
          if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
            next_attribute = attributes[(1+attributes.index(attribute))]
            next_attribute_count = mdf_train.loc[mdf_train[protected_feature]==next_attribute].shape[0]

            if len(seeds_available[attribute_count:]) < next_attribute_count:
              seeds_available = seeds_available[attribute_count:]

      del mdf_train[DPod_tempcolumn1]

    #for test data is just pass-through unless testnoise or traindata is activated
    if testnoise is True and test_flip_prob > 0:
      
      if test_flip_prob == 1:
        mdf_test[DPod_tempcolumn1] = np.ones(mdf_test.shape[0]).astype(int)
      else:
        #derive our sampled noise for injection
        sampling_id = 'binomial_test'
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
        mdf_test[DPod_tempcolumn1] = pd.DataFrame(nprandom.binomial(n=1, p=test_flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)
      test_binomial_activation_count = mdf_test[DPod_tempcolumn1].sum()
  
      sampling_id = 'choice_test'
      
      if protected_feature is False or test_weighted is False:
        
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
        
        if test_weighted is False:
          choice_samples = nprandom.choice(ord_encodings, size=(test_binomial_activation_count))
        elif test_weighted is True:
          choice_samples = nprandom.choice(ord_encodings, p=weights, size=(test_binomial_activation_count))

        #the samples were a shape based on number of binomial activations, now inject
        mdf_test.loc[mdf_test[DPod_tempcolumn1] == 1, DPod_column] = choice_samples
        
      elif protected_feature is not False:
        
        seeds_available = sampling_resource_dict[sampling_id + '_seeds']
        
        #for test data we'll also consider test attributes that weren't found in the train data
        #and apply aggregate feature weighting
        #since attribute_weightings_dict is inspected as part of sampling we'll need to add temporary entries
        temp_attribute_weightings_dict = self.__autocopy(attribute_weightings_dict)
        
        test_attributes = list(mdf_test[protected_feature].unique())
        extra_test_attributes = set(test_attributes) - set(attributes)
        
        for extra_test_attribute in extra_test_attributes:
          temp_attribute_weightings_dict.update({extra_test_attribute : weights})
        
        for attribute in (attributes + list(extra_test_attributes)):
          
          temp_sampling_resource_dict = self.__autocopy(sampling_resource_dict)
          
          if attribute == attribute:
            attribute_count = mdf_test.loc[(mdf_test[protected_feature]==attribute) & (mdf_test[DPod_tempcolumn1] == 1)].shape[0]
          elif attribute != attribute:
            attribute_count = mdf_test.loc[(mdf_test[protected_feature].isna()) & (mdf_test[DPod_tempcolumn1] == 1)].shape[0]
          
          attribute_seeds = seeds_available
          
          if attribute_count < len(seeds_available):
            
            attribute_seeds = seeds_available[:attribute_count]
            
          temp_sampling_resource_dict.update({sampling_id + '_seeds' : attribute_seeds})
          
          nprandom = self.__get_nprandom(sampling_id, temp_sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += attribute_count
          
          choice_samples = nprandom.choice(ord_encodings, 
                                           p=temp_attribute_weightings_dict[attribute], 
                                           size=(attribute_count))
          
          if attribute == attribute:
            #the samples were a shape based on number of binomial activations, now inject
            mdf_test.loc[(mdf_test[protected_feature]==attribute) & (mdf_test[DPod_tempcolumn1] == 1), DPod_column] = choice_samples
            
          elif attribute != attribute:
            #the samples were a shape based on number of binomial activations, now inject
            mdf_test.loc[(mdf_test[protected_feature].isna()) & (mdf_test[DPod_tempcolumn1] == 1), DPod_column] = choice_samples

          #if we have enough seeds left for the next attribute we'll expend the current used
          if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
            next_attribute = attributes[(1+attributes.index(attribute))]
            if next_attribute == next_attribute:
              next_attribute_count = mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0]
            elif next_attribute != next_attribute:
              next_attribute_count = mdf_test.loc[mdf_test[protected_feature].isna()].shape[0]

            if len(seeds_available[attribute_count:]) < next_attribute_count:
              seeds_available = seeds_available[attribute_count:]

      del mdf_test[DPod_tempcolumn1]
      
    #___

    if passthrough is False:
      #data type is conditional based on encoding space
      max_encoding = len(ord_encodings) - 1
      if max_encoding <= 255:
        mdf_train[DPod_column] = mdf_train[DPod_column].astype(np.uint8)
        mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_train[DPod_column] = mdf_train[DPod_column].astype(np.uint16)
        mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint16)
      else:
        mdf_train[DPod_column] = mdf_train[DPod_column].astype(np.uint32)
        mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint32)

    #create list of columns
    nmbrcolumns = [DPod_column]
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    nmbrnormalization_dict = {DPod_column : {'flip_prob' : flip_prob, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'ord_encodings' : ord_encodings, \
                                             'weighted' : weighted, \
                                             'test_weighted' : test_weighted, \
                                             'weights' : weights, \
                                             'suffix' : suffix, \
                                             'inplace' : inplace, \
                                             'passthrough' : passthrough, \
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'binomial_activation_count' : binomial_activation_count, \
                                             'test_binomial_activation_count' : test_binomial_activation_count, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'retain_basis' : retain_basis, \
                                             'protected_feature' : protected_feature, \
                                             'attribute_weightings_dict' : attribute_weightings_dict, \
                                             'attributes' : attributes, \
                                            }}
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_DPmc(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPmc(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to categoric training data, such as for differential privacy purposes
    #similar to DPod in that it is intended for application downstream of categoric trasnforms
    #but adds support for direct application downstream of categoric transforms returning multi-column sets
    #whereas DPod relies on feature being ordinally encoded
    
    #ratio of entries receiving injection is based on flip_prob parameter
    #which is used as a ratio for Bernoulli sampling
    
    #noise injection results in flipping to a randomly sampled alternate distribution set
    #(including potenitally the current set for no flip)
    #weighted by distribution of unique activation sets found in the training data
    #weighting can be deactivated with the weighted parameter to sample from uniform distribution
    
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through transform
    #unless testnoise is activated which applies test noise injection in both automunge and postmunge
    
    #note this assumes clean data as input since this will be intended for downstream application
    #in family trees, so no infill is performed

    #for distinguishing between test noise injection as on/off in postmunge can use postmunge traindata parameter
    #such that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default

    #consistent with other DP trasnforms DPmc is intended for downstream use
    #and so assumes data already integer encoded with infill
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = 0.01
      
    if 'swap_noise' in params:
      swap_noise = params['swap_noise']
    else:
      swap_noise = False
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True 

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    if 'weighted' in params:
      weighted = params['weighted']
    else:
      weighted = True
      
    if 'test_weighted' in params:
      test_weighted = params['test_weighted']
    else:
      test_weighted = weighted
      
    #max encoding for conditional data type inspects upstream hs10 hashing vocab_size when selected
    if 'upstream_hs10' in params:
      upstream_hs10 = params['upstream_hs10']
    else:
      upstream_hs10 = False
      
    if 'messy_data_headers' in params:
      messy_data_headers = params['messy_data_headers']
      if messy_data_headers == []:
        messy_data_headers = [column]
    else:
      messy_data_headers = False
      
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    if 'protected_feature' in params:
      protected_feature = params['protected_feature']

      if protected_feature == column \
      or column in postprocess_dict['column_dict'] \
      and protected_feature == postprocess_dict['column_dict'][column]['origcolumn'] \
      or protected_feature not in mdf_train:
        protected_feature = False

    else:
      protected_feature = False
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False
      
    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform']  specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      flip_prob_dist = False
      test_flip_prob_dist = False
      
    #___
      
    #this function is intended to be applied donstream of a potentially multi column non current encoding 
    #(e.g. muiltirt, 1010) - (for concurrent scenario we recomend applying noise injection through mlti)
    #meaning there may be multiple columns serving as target
    #we'll access the upstream categorylist stored in postprocess_dict
    #or otherwise if this is an upstream trasnform just apply to the received column
    if column in postprocess_dict['column_dict']:
      inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
    else:
      #origcolumn scenario when not applied as downstream transform
      #note that this scenario also is automatically applied 
      #when this transform applied downstream of a concurrent MLinfilltype with mlti
      #since at that point column won't be in column_dict
      #which is desired behavior for that use case
      inputtextcolumns = [column]
      
    if isinstance(messy_data_headers, list) and column not in postprocess_dict['column_dict']:
      inputtextcolumns = messy_data_headers.copy()

      none_categorical = True
      for inputtextcolumn in inputtextcolumns:
        if isinstance(mdf_train[inputtextcolumn].dtype, pd.api.types.CategoricalDtype) \
        or isinstance(mdf_test[inputtextcolumn].dtype, pd.api.types.CategoricalDtype):
          none_categorical = False

      if none_categorical is False:
        mdf_train[inputtextcolumns] = mdf_train[inputtextcolumns].astype('object')
        mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].astype('object')

      #since messy_data_headers is for input headers, we'll apply an adjinfill
      #since DPmc needs all valid entries
      #apply ffill to replace NArows with value from adjacent cell in preceding row
      mdf_train[inputtextcolumns] = mdf_train[inputtextcolumns].fillna(method='ffill')
      mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(method='ffill')

      #we'll follow with a bfill just in case first row had a nan
      mdf_train[inputtextcolumns] = mdf_train[inputtextcolumns].fillna(method='bfill')
      mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(method='bfill')

      mdf_train[inputtextcolumns] = mdf_train[inputtextcolumns].fillna(0)
      mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(0)
    
    #the returned columns will each have consistent suffix appending
    textcolumns = [(x + '_' + suffix) for x in inputtextcolumns]
    
    #this maps between received and returned columns
    textlabelsdict = dict(zip(inputtextcolumns, textcolumns))
    
    #we don't offer inplace support for most DP transforms since they usually involve support columns
    #we might be able to make inplace work for this one
    
    #check suffixoverlap before copying
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results, postprocess_dict['printstatus'])
    
    if inplace is not True:
      
      for inputtextcolumn in inputtextcolumns:
        
        mdf_train[textlabelsdict[inputtextcolumn]] = mdf_train[inputtextcolumn].copy()
        mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()
    
    else:
      
      mdf_train.rename(columns = textlabelsdict, inplace = True)
      mdf_test.rename(columns = textlabelsdict, inplace = True)
      
    #subsequent data type conversion relies on max encoding, we'll apply this before noise injection
    maxencodings = {}
    if swap_noise is False:
      if upstream_hs10 is False and not isinstance(messy_data_headers, list):
        for textcolumn in textcolumns:
          maxencoding = mdf_train[textcolumn].max()
          maxencodings.update({textcolumn : maxencoding})
      elif upstream_hs10 is True:
        
        if column in postprocess_dict['column_dict'] \
        and column in postprocess_dict['column_dict'][column]['normalization_dict'] \
        and 'vocab_size' in postprocess_dict['column_dict'][column]['normalization_dict'][column]:
          maxencoding = postprocess_dict['column_dict'][column]['normalization_dict'][column]['vocab_size']
        else:
          maxencoding = mdf_train[column].max()
        
        for textcolumn in textcolumns:
          maxencodings.update({textcolumn : maxencoding})

    if swap_noise is False:
    
#       if protected_features is False or weighted is False and test_weighted is False:

      #now we'll create a support dataframe df_unique from df_train
      #which we'll consolidate redundant rows in df_unique before saving to normalization_dict
      df_unique = mdf_train[textcolumns].copy()

      #first we derive a mask based on presence of duplicate rows
      mask = pd.DataFrame(df_unique.duplicated())

      #this operation inverts True and False in the mask for next operation
      mask = pd.Series(mask[list(mask)[0]].astype(int) - 1).abs().astype(bool)

      #now apply the mask to consolidate duplicate rows, this returns a dataframe with all unique rows
      #which may be much fewer rows than received df_unique 
      df_unique = df_unique.iloc[mask.to_numpy()]

      #reset index in df_unique to a range index
      df_unique = df_unique.reset_index(drop=True)

      weights = []
      if weighted is True:
        #for each unique row in the consolidated df_unique calculate a weight
        #by count of that row in mdf_train[textcolumns]
        #note this is applied in same order as unique_range fed to np.random.choice
        for i in range(df_unique.shape[0]):
          #to derive, we'll create a comparably shaped dataframe with all same row for comparison to mdf_train[textcolumns]
          df_allonerow = pd.concat([df_unique[i:i+1]]*mdf_train.shape[0], axis=0)
          df_allonerow.index = mdf_train.index
          count = ((mdf_train[textcolumns] == df_allonerow).sum(axis=1) == len(textcolumns)).sum()
          weight = count / mdf_train.shape[0]
          weights.append(weight)

      #attribute_weightings_dict = {attribute : {'weights' : weights, 'df_unique' : df_unique}}
      attribute_weightings_dict = {}
      attributes = []          
      if protected_feature is not False:
        #the protected_feature option when elected populates 
        #a dictionary of segment weightings as attribute_weightings_dict
        #which is used for scaling basis to both train and test data

        if protected_feature is not False and weighted is True:

          #if we are scaling noise based on segments from a protected feature
          #we will first populated a dictionary mapping unique entries of protectted feature to ratio of entries

          attributes = list(mdf_train[protected_feature].unique())

          for attribute in attributes:

            #now we'll create a support dataframe df_unique from df_train
            #which we'll consolidate redundant rows in df_unique before saving to normalization_dict
            attribute_df_unique = mdf_train[textcolumns].copy()
            
            if attribute == attribute:
              #this results in reducing the rows to only contain entries correponding to the target attribute in protected feature
              attribute_df_unique = attribute_df_unique.loc[(mdf_train[protected_feature] == attribute)].copy()
            else:
              attribute_df_unique = attribute_df_unique.loc[(mdf_train[protected_feature].isna())].copy()
            
            #first we derive a mask based on presence of duplicate rows
            mask = pd.DataFrame(attribute_df_unique.duplicated())
            
            #this operation inverts True and False in the mask for next operation
            mask = pd.Series(mask[list(mask)[0]].astype(int) - 1).abs().astype(bool)
            
            #now apply the mask to consolidate duplicate rows, this returns a dataframe with all unique rows
            #which may be much fewer rows than received df_unique 
            attribute_df_unique = attribute_df_unique.iloc[mask.to_numpy()]

            #reset index in df_unique to a range index
            attribute_df_unique = attribute_df_unique.reset_index(drop=True)
            
            attribute_weights = []
            if weighted is True:
              #for each unique row in the consolidated df_unique calculate a weight
              #by count of that row in mdf_train[textcolumns]
              #note this is applied in same order as unique_range fed to np.random.choice
              for i in range(df_unique.shape[0]):
                #to derive, we'll create a comparably shaped dataframe 
                #with all same row for comparison to mdf_train[textcolumns]

                if attribute == attribute:
                  df_allonerow = pd.concat([df_unique[i:i+1]]*mdf_train.loc[(mdf_train[protected_feature] == attribute)].shape[0], axis=0)
                  df_allonerow.index = mdf_train.loc[(mdf_train[protected_feature] == attribute)].index
                
                  #this is not pretty, but .loc only allows selecting one column, so have to populate an additional copy
                  #I believe only used for this protected feature setup
                  mdf_train_textcolumns = mdf_train[textcolumns].copy()
                  
                  count = ((mdf_train_textcolumns.loc[(mdf_train[protected_feature] == attribute)] == df_allonerow).sum(axis=1) == len(textcolumns)).sum()
                  attribute_weight = count / mdf_train.loc[(mdf_train[protected_feature] == attribute)].shape[0]

                  if attribute_weight != 0:
                    attribute_weights.append(attribute_weight)

                else:
                  df_allonerow = pd.concat([df_unique[i:i+1]]*mdf_train.loc[(mdf_train[protected_feature].isna())].shape[0], axis=0)
                  df_allonerow.index = mdf_train.loc[(mdf_train[protected_feature].isna())].index

                  #this is not pretty, but .loc only allows selecting one column, so have to populate an additional copy
                  #I believe only used for this protected feature setup
                  mdf_train_textcolumns = mdf_train[textcolumns].copy()
                  
                  count = ((mdf_train_textcolumns.loc[(mdf_train[protected_feature].isna())] == df_allonerow).sum(axis=1) == len(textcolumns)).sum()
                  attribute_weight = count / mdf_train.loc[(mdf_train[protected_feature].isna())].shape[0]

                  if attribute_weight != 0:
                    attribute_weights.append(attribute_weight)
            
            #attribute_weightings_dict = {attribute : {'weights' : weights, 'df_unique' : df_unique}}
            #attribute_df_unique is used in a similar fashion to ord_encodings in DPod
            #the difference is it encompasses multiple columns
            attribute_weightings_dict.update({attribute : {
              'weights' : attribute_weights,
              'df_unique' : attribute_df_unique
            }})

      #___

    elif swap_noise is True:
      #df_unique not used in swap_noise scenario
      df_unique = False
      #swap_noise doesn't have weighted support
      weighted = False
      test_weighted = False
      weights = []
      attribute_weightings_dict = {}
      attributes = []

    def _noise_inject(df, textcolumns, df_unique, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, traintest, protected_feature=False):
      """
      common support function used to inject noise to either mdf_train or mdf_test when applicable
      """
      
      sampling_resource_dict = self.__autocopy(sampling_resource_dict)
      
      unique_count = df_unique.shape[0]
      unique_range = list(range(unique_count))
    
      #df_noise is support dataframe for noise injection to data
      df_noise = pd.DataFrame(index = df.index)
      df_noise_tempcolumn1 = 1
      df_noise_tempcolumn2 = 2

      if flip_prob == 1:
        df_noise[df_noise_tempcolumn1] = np.ones(df.shape[0]).astype(int)
      else:
        #df_noise_tempcolumn1 will return 1 for rows receiving injection and 0 elsewhere
        sampling_id = 'binomial_' + traintest
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
        df_noise[df_noise_tempcolumn1] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(df.shape[0])), index=df.index)
      binomial_activation_count = df_noise[df_noise_tempcolumn1].sum()
      
      #_____
      if protected_feature is not False:
        
        seeds_available = sampling_resource_dict['choice_' + traintest + '_seeds']

        if binomial_activation_count < len(seeds_available):
          
          #in the protected_feature case this function is being called on the attribute partition
          attribute_seeds = seeds_available[:binomial_activation_count]
          
          sampling_id = 'choice_' + traintest
          sampling_resource_dict.update({sampling_id + '_seeds' : attribute_seeds})
          
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          
      elif protected_feature is False:
        
        sampling_id = 'choice_' + traintest
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
      if weighted is False:
        choice_samples = nprandom.choice(unique_range, size=(binomial_activation_count))
      elif weighted is True:
        choice_samples = nprandom.choice(unique_range, p=weights, size=(binomial_activation_count))
        
      #the samples were a shape based on number of binomial activations, now extract to full column
      df_noise.loc[:, df_noise_tempcolumn2] = 0
      df_noise.loc[df_noise[df_noise_tempcolumn1] == 1, df_noise_tempcolumn2] = choice_samples

      #df_unique2 will be used to populate replacement activation sets corresponding to indexes sampled in df_noise[df_noise_tempcolumn2]
      df_unique2 = df_unique.iloc[df_noise[df_noise_tempcolumn2]]
      df_unique2.index = df.index

      #now we can carry the values from df_unique2 to replace entries in df associated with noise injection
      #for cases where df_noise[df_noise_tempcolumn1] was sampled as 1
      #here the columns in df_unique2 match the columns in df[textcolumns]
      for textcolumn in textcolumns:

        #setting to dataframe for single column case
        df = \
        self.__autowhere(df, textcolumn, df_noise[df_noise_tempcolumn1] == 1, df_unique2[textcolumn], specified='replacement')
      
      return df, sampling_resource_dict, binomial_activation_count
    
    binomial_activation_count = 0
    test_binomial_activation_count = 0

    if trainnoise is True and flip_prob > 0:
      
      if swap_noise is False:
        if protected_feature is False: 
          #inject noise to mdf_train
          mdf_train, sampling_resource_dict, binomial_activation_count = \
          _noise_inject(mdf_train, textcolumns, df_unique, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, 'train')
        elif protected_feature is not False:
          
          binomial_activation_count = 0
          for attribute in attributes:
            
            if attribute == attribute:
              returned_noise_inject_df, temp_sampling_resource_dict, binomial_activation_count_attribute = \
              _noise_inject(mdf_train.loc[mdf_train[protected_feature] == attribute].copy(), 
                            textcolumns, 
                            attribute_weightings_dict[attribute]['df_unique'], 
                            flip_prob, 
                            weighted, 
                            attribute_weightings_dict[attribute]['weights'], 
                            sampling_resource_dict, 
                            nprandom_dict, 
                            'train',
                            protected_feature)
            
              for textcolumn in textcolumns:
                mdf_train.loc[mdf_train[protected_feature] == attribute, textcolumn] = returned_noise_inject_df[textcolumn]
              del returned_noise_inject_df

            elif attribute != attribute:
              returned_noise_inject_df, temp_sampling_resource_dict, binomial_activation_count_attribute = \
              _noise_inject(mdf_train.loc[mdf_train[protected_feature].isna()].copy(), 
                            textcolumns, 
                            attribute_weightings_dict[attribute]['df_unique'], 
                            flip_prob, 
                            weighted, 
                            attribute_weightings_dict[attribute]['weights'], 
                            sampling_resource_dict, 
                            nprandom_dict, 
                            'train',
                            protected_feature)

              for textcolumn in textcolumns:
                mdf_train.loc[mdf_train[protected_feature].isna(), textcolumn] = returned_noise_inject_df[textcolumn]
              del returned_noise_inject_df

            binomial_activation_count += binomial_activation_count_attribute
            
            #if we have enough seeds left for the next attribute we'll expend the current used
            if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
              next_attribute = attributes[(1+attributes.index(attribute))]
              
              if next_attribute == next_attribute:
                next_attribute_count = int(mdf_train.loc[mdf_train[protected_feature]==next_attribute].shape[0] * flip_prob * 0.15)
              else:
                next_attribute_count = int(mdf_train.loc[mdf_train[protected_feature].isna()].shape[0] * flip_prob * 0.15)
              
              if len(sampling_resource_dict['choice_train_seeds'][binomial_activation_count_attribute:]) < next_attribute_count:                
                #this expends the used seeds
                sampling_resource_dict['choice_train_seeds'] = sampling_resource_dict['choice_train_seeds'][binomial_activation_count_attribute:]
               
            #for protected_feature case we updated seeds for choice, now revert to orig but keep other logged entries
            temp_sampling_resource_dict.update({'choice_train_seeds' : sampling_resource_dict['choice_train_seeds']})
            sampling_resource_dict = temp_sampling_resource_dict

      #in swap_noise scenario we replace df_unique with the full dataframe
      elif swap_noise is True:
        mdf_train, sampling_resource_dict, binomial_activation_count = \
        _noise_inject(mdf_train, textcolumns, mdf_train, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, 'train')
    
    #inspect testnoise for determination of whether to inject noise to mdf_test
    if testnoise is True and test_flip_prob > 0:
      
      if swap_noise is False:
        
        if protected_feature is False: 
          #inject noise to mdf_test
          mdf_test, sampling_resource_dict, test_binomial_activation_count = \
          _noise_inject(mdf_test, textcolumns, df_unique, test_flip_prob, test_weighted, weights, sampling_resource_dict, nprandom_dict, 'test')
      
        elif protected_feature is not False:
          
          test_binomial_activation_count = 0
          
          #first we'll apply noise without scaling for test qattributes that weren't found in train set
          #this is neccesary because wse're inspecting protected_feature in the input state
          test_attributes = list(mdf_test[protected_feature].unique())
          extra_test_attributes = set(test_attributes) - set(attributes)
          
          if len(extra_test_attributes) > 0:

            returned_noise_inject_df, temp_sampling_resource_dict, test_binomial_activation_count_attribute = \
            _noise_inject(mdf_test.loc[mdf_test[protected_feature].isin(extra_test_attributes)].copy(), 
                          textcolumns, 
                          df_unique, 
                          test_flip_prob, 
                          test_weighted, 
                          weights, 
                          sampling_resource_dict, 
                          nprandom_dict, 
                          'test',
                          protected_feature)

            for textcolumn in textcolumns:
              if attribute == attribute:
                mdf_test.loc[mdf_test[protected_feature] == attribute, textcolumn] = returned_noise_inject_df[textcolumn]
              else:
                mdf_test.loc[mdf_test[protected_feature].isna(), textcolumn] = returned_noise_inject_df[textcolumn]
            del returned_noise_inject_df

            test_binomial_activation_count += test_binomial_activation_count_attribute

            #if we have enough seeds left for the next attribute we'll expend the current used
            if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
              next_attribute = attributes[0]

              if next_attribute == next_attribute:
                next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0] * test_flip_prob * 0.15)
              else:
                next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature].isna()].shape[0] * test_flip_prob * 0.15)

              if len(sampling_resource_dict['choice_test_seeds'][test_binomial_activation_count_attribute:]) < next_attribute_count:                
                #this expends the used seeds
                sampling_resource_dict['choice_test_seeds'] = sampling_resource_dict['choice_test_seeds'][test_binomial_activation_count_attribute:]

            #for protected_feature case we updated seeds for choice, now revert to orig but keep other logged entries
            temp_sampling_resource_dict.update({'choice_test_seeds' : sampling_resource_dict['choice_test_seeds']})
            sampling_resource_dict = temp_sampling_resource_dict
          
          #now cycle through the attributes
          for attribute in attributes:
            
            if attribute == attribute:
              returned_noise_inject_df, temp_sampling_resource_dict, test_binomial_activation_count_attribute = \
              _noise_inject(mdf_test.loc[mdf_test[protected_feature] == attribute].copy(), 
                            textcolumns, 
                            attribute_weightings_dict[attribute]['df_unique'], 
                            test_flip_prob, 
                            test_weighted, 
                            attribute_weightings_dict[attribute]['weights'], 
                            sampling_resource_dict, 
                            nprandom_dict, 
                            'test',
                            protected_feature)

              for textcolumn in textcolumns:
                mdf_test.loc[mdf_test[protected_feature] == attribute, textcolumn] = returned_noise_inject_df[textcolumn]
              del returned_noise_inject_df

            elif attribute != attribute:
              returned_noise_inject_df, temp_sampling_resource_dict, test_binomial_activation_count_attribute = \
              _noise_inject(mdf_test.loc[mdf_test[protected_feature].isna()].copy(), 
                            textcolumns, 
                            attribute_weightings_dict[attribute]['df_unique'], 
                            test_flip_prob, 
                            test_weighted, 
                            attribute_weightings_dict[attribute]['weights'], 
                            sampling_resource_dict, 
                            nprandom_dict, 
                            'test',
                            protected_feature)

              for textcolumn in textcolumns:
                mdf_test.loc[mdf_test[protected_feature].isna(), textcolumn] = returned_noise_inject_df[textcolumn]
              del returned_noise_inject_df

            test_binomial_activation_count += test_binomial_activation_count_attribute
            
            #if we have enough seeds left for the next attribute we'll expend the current used
            if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
              next_attribute = attributes[(1+attributes.index(attribute))]
              
              if next_attribute == next_attribute:
                next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0] * flip_prob * 0.15)
              else:
                next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature].isna()].shape[0] * flip_prob * 0.15)
              
              if len(sampling_resource_dict['choice_test_seeds'][test_binomial_activation_count_attribute:]) < next_attribute_count:                
                #this expends the used seeds
                sampling_resource_dict['choice_test_seeds'] = sampling_resource_dict['choice_test_seeds'][test_binomial_activation_count_attribute:]
               
            #for protected_feature case we updated seeds for choice, now revert to orig but keep other logged entries
            temp_sampling_resource_dict.update({'choice_test_seeds' : sampling_resource_dict['choice_test_seeds']})
            sampling_resource_dict = temp_sampling_resource_dict
          
      #in swap_noise scenario we replace df_unique with the full dataframe
      elif swap_noise is True:
        mdf_test, sampling_resource_dict, test_binomial_activation_count = \
        _noise_inject(mdf_test, textcolumns, mdf_test, test_flip_prob, test_weighted, weights, sampling_resource_dict, nprandom_dict, 'test')
        
    #___
  
    #now apply data type conversion, this should align with received data types, just applying in case of drift
    #in swap_noise scenario we'll default to the input data type since this might be applied downstream of floats
    if swap_noise is False and not isinstance(messy_data_headers, list):
      for textcolumn in textcolumns:

        max_encoding_for_dtype_convert = maxencodings[textcolumn]

        if max_encoding_for_dtype_convert <= 255:
          mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.uint8)
          mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint8)
        elif max_encoding_for_dtype_convert <= 65535:
          mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.uint16)
          mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint16)
        else:
          mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.uint32)
          mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint32)
      
    column_dict_list = []
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    normalization_dict = {}
    if len(textcolumns) > 0:
      normalization_dict = {textcolumns[0] : {'inputtextcolumns' : inputtextcolumns, \
                                              'textcolumns' : textcolumns, \
                                              'textlabelsdict' : textlabelsdict, \
                                              'flip_prob' : flip_prob, \
                                              'test_flip_prob' : test_flip_prob, \
                                              'flip_prob_dist' : flip_prob_dist, \
                                              'test_flip_prob_dist' : test_flip_prob_dist, \
                                              'swap_noise' : swap_noise, \
                                              'testnoise' : testnoise, \
                                              'trainnoise' : trainnoise, \
                                              'weighted' : weighted, \
                                              'test_weighted' : test_weighted, \
                                              'weights' : weights, \
                                              'df_unique' : df_unique, \
                                              'suffix' : suffix, \
                                              'maxencodings' : maxencodings, \
                                              'upstream_hs10' : upstream_hs10, \
                                              'messy_data_headers' : messy_data_headers, \
                                              'inplace' : inplace, \
                                              'sampling_resource_dict' : sampling_resource_dict, \
                                              'binomial_activation_count' : binomial_activation_count, \
                                              'test_binomial_activation_count' : test_binomial_activation_count, \
                                              'flip_prob_list' : flip_prob_list, \
                                              'test_flip_prob_list' : test_flip_prob_list, \
                                              'retain_basis' : retain_basis, \
                                              'protected_feature' : protected_feature, \
                                              'attribute_weightings_dict' : attribute_weightings_dict, \
                                              'attributes' : attributes, \
                                             }}
    
    for textcolumn in textcolumns:

      if textcolumn != textcolumns[0]:
        normalization_dict = {}
    
      column_dict = {textcolumn : {'category' : treecategory, \
                                   'origcategory' : category, \
                                   'normalization_dict' : normalization_dict, \
                                   'origcolumn' : column, \
                                   'inputcolumn' : column, \
                                   'columnslist' : textcolumns, \
                                   'categorylist' : textcolumns, \
                                   'infillmodel' : False, \
                                   'infillcomplete' : False, \
                                   'suffixoverlap_results' : suffixoverlap_results, \
                                   'deletecolumn' : False}}

      column_dict_list.append(column_dict)
    
    return mdf_train, mdf_test, column_dict_list

  def _process_DPsk(self, mdf_train, mdf_test, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #process_DPsk(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #makes no assumption about input data
    #injects a mask into sampled rows based on a binomial sampling
    #mask defaults to integer 0, can be specified by mask_value parameter
    #accepts flip_prob and test_flip_prob to set injection rate
    #flip_prob/test_flip_prob can also be passed as list of condidate values or as scipy stats distribution
    '''

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      flip_prob = 0.03
      
    if 'test_flip_prob' in params:
      test_flip_prob = params['test_flip_prob']
    else:
      #default also populated in __random_parameters_params_append
      test_flip_prob = 0.01
      
    if 'testnoise' in params:
      testnoise = params['testnoise']
    else:
      testnoise = False

    if 'trainnoise' in params:
      trainnoise = params['trainnoise']
    else:
      trainnoise = True

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    if 'mask_value' in params:
      mask_value = params['mask_value']
    else:
      mask_value = 0
      
    #additive noise means the mask value is added instead of replaced
    if 'additive' in params:
      additive = params['additive']
    else:
      additive = False
      
    #for cases where distribution parameters passed as list or distribution, 
    #activating retain_basis means the basis sampled in automunge is carried through to postmunge
    #or the default of False means a unique basis is sampled in automunge and postmunge
    if 'retain_basis' in params:
      retain_basis = params['retain_basis']
    else:
      retain_basis = False

    #________
      
    #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
    #postmunge(.) has a corresponding parameter to support
    if 'random_generator' in params:
      random_generator = params['random_generator']
    else:
      random_generator = np.random.PCG64
      
    nprandom_dict = {'custom' : random_generator,
                     'default': np.random.PCG64}
      
    #note that sampling_resource_dict populated externally 
    #based on automunge(.) sampling_dict and entropy_seeds parameters 
    #and not passed to postmunge
    #postmunge(.) has corresponding parameters to support
    if 'sampling_resource_dict' in params:
      sampling_resource_dict = params['sampling_resource_dict']
    # else:
    #as configured sampling_resource_dict is always populated in params by __random_parameters_params_append based on process_dict['noise_transform'] specification
    else:
      #a validation result logged in case user forgot to swpecify processdict['noise_transform'], noting that this will also register for postmunge drift report
      logger_dict['validations'].update({'missing_process_dict_noise_transform' : category})
      # logger = self.__logger_append(logger)

      sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                'binomial_train' : 'custom',
                                'binomial_train_seeds' : [],
                                'binomial_train_call_count' : 0,
                                'binomial_train_sample_count' : 0,
                                'binomial_test' : 'custom',
                                'binomial_test_seeds' : [],
                                'binomial_test_call_count' : 0,
                                'binomial_test_sample_count' : 0,
                                'distribution_train' : 'custom',
                                'distribution_train_seeds' : [],
                                'distribution_train_call_count' : 0,
                                'distribution_train_sample_count' : 0,
                                'distribution_test' : 'custom',
                                'distribution_test_seeds' : [],
                                'distribution_test_call_count' : 0,
                                'distribution_test_sample_count' : 0,
                                'choice_train' : 'custom',
                                'choice_train_seeds' : [],
                                'choice_train_call_count' : 0,
                                'choice_train_sample_count' : 0,
                                'choice_test' : 'custom',
                                'choice_test_seeds' : [],
                                'choice_test_call_count' : 0,
                                'choice_test_sample_count' : 0,
                                'parameterlist_train' : 'custom',
                                'parameterlist_train_seeds' : [],
                                'parameterlist_train_call_count' : 0,
                                'parameterlist_train_sample_count' : 0,
                                'parameterlist_test' : 'custom',
                                'parameterlist_test_seeds' : [],
                                'parameterlist_test_call_count' : 0,
                                'parameterlist_test_sample_count' : 0,
                                'statsdistribution_train' : 'custom',
                                'statsdistribution_train_seeds' : [],
                                'statsdistribution_train_call_count' : 0,
                                'statsdistribution_train_sample_count' : 0,
                                'statsdistribution_test' : 'custom',
                                'statsdistribution_test_seeds' : [],
                                'statsdistribution_test_call_count' : 0,
                                'statsdistribution_test_sample_count' : 0,
                                'random_generator_accepts_seeds' : True,
                                }

    #________
    
    #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
    flip_prob_list = False
    if isinstance(flip_prob, list):
      flip_prob_list = flip_prob.copy()
      flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
      
    test_flip_prob_list = False
    if isinstance(test_flip_prob, list):
      test_flip_prob_list = test_flip_prob.copy()
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
    if retain_basis is True:
      flip_prob_list = False
      test_flip_prob_list = False
      
    #________
      
    #scenarios where parameters passed as a scipy stats distribution
    flip_prob_dist = False
    if isinstance(flip_prob, type(stats.expon(1))):
      flip_prob_dist = flip_prob
      flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(flip_prob, sampling_resource_dict, nprandom_dict, 'train')
      if flip_prob < 0:
        flip_prob = 0
      if flip_prob > 1:
        flip_prob = 1

    test_flip_prob_dist = False
    if isinstance(test_flip_prob, type(stats.expon(1))):
      test_flip_prob_dist = test_flip_prob
      test_flip_prob, sampling_resource_dict = \
      self.__sample_from_scipy_stats(test_flip_prob, sampling_resource_dict, nprandom_dict, 'test')
      if test_flip_prob < 0:
        test_flip_prob = 0
      if test_flip_prob > 1:
        test_flip_prob = 1
        
    if retain_basis is True:
      flip_prob_dist = False
      test_flip_prob_dist = False
      
    #___
      
    DPbn_column = column + '_' + suffix
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(mdf_train, DPbn_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
    #now inject noise
    if trainnoise is True and flip_prob > 0:
      #first we'll derive our sampled noise for injection
      sampling_id = 'binomial_train'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += mdf_train.shape[0]
      mdf_train[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0])), index=mdf_train.index)
      
      if additive is False:
        #now inject
        mdf_train = \
        self.__autowhere(mdf_train, DPbn_column, mdf_train[DPbn_column]==1, mask_value, mdf_train[column], specified='replacementalternative')
      elif additive is True:
        #this adds the additive value only onto numeric entries to avoid bug for adding number to string
        mdf_train.loc[mdf_train[column]==pd.to_numeric(mdf_train[column], errors='coerce'), DPbn_column] = \
        mdf_train.loc[mdf_train[column]==pd.to_numeric(mdf_train[column], errors='coerce'), DPbn_column] * mask_value \
        + mdf_train.loc[mdf_train[column]==pd.to_numeric(mdf_train[column], errors='coerce'), column]
        #and this carries over non-numeric entries from column to the returned column DPbn_column
        mdf_train.loc[mdf_train[column]!=pd.to_numeric(mdf_train[column], errors='coerce'), DPbn_column] = \
        mdf_train.loc[mdf_train[column]!=pd.to_numeric(mdf_train[column], errors='coerce'), column]
        
    elif trainnoise is False:
      mdf_train[DPbn_column] = mdf_train[column].copy()
    
    #for test data is just pass-through unless testnoise or traindata is activated
    if testnoise is False:
      mdf_test[DPbn_column] = mdf_test[column].copy()
    elif testnoise is True and test_flip_prob > 0:
      #first we'll derive our sampled noise for injection
      sampling_id = 'binomial_test'
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
      mdf_test[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=test_flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)

      if additive is False:
        #now inject noise
        mdf_test = \
        self.__autowhere(mdf_test, DPbn_column, mdf_test[DPbn_column]==1, mask_value, mdf_test[column], specified='replacementalternative')
      elif additive is True:
        #this adds the additive value only onto numeric entries to avoid bug for adding number to string
        mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] = \
        mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] * mask_value \
        + mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), column]
        #and this carries over non-numeric entries from column to the returned column DPbn_column
        mdf_test.loc[mdf_test[column]!=pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] = \
        mdf_test.loc[mdf_test[column]!=pd.to_numeric(mdf_test[column], errors='coerce'), column]
        
    #create list of columns
    nmbrcolumns = [DPbn_column]
    
    sampling_resource_dict = self.__erase_seeds(sampling_resource_dict)

    nmbrnormalization_dict = {DPbn_column : {'flip_prob' : flip_prob, \
                                             'test_flip_prob' : test_flip_prob, \
                                             'flip_prob_dist' : flip_prob_dist, \
                                             'test_flip_prob_dist' : test_flip_prob_dist, \
                                             'suffix' : suffix, \
                                             'testnoise' : testnoise, \
                                             'trainnoise' : trainnoise, \
                                             'sampling_resource_dict' : sampling_resource_dict, \
                                             'flip_prob_list' : flip_prob_list, \
                                             'test_flip_prob_list' : test_flip_prob_list, \
                                             'mask_value' : mask_value, \
                                             'additive' : additive, \
                                             'retain_basis' : retain_basis, \
                                            }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return mdf_train, mdf_test, column_dict_list

  def _process_qbt1(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    #translates numerical entries to Q notation
    #which is a kind of binary encoding
    #in which there are n bits devoted to integers and m bits to fractionals
    #where m and n are configuratble by parameter, and default to 
    #integer_bits = 3, fractional_bits = 12, sign_bit = 1
    #these values are arbitary, and with these defaults 
    #entries exceeding/below +/- 8.000 are subject to overflow
    #returns encodings in a set of 16 columns
    #with suffix _qbt1_2^#
    #where # is entry in range(integer_bits) or entry in -1 * range(fractional_bits)
    #and suffix for sign bit if included is _qbt1_sign
    #and entries are in order of sign, integers max->min, fractional min->max
    #for cases of overflow (inadequate registers for number size) replaced with overflow value
    """

    suffixoverlap_results = {}
    
    #initialize parameters
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    if 'integer_bits' in params:
      integer_bits = params['integer_bits']
    else:
      integer_bits = 3
      
    if 'fractional_bits' in params:
      fractional_bits = params['fractional_bits']
    else:
      fractional_bits = 12
      
    #sign_bit accepted as True or False
    if 'sign_bit' in params:
      sign_bit = params['sign_bit']
    else:
      sign_bit = True

    #abs_zero accepts boolean defaulting to True for conversion of negative zero to positive
    #which is in place to ensure defaultinfill of negzeroinfill returns distinct encoding
    if 'abs_zero' in params:
      abs_zero = params['abs_zero']
    else:
      abs_zero = True

    #angle_bits accepts boolean to indicate that activations are returned as pi instead of 1
    #(as may be suitable for encoding binary decimal in a qubit angle of 0 for |0> and pi for |1>)
    if 'angle_bits' in params:
      angle_bits = params['angle_bits']
    else:
      angle_bits = False
      
    qbt1_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, qbt1_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, qbt1_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : qbt1_column}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[qbt1_column] = pd.to_numeric(df[qbt1_column], errors='coerce')
    
    #grab a few drift stats
    minimum = df[qbt1_column].min()
    maximum = df[qbt1_column].max()
    mean = df[qbt1_column].mean()
    stdev = df[qbt1_column].std()

    #we'll have convention that all floats of negative zero converted to zero prior to infill
    if abs_zero is True:
      df = \
      self.__autowhere(df, qbt1_column, df[qbt1_column] == 0, 0, specified='replacement')
    
    #default infill is 0, kind of arbitrary, there's no perfect solution
    #recomend supplementing with NArw if a marker needed
    #apply defaultinfill based on processdict entry
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, qbt1_column, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)

    #overflow is when entries have inadequate bits to represent, we'll set to 0 consistent with infill
    overflow = 0
    for i in range(integer_bits):
      overflow += 2**i
    for i in range(fractional_bits):
      overflow += 2**(-(i+1))
      
    #now replace overflow
    df = \
    self.__autowhere(df, qbt1_column, df[qbt1_column] > overflow, overflow, specified='replacement')
    
    if sign_bit is True:
      df = \
      self.__autowhere(df, qbt1_column, df[qbt1_column] < -overflow, -overflow, specified='replacement')
    else:
      df = \
      self.__autowhere(df, qbt1_column, df[qbt1_column] < 0, 0, specified='replacement')
      
    #list of sign columns
    if sign_bit is True:
      sign_columns = [qbt1_column + '_sign']
    else:
      sign_columns = []
      
    #list of integer columns
    integer_columns = []
    for integer in range(integer_bits-1, -1, -1):
      integer_columns.append(qbt1_column + '_2^' + str(integer))
      
    #list of fractional columns
    fractional_columns = []
    for fractional in range(fractional_bits):
      fractional_columns.append(qbt1_column + '_2^-' + str(fractional + 1))
      
    allcolumns = sign_columns + integer_columns + fractional_columns
    
    suffixoverlap_results = \
    self.__df_check_suffixoverlap(df, allcolumns, suffixoverlap_results, postprocess_dict['printstatus'])

    #populate sign column, note that 0 is positive, 1 is negative
    if sign_bit is True:

      #this returns sign column as -1 for negative entries (including negative zero) else 1
      df[sign_columns[0]] = 1

      df[sign_columns[0]] = np.copysign(df[sign_columns[0]], df[qbt1_column])

      #then convert so sign column is 1 for negative else 0
      df = \
      self.__autowhere(df, sign_columns[0], df[sign_columns[0]] == -1, 1, 0)
      
      #set data type
      df[sign_columns[0]] = df[sign_columns[0]].astype(np.int8)

      #now that sign bit is populated we'll take absolute of support column
      df[qbt1_column] = df[qbt1_column].abs()
      
    #populate integer columns
    for i, integer in enumerate(range(integer_bits-1, -1, -1)):
      
      #set value to integer column
      df[integer_columns[i]] = df[qbt1_column] / 2**integer
      
      #this rounds down to nearest integer which will be 0 or 1 unless case of overflow
      df[integer_columns[i]] = df[integer_columns[i]].astype(np.int8)

      df[qbt1_column] -= df[integer_columns[i]] * 2**integer
      
    #now populate the fractional columns
    df[qbt1_column] = df[qbt1_column] % 1

    for i, fractional in enumerate(range(fractional_bits)):

      fractional += 1
      fractional *= -1
      fractional = 2**fractional

#       #i is for indexing fractional_columns, j is the 2**(-j)
#       j = i+1

      df[fractional_columns[i]] = df[qbt1_column] / fractional

      df[fractional_columns[i]] = df[fractional_columns[i]].astype(np.int8)

      df[qbt1_column] -= df[fractional_columns[i]] * fractional

    if angle_bits is True:
      for allcolumn in allcolumns:
        #this results in 0 for |0> and pi for |1>
        df[allcolumn] *= np.pi
    
    #delete support column
    del df[qbt1_column]
    
    #allcolumns = sign_columns + integer_columns + fractional_columns
    
    column_dict_list = []

    normalization_dict = {}
    if len(allcolumns) > 0:
      normalization_dict = {allcolumns[0] : {'sign_columns' : sign_columns, \
                                            'integer_columns' : integer_columns, \
                                            'fractional_columns' : fractional_columns, \
                                            'suffix' : suffix, \
                                            'integer_bits' : integer_bits, \
                                            'fractional_bits' : fractional_bits, \
                                            'angle_bits' : angle_bits, \
                                            'sign_bit' : sign_bit, \
                                            'minimum' : minimum, \
                                            'maximum' : maximum, \
                                            'mean' : mean, \
                                            'stdev' : stdev, \
                                            'overflow' : overflow, \
                                            'abs_zero' : abs_zero, \
                                            'defaultinfill_dict' : defaultinfill_dict,
                                            'inplace' : inplace}}
    
    for ac in allcolumns:

      if ac != allcolumns[0]:
        normalization_dict = {}
      
      column_dict = {ac : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : allcolumns, \
                           'categorylist' : allcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
      
    return df, column_dict_list
  
  def _process_null(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #this is placeholder for columns given a null deletion operation
    #(such as is default for training sets containing all nan values)
    #(deletion takes place in circle of life function)
    #the returned empty column_dict_list is consistent with convention for other transforms
    #for scenarios where no columns are returned
    '''

    suffixoverlap_results = {}
    
    # df = df.drop([column], axis=1)
    #deletion takes place in circle of life function

    #note that when transforms return an empty set should return column_dict_list as empty list
    column_dict_list = []

    # column_dict = {column : {'category' : 'null', \
    #                                   'origcategory' : category, \
    #                                   'normalization_dict' : {column + '_null':{}}, \
    #                                   'origcolumn' : column, \
    #                                   'inputcolumn' : column, \
    #                                   'columnslist' : [], \
    #                                   'categorylist' : [], \
    #                                   'infillmodel' : False, \
    #                                   'infillcomplete' : False, \
    #                                   'suffixoverlap_results' : suffixoverlap_results, \
    #                                   'deletecolumn' : False}}
    
    # #now append column_dict onto postprocess_dict
    # column_dict_list.append(column_dict)

    return df, column_dict_list
  
  def _process_copy(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #copy function
    #accepts parameter 'suffix' for suffix appender
    #useful if want to apply same function more than once with different parameters
    #
    #this also may be useful when defining a family tree where the shortest path isn't desired inversion path
    #can add some intermediate copy operations to shortest path so that inversion selects the desired path
    #(as inversion operates on heuristic of selecting shortest path with full information retention, else shortest path)
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
      
    copy_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, copy_column, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, copy_column, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : copy_column}, inplace = True)

    column_dict_list = []

    column_dict = {copy_column : {'category' : treecategory, \
                                 'origcategory' : category, \
                                 'normalization_dict' : {copy_column:{'suffix' : suffix}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [copy_column], \
                                 'categorylist' : [copy_column], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_excl(self, df, column, category, treecategory, postprocess_dict, params = {}):
    """
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    #the excl trasnform is a very special exception, and this suffix is later
    #removed when automunge(*.)parameter excl_suffix passed as False
    
    #note that excl transform is also special in that it may only be applied
    #as a supplement primitive in a family tree (eg cousins)
    #as it replaces the source column internally (by a simple rename)
    
    #Note that the function check_transformdict(.) works 'under the hood'
    #to translate user passed excl transforms in family trees
    #from replacement primitives to corresponding supplement primitives
    """

    suffixoverlap_results = {}

    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    #external operations to subsequently scrub excl suffix rely on a known column + '_excl'
    # if 'suffix' in params:
    #   suffix = params['suffix']
    # else:
    #   suffix = treecategory
    suffix = 'excl'
    
    exclcolumn = column + '_' + suffix

    if inplace is not True:

      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

    else:
    
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : exclcolumn}, inplace = True)

    #decided against this to maximize efficiency
    #there are some workflow scenarios where a lot of excl columns in postmunge
    # #this moves exclcolumn to end of dataframe to maintain column order correspondance
    # df_columns = list(df)
    # df_columns.remove(exclcolumn)
    # df_columns.append(exclcolumn)
    # df = df.reindex(columns = df_columns)
    
    column_dict_list = []

    # column_dict = {exclcolumn : {'category' : treecategory, \
    column_dict = {exclcolumn : {'category' : 'excl', \
                                 'origcategory' : category, \
                                 'normalization_dict' : {exclcolumn:{'inplace' : inplace, 'suffix' : suffix}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict)

    return df, column_dict_list

  def _process_exc2(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    exclcolumn = column + '_' + suffix
    
    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[exclcolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : exclcolumn}, inplace = True)
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_train[exclcolumn] = pd.to_numeric(mdf_train[exclcolumn], errors='coerce')
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, exclcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, exclcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    exc2_normalization_dict = {exclcolumn : {'defaultinfill_dict' : defaultinfill_dict, 'inplace' : inplace, 'suffix' : suffix}}
    
    column_dict_list = []

    column_dict = {exclcolumn : {'category' : treecategory, \
                                 'origcategory' : category, \
                                 'normalization_dict' : exc2_normalization_dict, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list
  
  def _process_exc5(self, mdf_train, mdf_test, column, category, \
                         treecategory, postprocess_dict, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
      
    #we'll accept parameter integertype for determination of returned data type
    #where for 'singlct' we'll do a conditional uint based on max
    #and for 'integer' we'll just do a int32 (which allows negative values)
    #we'll define in processdict exc5 with singlct and exc8 with integer
    if 'integertype' in params:
      integertype = params['integertype']
    else:
      integertype = 'singlct'

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory
    
    exclcolumn = column + '_' + suffix
    
    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.__df_copy_train(mdf_train, column, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_test[exclcolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.__df_check_suffixoverlap(mdf_train, exclcolumn, suffixoverlap_results, postprocess_dict['printstatus'])

      mdf_train.rename(columns = {column : exclcolumn}, inplace = True)
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_train[exclcolumn] = pd.to_numeric(mdf_train[exclcolumn], errors='coerce')
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    #non integers are subject to infill
    mdf_train = \
    self.__autowhere(mdf_train, exclcolumn, mdf_train[exclcolumn] == mdf_train[exclcolumn].round(), alternative=np.nan, specified='alternative')
    mdf_test = \
    self.__autowhere(mdf_test, exclcolumn, mdf_test[exclcolumn] == mdf_test[exclcolumn].round(), alternative=np.nan, specified='alternative')

    #apply defaultinfill based on processdict entry
    #(this will default to adjinfill)
    mdf_train, defaultinfill_dict = \
    self.__apply_defaultinfill(mdf_train, exclcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    mdf_test, _1 = \
    self.__apply_defaultinfill(mdf_test, exclcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=defaultinfill_dict)
    
    #set data type based on integertype parameter
    if integertype == 'singlct':
      
      #assume encoding space is max of train data
      encodingspace = mdf_train[exclcolumn].max()

      if encodingspace <= 255:
        mdf_train[exclcolumn] = mdf_train[exclcolumn].astype(np.uint8)
        mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint8)
      elif encodingspace <= 65535:
        mdf_train[exclcolumn] = mdf_train[exclcolumn].astype(np.uint16)
        mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint16)
      else:
        mdf_train[exclcolumn] = mdf_train[exclcolumn].astype(np.uint32)
        mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint32)
      
    if integertype == 'integer':
      
      encodingspace = False
      
      mdf_train[exclcolumn] = mdf_train[exclcolumn].astype(np.int32)
      mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.int32)
      
    exc2_normalization_dict = {exclcolumn : {'defaultinfill_dict' : defaultinfill_dict, 
                                             'inplace' : inplace, 
                                             'suffix' : suffix,
                                             'encodingspace' : encodingspace,
                                             'integertype' : integertype}}
    
    column_dict_list = []

    column_dict = {exclcolumn : {'category' : treecategory, \
                                 'origcategory' : category, \
                                 'normalization_dict' : exc2_normalization_dict, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict)

    return mdf_train, mdf_test, column_dict_list
  
  def _process_shfl(self, df, column, category, treecategory, postprocess_dict, params = {}):
    '''
    #function to shuffle data in a column
    #non-numeric entries allowed
    #for missing values, uses adjacent cell infill as default
    '''

    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = treecategory

    suffixcolumn = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.__df_copy_train(df, column, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
    
    else:
      
      suffixoverlap_results = \
      self.__df_check_suffixoverlap(df, suffixcolumn, suffixoverlap_results, postprocess_dict['printstatus'])
      
      df.rename(columns = {column : suffixcolumn}, inplace = True)
    
    #randomseed is accessible in the postprocess_dict
    #where if this function is being applied in postmunge we'll give precedence to postmunge randomseed
    if 'postmunge_randomseed' in postprocess_dict:
      random = postprocess_dict['postmunge_randomseed']
    else:
      random = postprocess_dict['randomseed']
    
    #uses support function
    df = self.__df_shuffle_series(df, suffixcolumn, random)
    
    #we'll do the adjacent cell infill after the shuffle operation
    
    #apply defaultinfill based on processdict entry
    #(this will default to naninfill)
    df, defaultinfill_dict = \
    self.__apply_defaultinfill(df, suffixcolumn, postprocess_dict, treecategory=treecategory, defaultinfill_dict=False)
    
    #create list of columns
    nmbrcolumns = [suffixcolumn]

    nmbrnormalization_dict = {suffixcolumn : {'inplace' : inplace, 'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      if nc != nmbrcolumns[0]:
        nmbrnormalization_dict = {}

      column_dict = { nc : {'category' : treecategory, \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict)
        
    return df, column_dict_list

  def _custom_train_trig(self, df, column, normalization_dict):
    """
    single process function in custom_train convention 
    for use to apply trigometric transformations to a numeric feature
    built on top of numpy trigometric operations np.sin, np.cos, np.tan, np.arcsin, np.arccos, np.arctan
    accepts parameter 'operation' as one of {'sin', 'cos', 'tan', 'arcsin', 'arccos', 'arctan'} defaulting to sin
    default infill per process_dict entry
    note that numpy trig fucntions sometimes return flag printout for invalid input values
    in returned sets these instances will have a defaultinfill of 0 applied followed by any MLinfill when applicable
    """

    #check for parameter
    if 'operation' in normalization_dict:
      operation = normalization_dict['operation']
      
    #or otherwise assign and save a default value
    else:
      operation = 'sin'
      normalization_dict.update({'operation' : operation})
      
    #a few drift stats before transform
    minimum = df[column].min()
    maximum = df[column].max()
    normalization_dict.update({'minimum' : minimum,
                              'maximum' : maximum})
      
    if operation == 'sin':
      df[column] = np.sin(df[column])
      
    if operation == 'cos':
      df[column] = np.cos(df[column])
      
    if operation == 'tan':
      df[column] = np.tan(df[column])
      
    if operation == 'arcsin':
      df[column] = np.arcsin(df[column])
      
    if operation == 'arccos':
      df[column] = np.arccos(df[column])
      
    if operation == 'arctan':
      df[column] = np.arctan(df[column])
      
    return df, normalization_dict

  #__FunctionBlock: evalcategory and NArows

  def __evalcategory(self, df_source, column, randomseed, eval_ratio, \
                   numbercategoryheuristic, powertransform, labels = False):
    '''
    #Function that takes as input a dataframe and associated column id \
    #evaluates the contents of cells and classifies the column into one of following categories
    
    defaultcategorical = '1010'
    1010: for categorical data excluding special cases described following, 
    columns are subject to binarization encoding via '1010'. If the number 
    of unique entries in the column exceeds the parameter 'numbercategoryheuristic' 
    (which defaults to 255), the encoding will instead be by hashing.    
    
    defaultordinal = 'hsh2'
    hsh2: for categorical data, if the number of unique entries in the column exceeds 
    the parameter 'numbercategoryheuristic' (which defaults to 255), the encoding will 
    instead be by 'hsh2' which is an ordinal (integer) encoding based on hashing. hsh2 
    is excluded from ML infill.

    defaultordinal_allunique = 'hash'
    hash: for all unique entry categoric sets (based on sets with >75% unique entries), 
    the encoding will be by hash which extracts distinct words within entries returned 
    in a set of columns with an integer hashing. hash is excluded from ML infill. Note 
    that for edge cases with large string entries resulting in too high dimensionality, 
    the max_column_count parameter can be passed to default_assignparam in assignparam 
    to put a cap on returned column count.

    defaultbnry = 'bnry'
    bnry: for categorical data of <=2 unique values excluding infill (eg NaN), 
    the column is encoded to 0/1. Note that numerical sets with 2 unique values 
    in train set also default to bnry.
      
    defaultnumerical = 'nmbr'
    nmbr: for numerical data, columns are treated with z-score normalization. 
    If binstransform parameter was activated this will be supplemented by a 
    collection of bins indicating number of standard deviations from the mean.
    
    defaultdatetime = 'dat6'
    dat6: for time-series data, a set of derivations are performed returning 
    'year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'

    defaultnull = 'null'
    null: for columns without any valid values in training set (e.g. all NaN) 
    column is deleted
    
    note that eval_ratio sets the ratio of rows to be inspected (as may speed things up)
    
    powertransform defaults to False to be consistent with defaults above
    powertransform when activated changes the defaults in a few different scenarios
    when powertransform = 'excl' columns not explicitly assigned to a root category in assigncat will be given excl passthorugh transform
    when powertransform = 'exc2' columns not explicitly assigned to a root category in assigncat will be given exc2 passthorugh transform (forced to numeric and subject to default adjinfill). 
    when powertransform = 'infill' If the data is already numerically encoded with NaN entries for missing data, ML infill can be applied without further preprocessing transformations by passing
    when powertransform = 'infill2' data is treated comparable to infill scenario with allowance for passthrough of nonnumeric data
    when powertransform = True a statistical evaluation will be performed on numerical sets to distinguish between columns to be subject to bxcx, MAD3, nmbr, mnmx.
    when powertrasnform = 'DP1', default numerical replaced with DPnb, categoric with DP10 and binary with DPbn, etc
    when powertrasnform = 'DP2', default numerical replaced with DPrt, categoric with DPod and binary with DPbn, etc
    when powertrasnform = 'DT1', default numerical replaced with DTnb, categoric with DT10 and binary with DTbn, etc
    when powertrasnform = 'DT2', default numerical replaced with DTrt, categoric with DTod and binary with DTbn, etc
    when powertrasnform = 'DB1', default numerical replaced with DBnb, categoric with DB10 and binary with DBbn, etc
    when powertrasnform = 'DB2', default numerical replaced with DBrt, categoric with DBod and binary with DBbn, etc
    
    note that labels = True also changes the defaults to align with conventions for label sets
    which replaces the default transformations from powertransform=False to following
    lbnm: for numerical data, a label set is treated with an 'exc2' pass-through transform (without normalization).
    lbor: for categoric data of >2 unique values, a label set is treated with an 'lbor' ordinal encoding (frequency sorted encodings with no missing data bucket).
    lbbn: for categoric data of <3 unique values, a label set is treated with a bnry binary encoding (single column binarization), also applied to numeric sets with 2 unique values
    
    Note that label sets do not receive the distribution evaluation associated with powertransform=True
    And if that treatment is desired they can instead by assigned to category ptfm in assigncat

    #_evalcategory returns category id as a string

    #a future extension may allow passing powertransform as a list or dicitonary of selected options
    #for now the range of options is short enough that a single string specificaiton is sufficient
    '''

    #first we consider special cases associated with powertransform parameter passed as one of {'excl', 'exc2', 'infill'}
    #an additional powertransform parameter scenario (when passed as True) is presented further below
    
    #we'll introduce convention of special values for powertransform to change default
    #we'll allow powertransform == 'excl' to signal that nonassigned columns should
    #be left untouched (a simpler version of existing functionality of assigning excl in assigncat)
    if powertransform == 'excl':
      category = 'excl'
      
    #or powertransform == 'exc2' for unprocessed but subject to force to numeric and modeinfill
    elif powertransform == 'exc2':
      category = 'exc2'

    #____
    
    #powertransform == 'infill' is for cases where data is already numerically encoded and just want infill
    elif powertransform in {'infill', 'infill2'}:
      rowcount = df_source.shape[0]

      #we'll have convention that eval_ratio only applied for sets with >2,000 rows
      if rowcount < 2000:
        eval_ratio = 1.
      else:
        if eval_ratio > 0 and eval_ratio <= 1:
          eval_ratio = eval_ratio
        else:
          eval_ratio = eval_ratio / rowcount
      
      #take a random sample of rows for evaluation based on eval_ratio heuristic
      df = pd.DataFrame(df_source[column]).sample(frac=eval_ratio, random_state=randomseed)

      #nonintegercount = (pd.to_numeric(df[column], errors='coerce') != pd.to_numeric(df[column], errors='coerce').round()).astype(int).sum()
      nonintegerset = pd.to_numeric(df[column], errors='coerce') != pd.to_numeric(df[column], errors='coerce').round()
      numericset = pd.to_numeric(df[column], errors='coerce') == pd.to_numeric(df[column], errors='coerce')
      numericcount = numericset.sum()
      stringset = df[column].astype(str) == df[column]
      bytesset = df[column].astype(bytes) == df[column]
      stringcount = stringset.sum() + bytesset.sum()
      numericstringcount = ((stringset == True) & (numericset == True)).sum()
      nancount = (df[column] != df[column]).sum()
      integercount = (pd.to_numeric(df[column], errors='coerce') == pd.to_numeric(df[column], errors='coerce').round()).sum()
      integerset = (pd.to_numeric(df[column], errors='coerce') == pd.to_numeric(df[column], errors='coerce').round())
      integerstring = (integerset == True) & (stringset == True)
      integerstringcount = integerstring.sum()
      floatstringcount = ((nonintegerset == True) & ((stringset == True) & (numericset == True))).sum()
      actualfloatcount = numericcount - integercount - floatstringcount
      actualintegercount = integercount - integerstringcount
      uniquecount = df[column].nunique()
      rowcount = df[column].shape[0]
      setminimum = pd.to_numeric(df[column], errors='coerce').min()
      stringratio = stringcount / rowcount
      actualfloatratio = actualfloatcount / rowcount
      actualintegerratio = actualintegercount / rowcount
      uniqueratio = uniquecount / rowcount

      if powertransform == 'infill':

        #null if no numeric entries
        if actualfloatratio == 0 and actualintegerratio == 0:
          category = 'null'

        #exc2 for numeric types
        elif actualfloatratio > 0:
          category = 'exc2'

        #exc8 for integer type with unique ratio > 0.75 or if any negative integers present
        #exc8 is integer MLinfilltype (ML infill applies integer regression)
        elif (actualfloatratio == 0 and actualintegerratio > 0 and uniqueratio > 0.75) \
        or (actualfloatratio == 0 and actualintegerratio > 0 and setminimum < 0):
          category = 'exc8'

        #exc5 for integers
        #exc5 is singlct MLinfilltype (ML infill applies ordinal classification)
        else:
          category = 'exc5'

      if powertransform == 'infill2':

        #excl if no numeric entries
        if actualfloatratio == 0 and actualintegerratio == 0:
          category = 'excl'
          
        #excl if set is majority string
        elif stringratio > actualintegerratio and stringratio > actualfloatratio:
          category = 'excl'

        #exc2 for numeric types
        elif actualfloatratio > 0:
          category = 'exc2'

        #exc8 for integer type with unique ratio > 0.75 or if any negative integers present
        #exc8 is integer MLinfilltype (ML infill applies integer regression)
        elif (actualfloatratio == 0 and actualintegerratio > 0 and uniqueratio > 0.75) \
        or (actualfloatratio == 0 and actualintegerratio > 0 and setminimum < 0):
          category = 'exc8'

        #exc5 for integers
        #exc5 is singlct MLinfilltype (ML infill applies ordinal classification)
        else:
          category = 'exc5'
        
      #____
        
    else:
      
      #here are the default categories to be returned based on the evaluation
      #note that these defaults are potentially substituted at close of this function when labels = True

      if powertransform == 'DP1':

        defaultcategorical = 'DP10'
        defaultbnry = 'DPbn'
        defaultordinal = 'DPh2'
        defaultordinal_allunique = 'DPhs'
        defaultnumerical = 'DPnb'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      elif powertransform == 'DP2':

        defaultcategorical = 'DPod'
        defaultbnry = 'DPbn'
        defaultordinal = 'DPh2'
        defaultordinal_allunique = 'DPhs'
        defaultnumerical = 'DPrt'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      elif powertransform == 'DT1':

        defaultcategorical = 'DT10'
        defaultbnry = 'DTbn'
        defaultordinal = 'DTh2'
        defaultordinal_allunique = 'DThs'
        defaultnumerical = 'DTnb'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      elif powertransform == 'DT2':

        defaultcategorical = 'DTod'
        defaultbnry = 'DTbn'
        defaultordinal = 'DTh2'
        defaultordinal_allunique = 'DThs'
        defaultnumerical = 'DTrt'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      elif powertransform == 'DB1':

        defaultcategorical = 'DB10'
        defaultbnry = 'DBbn'
        defaultordinal = 'DBh2'
        defaultordinal_allunique = 'DBhs'
        defaultnumerical = 'DBnb'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      elif powertransform == 'DB2':

        defaultcategorical = 'DBod'
        defaultbnry = 'DBbn'
        defaultordinal = 'DBh2'
        defaultordinal_allunique = 'DBhs'
        defaultnumerical = 'DBrt'
        defaultdatetime = 'dat6'
        defaultnull = 'null'

      else:
      
        #default categorical
        #defaultcategorical = 'text'
        defaultcategorical = '1010'
        
        #defaultbnry is for categoric or numeric sets with nunique == 2
        defaultbnry = 'bnry'
        
        #defaultordinal applied when unique values exceeds numbercategoryheuristic
        #defaultordinal = 'ord3'
        defaultordinal = 'hsh2'
        
        #defaultordinal replaced with defaultordinal_allunique when nunique > 0.75*shape[0]
        defaultordinal_allunique = 'hash'

        #defaultnumerical is the default normalization to numeric sets
        defaultnumerical = 'nmbr'
        
        #defaultdatetime is the default encoding to datetime sets
        defaultdatetime = 'dat6'
        
        #defaultnull is for sets with all missing data
        defaultnull = 'null'

        #not rolling this scenario out in interest of simplicity, if there is a need for it let me know
        #(it is not the default because it may result in returned sets not suitable for direct application of ML)
        # if powertransform == 'excl_null':
        #   defaultnull = 'excl'
      
      #____
      
      #Now we'll sample a selection of rows from df_source to speed up this operation based on eval_ratio
      
      rowcount = df_source.shape[0]
      
      #we'll have convention that eval_ratio only applied for sets with >2,000 rows
      if rowcount < 2000:
        eval_ratio = 1.
      else:
        #if eval_ratio was passed as float between 0-1 we apply it directly
        if eval_ratio > 0 and eval_ratio <= 1:
          eval_ratio = eval_ratio
        #else if eval_ratio was passed as a number of rows we convert to a ratio
        else:
          eval_ratio = eval_ratio / rowcount
          
      #take a random sample of rows for evaluation based on eval_ratio heuristic
      df = pd.DataFrame(df_source[column]).sample(frac=eval_ratio, random_state=randomseed)
      
      #____
      
      #Now we'll count the different considered data types
      #In a few cases there will be deviations
      #such as to account for our modeling missing data as NaN which is a float data type
      #and datetime first attempts to convert entries to datetime 
      #(in case entries are string or integer representations of datetimes)
      
      #we'll determine counts of NaN, np.number, str, datetime
      #(it would be an easy extension to also check for integer count, leaving that for future consideration)
      
      #type_tuple_list will be populated with a tuple as ('type', type_ratio)
      type_tuple_list = []
      rowcount = df.shape[0]
      
      #we'll use .loc to select and count a slice of rows as a function of boolean array derived from the condition
      
      numericcount = df.loc[pd.to_numeric(df[column], errors='coerce') == df[column]].shape[0]
      type_tuple_list.append(('number', numericcount / rowcount))

      numericstringcount = (((df[column].astype(str) == df[column]) == True) & ((pd.to_numeric(df[column], errors='coerce') == pd.to_numeric(df[column], errors='coerce')) == True)).sum()
      type_tuple_list.append(('numericstring', numericstringcount / rowcount))

      nancount = df[column].isna().sum()
      type_tuple_list.append(('nan', nancount / rowcount))
      
      stringcount = df.loc[df[column].astype(str) == df[column]].shape[0]
      type_tuple_list.append(('string', stringcount / rowcount))

      bytescount = df.loc[df[column].astype(bytes) == df[column]].shape[0]
      type_tuple_list.append(('bytes', bytescount / rowcount))
      
      datetimecount = \
      df.loc[pd.to_datetime(df[column].astype(str), errors='coerce', format='mixed') == \
             pd.to_datetime(df[column].astype(str), errors='coerce', format='mixed')].shape[0]
      type_tuple_list.append(('datetime', datetimecount / rowcount))
      
      #now sort type_tuple_list by the type_ratio
      type_tuple_list = sorted(type_tuple_list, key=lambda type_tuple: type_tuple[1], reverse=True)
      
      #mostcommon_type is type with highest ratio
      mostcommon_type = 'nan'
      if len(type_tuple_list) > 0:
        mostcommon_type = type_tuple_list[0][0]
        
      #we'll also consider second_mostcommon_type such as for cases where most common is missing data
      second_mostcommon_type = mostcommon_type
      if len(type_tuple_list) > 1:
        if type_tuple_list[1][1] != 0.:
          second_mostcommon_type = type_tuple_list[1][0]

      #note that if most common is nan, we'll instead treat second most common as the most common
      if mostcommon_type == 'nan':
        mostcommon_type = second_mostcommon_type
      
      #Note that we'll also consider the pandas dtype associated with a column, particularily if is 'category'
      pandas_dtype = df[column].dtype.name
      
      #We'll treat mostcommon_type as 'string' when pandas_dtype == 'category', which will result in a categoric encoding
      if pandas_dtype == 'category':
        mostcommon_type = 'string'
      
      #a few more statistics considered
      nunique = df[column].nunique()

      #____

      #now for categoric sets (where most common is string or we received column with pandas dtype of 'category')
      #we have four scenarios
      if mostcommon_type in {'string', 'numericstring'} and numericstringcount < (stringcount / 2) \
      or mostcommon_type in {'bytes'} \
      or mostcommon_type == 'number' and nunique == 2:
        
        #base configuration is defaultcategorical (binarization)
        category = defaultcategorical
        
        #we have a single column binarization for two unique entries
        if nunique <= 2:
          category = defaultbnry
          
        #hashing is applied when nunique exceeds the automunge parameter numbercategoryheuristic
        #which is an integer defaulting to 255
        if numbercategoryheuristic is not False and nunique > numbercategoryheuristic:
          category = defaultordinal
          
        #multi-column hasing is reserved for unstructured text as evidenced by nearly all unique
        if nunique > 0.75 * rowcount:
          category = defaultordinal_allunique
      
      #now for numeric sets we default to z-score normalziation
      #note this may be overwritten below based on powertransform parameter
      elif mostcommon_type == 'number' \
      and nunique != 2 \
      or mostcommon_type in {'string', 'numericstring'} and numericstringcount >= (stringcount / 2):
        
        category = defaultnumerical
        
      #now for datetime sets we apply a set of encodings per dat6 root category
      elif mostcommon_type == 'datetime':
        
        category = defaultdatetime
        
      #final scenario is for all nan training data sets which are deleted by default
      elif mostcommon_type == 'nan':
        
        category = defaultnull
        
      #____
      
      #When powertransform = True, we measure statistics of numeric sets for potential overwrite
      if category in {defaultnumerical} \
      and powertransform is True \
      and labels is False:
        
        #these statistic tests require at least three unique entries
        if df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float).nunique() >= 3:
          
          #shapiro tests for normality, p is the metric we can compare to threshold to reject the normality hypothesis
          stat, p = stats.shapiro(df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float))
          
          #a typical threshold to test for normality is >0.05, let's try a lower bar for this application
          if p > 0.025:
            category = defaultnumerical
            
          elif p <= 0.025:
            
            #skewness helps recognize exponential distributions, reference wikipedia
            #reference from wikipedia
            # A normal distribution and any other symmetric distribution with finite third moment has a skewness of 0
            # A half-normal distribution has a skewness just below 1
            # An exponential distribution has a skewness of 2
            # A lognormal distribution can have a skewness of any positive value, depending on its parameters
            
            skewness = stats.skew(df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float))
            
            if skewness < 1.5:
              category = 'mnmx'
              
            else:
              
              #note we'll only allow bxcx category if all values greater than a clip value
              #>0 (currently set at 0.1) since there is an asymptote for box-cox at 0
              if (df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float) >= 0.1).all():
                category = 'bxcx'
                
              else:
                category = 'MAD3'
                
      #____
      
      del df
      
      #we'll overwrite to the label defaults when this is a labels set as would be designated by labels parameter
      if labels is True \
      and powertransform in {True, False, 'DP1', 'DP2'}:

        #(defaultnumerical = 'nmbr')
        if category == defaultnumerical:
          #category = 'lbnm'
          category = 'lbnb'
          
        #(defaultcategorical = '1010')
        if category == defaultcategorical:
          #category = 'lb10'
          category = 'lbor'
          
        #(defaultordinal = 'ord3')
        if category == defaultordinal:
          category = 'lbor'

        #(defaultordinal_allunique = 'ord5')
        if category == defaultordinal_allunique:
          category = 'lbor'
          
        # if category == 'text':
        #   category = 'lbor'
          
        if category == defaultbnry:
          category = 'lbbn'
          
        #(defaultdatetime = 'dat6')
        if category == defaultdatetime:
          category = 'lbda'
    
    return category

  def __getNArows(self, df, column, category, postprocess_dict, \
                drift_dict = {}, driftassess = False):
    '''
    #NArows(df, column), function that when fed a dataframe, \
    #column id, and category label outputs a single column dataframe composed of \
    #True and False with the same number of rows as the input and the True's \
    #coresponding to those rows of the input that had missing or NaN data. This \
    #output can later be used to identify which rows for a column to infill with ML\
    # derived plug data
    
    #also accepts a dictionary to store results of a drfit assessment available
    #by passing driftassess = True
    #if drift assessment performed returns an updated dictionary withj results
    
    #by default all NArowtypes recognize np.inf as NaN
    #(option activated external to this function)
    '''

    NArowtype = postprocess_dict['process_dict'][category]['NArowtype']
    
    #There is a small cost of memory overhead associated with this copy operation
    #performing on a copy to preserve properties e.g. data type retention
    #as there are scenarios where could impact consistency of returned data
    #(such as when this operation performed in automunge but not in postmunge)
    df2 = pd.DataFrame(df[column].copy())
    
    #if category == 'text':
    if NArowtype in {'justNaN'}:
      
      if driftassess is True:
        
        nunique = df2[column].nunique()
        
        #this is to ensure postprocess_dict file size doesn't get out of control so 
        #only collect unique entries in source column drift stats
        #if number of unique is below a threshold (arbrily set to 5000)
        if nunique < 5000:

          unique = df2[column].unique()

          unique_ratio = {}
          for entry in unique:
            if entry == entry:
              unique_ratio.update({entry : df2.loc[df2[column] == entry].shape[0] / df2.shape[0]})
            else:
              unique_ratio.update({entry : df2.loc[df2[column].isna()].shape[0] / df2.shape[0]})

          drift_dict.update({column : {'unique' : unique, \
                                       'unique_ratio' : unique_ratio, \
                                       'nunique' : nunique, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
          
        else:
          
          #unique entry specific drift stats not recorded when nunique > 500
          drift_dict.update({column : {'unique' : False, \
                                       'unique_ratio' : False, \
                                       'nunique' : nunique, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
          
      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})

    if NArowtype in {'binary'}:
        
      tempstring = 'asdf'
      if tempstring in df2:
        while tempstring in df2:
          tempstring += 'z'
      #valuecounts is a list of unique entries sorted by frequency (from most to least) and then alphabetic, excluding nan
      valuecounts = pd.DataFrame(df2[column].astype(str).value_counts())
      valuecounts = valuecounts.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
      valuecounts = list(valuecounts.index)

      if len(valuecounts) == 0:
        binary_1 = np.nan
        binary_2 = np.nan
      elif len(valuecounts) == 1:
        binary_1 = valuecounts[0]
        binary_2 = np.nan
      elif len(valuecounts) >= 2:
        binary_1 = valuecounts[0]
        binary_2 = valuecounts[1]
        
      if driftassess is True:
        
        nunique = df2[column].nunique()
          
        if nunique < 5000:

          drift_dict.update({column : {'unique' : valuecounts, \
                                       'nunique' : nunique, \
                                       'binary_1' : binary_1, \
                                       'binary_2' : binary_2, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
          
        else:
          
          drift_dict.update({column : {'unique' : True, \
                                       'nunique' : nunique, \
                                       'binary_1' : binary_1, \
                                       'binary_2' : binary_2, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      if len(valuecounts) < 2:

        #populates as 1 for other data else 0 for the targets
        NArows = \
        self.__autowhere(pd.DataFrame(index=df2.index), column+'_NArows', df2[column]==binary_1, False, True, specified='replacementalternative')

      if len(valuecounts) >= 2:
        #consolidate the two targets to single entry to support next operation
        df2 = \
        self.__autowhere(df2, column, df2[column]==binary_1, binary_2, specified='replacement')

        #populates as 1 for other data else 0 for the two targets
        NArows = \
        self.__autowhere(pd.DataFrame(index=df2.index), column+'_NArows', df2[column]!=binary_2, True, False, specified='replacementalternative')

      # NArows[column+'_NArows'] = NArows[column+'_NArows'].astype(bool)

    if NArowtype in {'numeric'}:

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = stats.skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
          
#         W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
#         skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                    #  'MAD' : df2[column].mad(), \
                                     'MAD' : (df2[column] - df2[column].median(axis=0)).abs().median(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in {'integer'}:

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      
      #non integers are subject to infill
      df2 = \
      self.__autowhere(df2, column, df2[column] == df2[column].round(), alternative=np.nan, specified='alternative')
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = stats.skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
          
#         W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
#         skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                    #  'MAD' : df2[column].mad(), \
                                     'MAD' : (df2[column] - df2[column].median(axis=0)).abs().median(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in {'positivenumeric'}:
      
      #this is so don't edit source column when reset values <= 0
      df2 = pd.DataFrame(df2[column]).copy()
      
      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      nonpositive_ratio = df2[df2[column] <= 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] <= 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = stats.skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                    #  'MAD' : df2[column].mad(), \
                                     'MAD' : (df2[column] - df2[column].median(axis=0)).abs().median(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nonpositive_ratio' : nonpositive_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
    
      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in {'nonnegativenumeric'}:
      
      #this is so don't edit source column when reset values < 0
      df2 = pd.DataFrame(df2[column]).copy()

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      negative_ratio = df2[df2[column] < 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] < 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = stats.skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                    #  'MAD' : df2[column].mad(), \
                                     'MAD' : (df2[column] - df2[column].median(axis=0)).abs().median(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'negative_ratio' : negative_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in {'nonzeronumeric'}:

      #this is so don't edit source column when reset values == 0
      df2 = pd.DataFrame(df2[column]).copy()

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      zero_ratio = df2[df2[column] == 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] == 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = stats.shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = stats.skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                    #  'MAD' : df2[column].mad(), \
                                     'MAD' : (df2[column] - df2[column].median(axis=0)).abs().median(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'zero_ratio' : zero_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in {'parsenumeric'}:
      
      NArows = self.__parsenumeric(df2, column)

      drift_dict.update({column : {'nunique' : df2[column].nunique(), \
                                   'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
      
    if NArowtype in {'datetime'}:
      
      df2[column] = pd.to_datetime(df2[column], errors = 'coerce', utc=True, format='mixed')

      if driftassess is True:
        drift_dict.update({column : {'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
      
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
#       NArows = self.__parsedate(df2, column)
      
    if NArowtype in {'exclude', 'totalexclude'}:
      
      if driftassess is True:
        drift_dict.update({column : {}})
      
#       NArows = pd.DataFrame(np.zeros((df2.shape[0], 1)), columns=[column+'_NArows'])
      #NArows = NArows.rename(columns = {column:column+'_NArows'})
      
      NArows = pd.DataFrame(df2[column].copy())
      NArows[column] = False
      NArows = NArows.rename(columns = {column:column+'_NArows'})
#       NArows[column+'_NArows'] = False
      
    del df2
    
    if driftassess is False:
      
      return NArows
    
    else:
      
      return NArows, drift_dict

  def __assemble_excluded_from_postmunge_getNArows(self, postprocess_dict):
    """
    #Creates a list of orig columns which can be excluded
    #from running getNArows in postmunge
    #based on infill assignment
    #(stdrdinfill does not need to run getNArows)
    """

    #get lists of which orig columns will need to run getNArows in postmunge
    excluded_from_postmunge_getNArows = []
    included_in_postmunge_getNArows = []

    #postprocess_assigninfill_dict has returned column headers per infill assignments
    for infilltype in postprocess_dict['postprocess_assigninfill_dict']:

      if infilltype == 'stdrdinfill':

        for entry in postprocess_dict['postprocess_assigninfill_dict'][infilltype]:

          infill_origcolumn = postprocess_dict['column_dict'][entry]['origcolumn']

          excluded_from_postmunge_getNArows.append(infill_origcolumn)

      #unspecified is a redundant list so excluded
      elif infilltype != 'unspecified':

        for entry in postprocess_dict['postprocess_assigninfill_dict'][infilltype]:

          infill_origcolumn = postprocess_dict['column_dict'][entry]['origcolumn']

          included_in_postmunge_getNArows.append(infill_origcolumn)

    #consolidate redundancies
    excluded_from_postmunge_getNArows = set(excluded_from_postmunge_getNArows)
    included_in_postmunge_getNArows = set(included_in_postmunge_getNArows)

    #for cases where entry in both lists default to run get_NArows in postmunge
    excluded_from_postmunge_getNArows = \
    excluded_from_postmunge_getNArows - included_in_postmunge_getNArows

    return excluded_from_postmunge_getNArows

  #__FunctionBlock: parsing support functions
  
  def __parsedate(self, df, column):
    """
    #support function for NArows
    #parses datetime entries and returns a column with boolean identification
    #for entries that aren't registering as datetime objects
    #wherein activations are 0 if a datetime is present and 1 if not
    """

    df[column] = pd.to_datetime(df[column], errors = 'coerce', format='mixed')

    NArows = pd.isna(df[column])
    NArows = pd.DataFrame(NArows)
    NArows = NArows.rename(columns = {column:column+'_NArows'})
    
    return NArows
  
  def __is_number(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric
    #partly inspired by stack overflow discussion 
    #https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
    """

    try:
      s = float(s)
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False

  def __is_number_comma(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric after stripping out commas
    #partly inspired by stack overflow discussion 
    #https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
    """

    try:
      #strips out commas
      s = float(s.replace(',',''))
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False
    
  def __is_number_EU(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric after stripping out periods
    #and replacing commas with periods
    #such as to convert from international conventions to US
    #this is relavent for cases where working with international data on US OS
    #I expect if working on international OS a different convention may be appropriate
    #based on what is recognized as a float
    """

    try:
      #strips out spaces, periods other than first and last character, replaces commas with periods, cast as float
      s = float(s[0] + s[1:-1].replace(' ','').replace('.','').replace(',','.') + s[-1])
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False
    
  def __parsenumeric(self, df, column):
    """
    #support function for process_nmrc and NArows
    #parses string entries and returns a column with boolean identification
    #for entries that include numeric string portions
    #wherein activations are 0 if a number is present and 1 if not
    #treats numeric entries as number as well
    """

    #first we find overlaps from mdf_train
    
    unique_list = list(df[column].astype(str).unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.__is_number(extract):
        
                    overlap_dict.update({unique : False})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.__is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : False})
                  
              if in_dict is False:

                overlap_dict.update({unique : True})
    
    NArows = pd.DataFrame(df[column].copy())

    NArows[column] = NArows[column].astype(str)
    NArows[column] = NArows[column].replace(overlap_dict)
#     df[column] = df[column].astype(np.int8)
    
    NArows.columns = [column+'_NArows']
    
    return NArows

  #__FunctionBlock: ML infill master functions

  def __MLinfillfunction(self, df_train, df_test, column, postprocess_dict, \
                        masterNArows_train, masterNArows_test, randomseed, \
                        ML_cmnd, printstatus, logger):
    '''
    #new function ML infill, generalizes the MLinfill application between categories
    #def __MLinfill (df_train, df_test, column, postprocess_dict, \
    #masterNArows_train, masterNArows_test, randomseed)
    #function that applies series of functions of createMLinfillsets, 
    #predictinfill, and insertinfill to a categorical encoded set.
    #please note that general convention is even single column sets are cast as dataframes instead of series
    #we prefer this convention for support of common operations independant of single or multi column sets
    '''

    #df_traininfill later returned to support infilliterate early stopping criteria
    #returned as False when infill performed with another column from categorylist
    df_traininfill = False
    
    if postprocess_dict['column_dict'][column]['infillcomplete'] is False:

      columnslist = postprocess_dict['column_dict'][column]['columnslist']
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      category = postprocess_dict['column_dict'][column]['category']
      autoMLer = postprocess_dict['autoMLer']
      
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_train[column][:0]).copy()

      elif len(categorylist) > 1:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_train[categorylist][:0]).copy()

      #createMLinfillsets
      df_train_filltrain, df_train_filllabel, df_train_fillfeatures, df_test_fillfeatures = \
      self.__createMLinfillsets(df_train, \
                         df_test, column, \
                         pd.DataFrame(masterNArows_train[origcolumn+'_NArows']), \
                         pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                         category, randomseed, postprocess_dict, \
                         ML_cmnd, columnslist = columnslist, \
                         categorylist = categorylist)
      
      #run validations of all valid numeric
      postprocess_dict = \
      self.__check_ML_infill_2(df_train_filltrain, df_train_filllabel, 
                             df_train_fillfeatures, df_test_fillfeatures, printstatus,
                             column, postprocess_dict, logger, ampm = 'am')

      #predict infill values using defined function predictinfill(.)
      df_traininfill, df_testinfill, model, postprocess_dict = \
      self.__predictinfill(column, category, df_train_filltrain, df_train_filllabel, \
                        df_train_fillfeatures, df_test_fillfeatures, randomseed, \
                        postprocess_dict, ML_cmnd, autoMLer, printstatus, logger, categorylist = categorylist)

      #now we'll add our trained model to the postprocess_dict
      postprocess_dict['column_dict'][column]['infillmodel'] \
      = model

      #note: we're only saving trained model in the postprocess_dict for one 
      #of columns from multicolumn set to reduce file size
      
      #only insert infill if we have a valid model
      if model is not False:

        #apply _stochastic_impute to train data imputations based on ML_cmnd['stochastic_impute_categoric'] or ML_cmnd['stochastic_impute_numeric']
        df_traininfill, postprocess_dict = \
        self.__stochastic_impute(ML_cmnd, df_traininfill, column, postprocess_dict, df_train=df_train_filllabel)

        #apply the function insertinfill(.) to insert missing value predictions
        df_train = self.__insertinfill(df_train, column, df_traininfill, category, \
                              pd.DataFrame(masterNArows_train[origcolumn+'_NArows']), \
                              postprocess_dict, columnslist = columnslist, \
                              categorylist = categorylist)

        if masterNArows_test[origcolumn+'_NArows'].astype(bool).any():

          #apply _stochastic_impute to test data based on ML_cmnd['stochastic_impute_categoric'] or ML_cmnd['stochastic_impute_numeric']
          df_testinfill, postprocess_dict = \
          self.__stochastic_impute(ML_cmnd, df_testinfill, column, postprocess_dict, df_train=False)

          df_test = self.__insertinfill(df_test, column, df_testinfill, category, \
                             pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                             postprocess_dict, columnslist = columnslist, \
                             categorylist = categorylist)

      #now change the infillcomplete marker in the text_dict for each \
      #associated text column unless in concurrent_activations MLinfilltype
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        
        postprocess_dict['column_dict'][column]['infillcomplete'] = True

        #now we'll add our trained text model to the postprocess_dict
        postprocess_dict['column_dict'][column]['infillmodel'] \
        = model
        
      else:
        
        for columnname in categorylist:
          postprocess_dict['column_dict'][columnname]['infillcomplete'] = True

        #now we'll add our trained model to the postprocess_dict (model only saved in first categorylist column)
        postprocess_dict['column_dict'][column]['infillmodel'] \
        = model

      #reset data type to ensure returned data is consistent with what was passed
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        df_train[column] = \
        df_train[column].astype({column:df_temp_dtype[column].dtypes})
        
        df_test[column] = \
        df_test[column].astype({column:df_temp_dtype[column].dtypes})

      elif len(categorylist) > 1:
        for dtype_column in categorylist:
          df_train[dtype_column] = \
          df_train[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})
          
          df_test[dtype_column] = \
          df_test[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})

    return df_train, df_test, postprocess_dict, df_traininfill

  def __predictinfill(self, column, category, df_train_filltrain, df_train_filllabel, \
                    df_train_fillfeatures, df_test_fillfeatures, randomseed, \
                    postprocess_dict, ML_cmnd, autoMLer, printstatus, logger, categorylist = []):
    '''
    #predictinfill(category, df_train_filltrain, df_train_filllabel, \
    #df_train_fillfeatures, df_test_fillfeatures, randomseed, categorylist), \
    #function that takes as input \
    #a category string, the output of createMLinfillsets(.), a seed for randomness \
    #and a list of columns produced by a text class preprocessor when applicable and 
    #returns predicted infills for the train and test feature sets as df_traininfill, \
    #df_testinfill based on derivations using scikit-learn, with the lenth of \
    #infill consistent with the number of True values from NArows, and the trained \
    #model
    #accepts autoMLer populated with architecture options which is applied based on entries to ML_cmnd
    '''

    #this validation result populated below may be relevant 
    #when after leakage carveouts there aren't enough remaining features to serve as basis
    if 'not_enough_samples_or_features_for_MLinfill_result' not in logger_dict['validations']:
      logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'] = {}

    #note that randomseed is received as the global automunge seed, 
    #which may either be a specified value through randomseed parameter or derived as a random integer when not specified
    #to introduce an option for stochasticity between iterations, we'll have a stochastic random seed 
    #unique to each prediction model training, which will be on by default
    #user can deactivate for more deterministic imputations with ML_cmnd['stochastic_training_seed'] = False
    #currently randomseed is inspected in randomforest, catboost
    if 'stochastic_training_seed' in ML_cmnd and ML_cmnd['stochastic_training_seed'] is False:
      randomseed = randomseed
    else:
      randomseed = random.randint(0,2147483647)
    
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = ML_cmnd['autoML_type']
  
    #MLinfilltype distinguishes between classifier/regressor, single/multi column, ordinal/onehot/binary, etc
    #see potential values documented in assembleprocessdict function
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']

    #concurrent MLinfilltypes target a single column at a time, so categorylist is reset
    if MLinfilltype in {'concurrent_nmbr', 'concurrent_act', 'concurrent_ordl'}:
      categorylist = [column]
    
    #if a numeric target set
    if MLinfilltype in {'numeric', 'concurrent_nmbr', 'integer'}:
      
      #edge case if training data has zero rows (such as if column was all NaN) 
      #also returns model as False if after carveouts there was less than 2 input features
      if df_train_filltrain.shape[0] == 0 or \
      len(self.__column_convert_support(list(df_train_filltrain), postprocess_dict, convert_to='input')) < 2:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : True}
        )
        logger = self.__logger_append(logger)

        # if printstatus is True:
        self.__autoprint('debug', "ML infill model had insufficient features or samples to serve as basis, training halted.")
        self.__autoprint('debug', "This may be a result of automated leakage carveouts from ML_cmnd['leakage_tolerance'].")
        self.__autoprint('debug', '')
      
      else:

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : False}
        )
        logger = self.__logger_append(logger)
        
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'regression'
        
        model, postprocess_dict = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict)

        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          df_traininfill = np.array([0])

        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)

      # #this might be useful for tlbn, leaving out or now since don't want to clutter mlinfilltypes
      # #as concurrent_nmbr is intended as a resource for more than just tlbn
      # if MLinfilltype == 'concurrent_nmbr':
        # df_traininfill = \
        # self.__autowhere(df_traininfill, column, df_traininfill[column] < 0, -1, specified='replacement')
        # df_testinfill = \
        # self.__autowhere(df_testinfill, column, df_testinfill[column] < 0, -1, specified='replacement')

      if MLinfilltype == 'integer':

        df_traininfill = df_traininfill.round()
        df_testinfill = df_testinfill.round()
      
    #if target is categoric, such as ordinal or boolean integers
    if MLinfilltype in {'singlct', 'binary', 'concurrent_act', 'concurrent_ordl'}:
      
      #edge case if training data has zero rows (such as if column was all NaN) 
      #also returns model as False if after carveouts there was less than 2 input features
      if df_train_filltrain.shape[0] == 0 or \
      len(self.__column_convert_support(list(df_train_filltrain), postprocess_dict, convert_to='input')) < 2:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : True}
        )
        logger = self.__logger_append(logger)

        # if printstatus is True:
        self.__autoprint('debug', "ML infill model had insufficient features or samples to serve as basis, training halted.")
        self.__autoprint('debug', "This may be a result of automated leakage carveouts from ML_cmnd['leakage_tolerance'].")
        self.__autoprint('debug', '')

      else:

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : False}
        )
        logger = self.__logger_append(logger)
        
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        if MLinfilltype in {'singlct', 'concurrent_ordl'}:
          ML_application = 'ordinalclassification'
        else:
          ML_application = 'booleanclassification'
        
        model, postprocess_dict = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict)
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          df_traininfill = np.array([0])

        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
      
    #if target is multi column categoric (onehot encoded) / (binary encoded handled seperately)
    if MLinfilltype in {'multirt'}:

      #edge case if training data has zero rows (such as if column was all NaN) 
      #also returns model as False if after carveouts there was less than 2 input features
      if df_train_filltrain.shape[0] == 0 or \
      len(self.__column_convert_support(list(df_train_filltrain), postprocess_dict, convert_to='input')) < 2:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : True}
        )
        logger = self.__logger_append(logger)

        # if printstatus is True:
        self.__autoprint('debug', "ML infill model had insufficient features or samples to serve as basis, training halted.")
        self.__autoprint('debug', "This may be a result of automated leakage carveouts from ML_cmnd['leakage_tolerance'].")
        self.__autoprint('debug', '')

      else:

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : False}
        )
        logger = self.__logger_append(logger)
        
        #future extension - Label Smoothing for ML infill
        #(might incorporate this into the training function to be activated by ML_cmnd)
          
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'onehotclassification'
        
        model, postprocess_dict = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict)
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns as text category
          df_traininfill = np.zeros(shape=(1,len(categorylist)))
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns as text category
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
          
      #convert infill values to dataframe (this column labeleling also works for single column case)
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
    
    #if target is a binary encoded categoric set
    if MLinfilltype in {'1010'}:
      
      #edge case if training data has zero rows (such as if column was all NaN) 
      #also returns model as False if after carveouts there was less than 2 input features
      if df_train_filltrain.shape[0] == 0 or \
      len(self.__column_convert_support(list(df_train_filltrain), postprocess_dict, convert_to='input')) < 2:

        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : True}
        )
        logger = self.__logger_append(logger)

        # if printstatus is True:
        self.__autoprint('debug', "ML infill model had insufficient features or samples to serve as basis, training halted.")
        self.__autoprint('debug', "This may be a result of automated leakage carveouts from ML_cmnd['leakage_tolerance'].")
        self.__autoprint('debug', '')

      else:

        logger_dict['validations']['not_enough_samples_or_features_for_MLinfill_result'].update(
          {column : False}
        )
        logger = self.__logger_append(logger)

        #convert from binary to one-hot encoding
        df_train_filllabel = \
        self.__convert_1010_to_onehot(df_train_filllabel)
          
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'onehotclassification'
        
        model, postprocess_dict = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict)

        #this is to support 1010 infill predictions in postmunge
        for entry in categorylist:
          postprocess_dict['column_dict'][entry].update({'_1010_categorylist_proxy_for_postmunge_MLinfill' : list(range(df_train_filllabel.shape[1]))})
        
        #only run following if we have any train rows needing infill

        if df_train_fillfeatures.shape[0] > 0:

          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, list(range(df_train_filllabel.shape[1])))

          df_traininfill = \
          self.__convert_onehot_to_1010(df_traininfill)

        else:
          #this needs to have same number of columns as text category
          df_traininfill = np.zeros(shape=(1,len(categorylist)))
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, list(range(df_train_filllabel.shape[1])))

          df_testinfill = \
          self.__convert_onehot_to_1010(df_testinfill)

        else:
          #this needs to have same number of columns as text category
          df_testinfill = np.zeros(shape=(1,len(categorylist)))

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
      
    #if target category excluded from ML infill:
    if MLinfilltype in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

      #create empty sets for now
      #an extension of this method would be to implement a comparable infill \
      #method for the time category, based on the columns output from the \
      #preprocessing
      df_traininfill = pd.DataFrame({column : [0]}) 
      df_testinfill = pd.DataFrame({column : [0]}) 

      model = False
    
    return df_traininfill, df_testinfill, model, postprocess_dict

  def __createMLinfillsets(self, df_train, df_test, column, trainNArows, testNArows, \
                         category, randomseed, postprocess_dict, ML_cmnd, columnslist = [], \
                         categorylist = []):
    '''
    #update createMLinfillsets as follows:
    #instead of diferientiation by category, do a test for whether categorylist = []
    #if so do a single column transform excluding those other columns from columnslist
    #in the sets comparable to , otherwise do a transform comparable to text category
    #createMLinfillsets(df_train, df_test, column, trainNArows, testNArows, \
    #category, columnslist = []) function that when fed dataframes of train and\
    #test sets, column id, df of True/False corresponding to rows from original \
    #sets with missing values, a string category of 'text', 'date', 'nmbr', or \
    #'bnry', and a list of column id's for the text category if applicable. The \
    #function returns a seris of dataframes which can be applied to training a \
    #machine learning model to predict apppropriate infill values for those points \
    #that had missing values from the original sets, indlucing returns of \
    #df_train_filltrain, df_train_filllabel, df_train_fillfeatures, \
    #and df_test_fillfeatures
    '''

    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #create 3 new dataframes for each train column - the train and labels \
    #for rows not needing infill, and the features for rows needing infill \
    #also create a test features column 
    
    if MLinfilltype in {'numeric', 'singlct', 'integer', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

      #if this is a single column set or concurrent_act
      #note that in edge cases multirt and 1010 may have len(categorylist) == 1
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][category]['MLinfilltype'] in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

        #create copy of df_train to serve as training set for fill
        df_train_filltrain = df_train.copy()
        #now delete rows coresponding to True
        df_train_filltrain = df_train_filltrain.iloc[(trainNArows[trainNArows.columns[0]] == False).to_numpy()]

        #now delete columns = columnslist
        df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        leakage_set = []
        if column in ML_cmnd['leakage_dict']:
          leakage_set = ML_cmnd['leakage_dict'][column]
          #this drop entries not in dataframe (just included as a precaution)
          #this same derived leakage_set is used again below
          leakage_set = list(set(leakage_set) & set(df_train_filltrain))
          df_train_filltrain = df_train_filltrain.drop(leakage_set, axis=1)

        #create a copy of df_train[column] for fill train labels
        df_train_filllabel = pd.DataFrame(df_train[column].copy())
        #drop rows corresponding to True
        df_train_filllabel = df_train_filllabel.iloc[(trainNArows[trainNArows.columns[0]] == False).to_numpy()]

        #create features df_train for rows needing infill
        #create copy of df_train
        df_train_fillfeatures = df_train.copy()
        #delete rows corresponding to False
        df_train_fillfeatures = df_train_fillfeatures.iloc[(trainNArows[trainNArows.columns[0]] == True).to_numpy()]
        #delete columnslist and column+'_NArows'
        df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        df_train_fillfeatures = df_train_fillfeatures.drop(leakage_set, axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures.iloc[(testNArows[testNArows.columns[0]] == True).to_numpy()]
        #delete column and column+'_NArows'
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        df_test_fillfeatures = df_test_fillfeatures.drop(leakage_set, axis=1)

      #else if categorylist wasn't single entry
      else:

        #create copy of df_train to serve as training set for fill
        df_train_filltrain = df_train.copy()
        #now delete rows coresponding to True
        df_train_filltrain = df_train_filltrain.iloc[(trainNArows[trainNArows.columns[0]] == False).to_numpy()]

        #now delete columns = columnslist
        df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        leakage_set = []
        if column in ML_cmnd['leakage_dict']:
          leakage_set = ML_cmnd['leakage_dict'][column]
          #this drop entries not in dataframe, such as based on other drops like for NArows above
          #this same derived leakage_set is used again below
          leakage_set = list(set(leakage_set) & set(df_train_filltrain))
          df_train_filltrain = df_train_filltrain.drop(leakage_set, axis=1)

        #create a copy of df_train[categorylist] for fill train labels
        df_train_filllabel = df_train[categorylist].copy()
        #drop rows corresponding to True
        df_train_filllabel = df_train_filllabel.iloc[(trainNArows[trainNArows.columns[0]] == False).to_numpy()]

        #create features df_train for rows needing infill
        #create copy of df_train (note it already has NArows included)
        df_train_fillfeatures = df_train.copy()
        #delete rows coresponding to False
        df_train_fillfeatures = df_train_fillfeatures.iloc[(trainNArows[trainNArows.columns[0]] == True).to_numpy()]
        
        #delete columnslist
        df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        df_train_fillfeatures = df_train_fillfeatures.drop(leakage_set, axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures.iloc[(testNArows[testNArows.columns[0]] == True).to_numpy()]
        #delete column
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        df_test_fillfeatures = df_test_fillfeatures.drop(leakage_set, axis=1)

    #elif MLinfilltype not in supported entries:
    else:

      #create empty sets for now
      df_train_filltrain = pd.DataFrame({'foo' : []}) 
      df_train_filllabel = pd.DataFrame({'foo' : []})
      df_train_fillfeatures = pd.DataFrame({'foo' : []})
      df_test_fillfeatures = pd.DataFrame({'foo' : []})
    
    return df_train_filltrain, df_train_filllabel, df_train_fillfeatures, df_test_fillfeatures

  def __insertinfill(self, df, column, infill, category, NArows, postprocess_dict, \
                     columnslist = [], categorylist = [], singlecolumncase = False):
    '''
    #uses the boolean indicators for presence of infill in NArows to apply infill
    #passed in infill dataframe to df[column]
    #note that infill dataframe is multicolumn when categorylist length > 1
    #and singlecolumn case is False
    #singlecolumn case is for special case (used in adjinfill) when we want to 
    #override the categorylist >1 methods
    '''

    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #NArows column name uses original column name + _NArows as key
    NArowcolumn = NArows.columns[0]

    if MLinfilltype in {'numeric', 'singlct', 'integer', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

      #if this is a single column set (not categorical)
      #or a multicolumn set with distinct infill to each column
      #note that in edge cases multirt and 1010 may have len(categorylist) == 1
      #singlecolumncase refers to cases where one column at a time is inserted into multicolumn sets, such as for particular assigninfill scenarios
      if len(categorylist) == 1 or singlecolumncase is True \
      or MLinfilltype in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

        #only perform if any rows needing infill
        if NArows[NArowcolumn].astype(bool).any():

          #infill has number of rows equal to number of infill targets
          #we'll create infillindex to match infill to the target rows for insertion
          infillindex = pd.DataFrame(NArows[NArowcolumn].copy())
      
          infillindex.loc[NArows[NArowcolumn].astype(bool), NArowcolumn] = infill.to_numpy().ravel()
        
          #now carry that infill over to the target column for rows where NArows is True
          df = \
          self.__autowhere(df, column, NArows[NArowcolumn].astype(bool), infillindex[NArowcolumn], specified='replacement')

      #else if categorylist wasn't single value
      else:

        #only perform if any rows needing infill
        if NArows[NArowcolumn].astype(bool).any():

          for textcolumnname in categorylist:
            
            #infill has number of rows equal to number of infill targets
            #we'll create infillindex to match infill to the target rows for insertion
            infillindex = pd.DataFrame(NArows[NArowcolumn].copy())

            infillindex.loc[NArows[NArowcolumn].astype(bool), NArowcolumn] = infill[textcolumnname].to_numpy().ravel()

            #now carry that infill over to the target column for rows where NArows is True
            df = \
            self.__autowhere(df, column, NArows[NArowcolumn].astype(bool), infillindex[NArowcolumn], specified='replacement')

    elif MLinfilltype in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:
      pass

    return df

  def __inspect_ML_cmnd(self, ML_cmnd, autoML_type, MLinfill_alg):
    """
    #Inspects ML_cmnd to determine if any of the parameters passed
    #for regressor or classifier are passed as lists instead of distinct
    #values, in which case they will be evaluated via grid search
    #or in a future extension random search or other hyperparameter tuning methods
    
    #takes as input user-passed ML_cmnd, returns tune_marker
    #where tune_marker = True indicates sets were passed, else False
    
    #autoML_type refers to type of predictive algorithm applied,
    #default is scikit Random Forest via 'randomforest'
    """

    #initialize tune_marker to default
    tune_marker = False
    
    if autoML_type in {'randomforest'}:
    
      if 'MLinfill_cmnd' in ML_cmnd:

        if MLinfill_alg in ML_cmnd['MLinfill_cmnd']:

          for key in ML_cmnd['MLinfill_cmnd'][MLinfill_alg]:

            #if passed parameter is a list, range, or distribution
            if type(ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]) \
            in {type([1]), type(range(1)), type(stats.expon(1))}:

              tune_marker = True
    
    return tune_marker
  
  def __assemble_param_sets(self, ML_cmnd, autoML_type, MLinfill_alg):
    """
    #assembles ML_cmnd passed parameters into two sets
    #for hyoeroparameter tuning operation
    
    #those parameters that were passed as sets 
    #will be saved in tune_params dictionary
    
    #those parameters that were otherwise passed
    #will be saved in static_params dictionary
    
    #returns those two dictionaries tune_params & static_params
    
    #autoML_type refers to type of predictive algorithm applied,
    #defaults to scikit Random Forest via 'default'
    
    #MLinfill_cmnd supports passing parameters to target algorithms
    #e.g. under default supports 'RandomForestRegressor' & 'RandomForestClassifier'
    """

    #initialize returned dictionaries
    static_params = {}
    tune_params = {}
    
    if autoML_type in {'randomforest'}:
      
      if 'MLinfill_cmnd' in ML_cmnd:
        
        if MLinfill_alg in ML_cmnd['MLinfill_cmnd']:
          
          for key in ML_cmnd['MLinfill_cmnd'][MLinfill_alg]:
            
            #if passed parameter is a set
            if type(ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]) \
            in {type([1]), type(range(1)), type(stats.expon(1))}:
              
              #add set to tune_params which will be targeted for grid search
              tune_params.update({key : ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]})
              
            else:
              
              #else add to static_params which will overwrite defaults
              static_params.update({key : ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]})
        
    return static_params, tune_params

  def __optuna_XG1(self, train, labels, valratio, ML_cmnd, randomseed, classify_regress):
    """
    performs a bayesian hyperparameter tuning of XGBoost
    using a validation ratio of valratio
    and a number of iterations and max iterations time based on ML_cmnd['optuna_n_iter'] and ['optuna_timeout']
    which default to 100 and 600. 
    
    To set a gpu for accelerated training can pass a gpu_id as 
    ML_cmnd['xgboost_gpu_id'] = 0
    (if you have one gpu this will probably be integer 0)
    when a gpu is designated the tree_method will automatically apply gpu_hist
    (I think that if a gpu is used for training it will be needed for inference)
    
    classify_regress is one of {'classify', 'regress'}
    
    can be activated for ml infill with XGBoost by ML_cmnd['hyperparam_tuner'] = 'optuna_XGB1'
    
    requires installation of the optuna library
    this function is based on an optuna published tutorial shared on their github
    #https://github.com/optuna/optuna-examples/blob/main/xgboost/xgboost_simple.py
    
    the github doesn't explicitly state license
    since the optuna library is MIT licensed
    I am operating on assumption that their tutorials are similar open source
    
    optuna license information and citation provided in read me

    a tradeoff of using the scikit api is there doesn't appear to be a native cross validation feature in xgboost
    so implemented a custom kfold cross validation 
    that can be activated by passing k count to ML_cmnd['optuna_kfolds'] (defaults to 1)

    early stopping can be activated by passing integer count of max rounds without metric improvement to ML_cmnd['optuna_early_stop']
    (defaults to 50)

    max depth tuning step size defaults to 2
    with higher tuning durations we expect it may be beneficial to decrease the maxdepth step size from 2 to 1, 
    the default to 2 was selected based on an optuna tutorial
    this can be set to 1 with ML_cmnd['optuna_max_depth_tuning_stepsize']
    """

    if classify_regress in {'classify', 'boolean'}:
      from xgboost import XGBClassifier
    elif classify_regress == 'regress':
      from xgboost import XGBRegressor
    
    import optuna

    #dictionaries allow us to set variables with memory between trial runs (a quirk of python)
    global_var_dict = {
                       'metrics_without_improvement_count' : 0,
                       'best_metric':False,
                       'trial_count':0,
                      }

    def objective(trial, global_var_dict):

      kfolds = ML_cmnd['optuna_kfolds']

      param = {
        "use_label_encoder" : False,
        "verbosity": 0,
        "tree_method": "exact",
        # defines booster, gblinear for linear functions.
        "booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]),
        # L2 regularization weight.
        "lambda": trial.suggest_float("lambda", 1e-8, 1.0, log=True),
        # L1 regularization weight.
        "alpha": trial.suggest_float("alpha", 1e-8, 1.0, log=True),
        # sampling ratio for training data.
        "subsample": trial.suggest_float("subsample", 0.2, 1.0),
        # sampling according to each tree.
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.2, 1.0),
      }
      
      if classify_regress == 'boolean':
        param['objective'] = 'binary:logistic'

      if ML_cmnd['xgboost_gpu_id'] is not False:
        param.update({'tree_method'   : 'gpu_hist',
                            'gpu_id'  : ML_cmnd['xgboost_gpu_id'] })

      if param["booster"] in ["gbtree", "dart"]:
        # maximum depth of the tree, signifies complexity of the tree.
        #ML_cmnd['optuna_max_depth_tuning_stepsize'] defaults to 2 based on an optuna demonstration
        #we expect setting to 1 could be beneficial with higher tuning durations
        param["max_depth"] = trial.suggest_int("max_depth", 3, 9, step=ML_cmnd['optuna_max_depth_tuning_stepsize'])
        # minimum child weight, larger the term more conservative the tree.
        param["min_child_weight"] = trial.suggest_int("min_child_weight", 2, 10)
        param["eta"] = trial.suggest_float("eta", 1e-8, 1.0, log=True)
        # defines how selective algorithm is.
        param["gamma"] = trial.suggest_float("gamma", 1e-8, 1.0, log=True)
        param["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"])

      if param["booster"] == "dart":
        param["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"])
        param["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"])
        param["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0, log=True)
        param["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True)

      if classify_regress in {'classify', 'boolean'}:
    
        kfolds_metric_log = []
        
        for i in range(kfolds):
          
          if global_var_dict['trial_count'] % kfolds == i \
          or ML_cmnd['optuna_fasttune'] is False:

            #val_tuple specifies the boundaries of current fold's validation split
            val_tuple = (i/kfolds, (i+1)/kfolds)
            #we'll have a unique random seed for shuffling each fold
            trial_randomseed = random.randint(0,2147483647)

            #if only one fold (the default) will randomly sample based on valratio (currently set as 0.25)
            #we'll apply a unique random seed with each trial
            if kfolds == 1:
              train_x, valid_x = self.__df_split(train, valratio, True, trial_randomseed)
              train_y, valid_y = self.__df_split(labels, valratio, True, trial_randomseed)

            #else for kfolds > 1 we'll extract the folds in sequential chunks
            elif kfolds > 1:
              train_x, valid_x = self.__df_split_specified(train, val_tuple, False, 42)
              train_y, valid_y = self.__df_split_specified(labels, val_tuple, False, 42)

            #there is an edge case where this validation split results in not fully represented range integer classificaiotn labels
            #which is an xgboost quirk
            #since this is just for tuning we'll convert labels, noting that this means we won't be able to use saved model as final
            if classify_regress in {'classify', 'boolean'}:
              #train_y is a pandas series of integers
              unique_set = set(train_y.unique())
              intended_unique_set = set(range(train_y.astype(int).max()))
              if len(intended_unique_set - unique_set) > 0:
                range_conversion_dict = dict(zip(list(unique_set), list(range(len(unique_set)))))
                train_y = train_y.replace(range_conversion_dict).astype(int)
                valid_y = valid_y.replace(range_conversion_dict).astype(int)

            #it's generally good practice to shuffle before training for non-sequential learning
            train_x = self.__df_shuffle(train_x, trial_randomseed, axis=0)
            train_y = self.__df_shuffle(train_y, trial_randomseed, axis=0)

            model = XGBClassifier(**param)

            bst = model.fit(train_x, train_y)
            preds = bst.predict(valid_x)

            #use of f1 as default is per recommendation from Deep Learning with Pytorch by Stevens and Antiga
            metric = f1_score(valid_y, preds, zero_division=0, average='weighted')

            kfolds_metric_log.append(metric)

        #now average metric accross folds
        metric = pd.DataFrame({'kfolds_metric_log':kfolds_metric_log})['kfolds_metric_log'].mean()

        #log values for consideration of early stopping
        if global_var_dict['best_metric'] is False:
          global_var_dict['best_metric'] = metric
        elif global_var_dict['best_metric'] is not False:
          if metric > global_var_dict['best_metric']:
            global_var_dict['best_metric'] = metric
            global_var_dict['metrics_without_improvement_count'] = 0
          else:
            global_var_dict['metrics_without_improvement_count'] += 1
        global_var_dict['trial_count'] += 1

      if classify_regress == 'regress':
    
        kfolds_metric_log = []
        
        for i in range(kfolds):
          
          if global_var_dict['trial_count'] % kfolds == i \
          or ML_cmnd['optuna_fasttune'] is False:
          
            #val_tuple specifies the boundaries of current fold's validation split
            val_tuple = (i/kfolds, (i+1)/kfolds)
            #we'll have a unique random seed for shuffling each fold
            trial_randomseed = random.randint(0,2147483647)

            #if only one fold (the default) will randomly sample based on valratio (currently set as 0.25)
            #we'll apply a unique random seed with each trial
            if kfolds == 1:
              train_x, valid_x = self.__df_split(train, valratio, True, trial_randomseed)
              train_y, valid_y = self.__df_split(labels, valratio, True, trial_randomseed)

            #else for kfolds > 1 we'll extract the folds in sequential chunks
            elif kfolds > 1:
              train_x, valid_x = self.__df_split_specified(train, val_tuple, False, 42)
              train_y, valid_y = self.__df_split_specified(labels, val_tuple, False, 42)

            #it's generally good practice to shuffle before training for non-sequential learning
            train_x = self.__df_shuffle(train_x, trial_randomseed, axis=0)
            train_y = self.__df_shuffle(train_y, trial_randomseed, axis=0)

            model = XGBRegressor(**param)

            bst = model.fit(train_x, train_y)
            preds = bst.predict(valid_x)

            #this handles edge case for an overflow in derived metric
            try:
              metric = mean_squared_error(valid_y, preds)
            except ValueError:
              metric = 3.4E38

            kfolds_metric_log.append(metric)
          
        #now average metric accross folds
        metric = pd.DataFrame({'kfolds_metric_log':kfolds_metric_log})['kfolds_metric_log'].mean()

        #log values for consideration of early stopping
        if global_var_dict['best_metric'] is False:
          global_var_dict['best_metric'] = metric
        elif global_var_dict['best_metric'] is not False:
          if metric < global_var_dict['best_metric']:
            global_var_dict['best_metric'] = metric
            global_var_dict['metrics_without_improvement_count'] = 0
          else:
            global_var_dict['metrics_without_improvement_count'] += 1
        global_var_dict['trial_count'] += 1

      #check for early stopping (note that ML_cmnd['optuna_early_stop'] defaults to np.inf)
      if global_var_dict['metrics_without_improvement_count'] > ML_cmnd['optuna_early_stop']:
        raise EarlyStoppingExceeded()

      return metric

    class EarlyStoppingExceeded(optuna.exceptions.OptunaError):
      pass
    
    #initialized in __check_ML_cmnd when not user defined
    optuna_n_iter = ML_cmnd['optuna_n_iter']
    optuna_timeout = ML_cmnd['optuna_timeout']
    
    if classify_regress in {'classify', 'boolean'}:
      #f1 score is in range 0-1, with 1 being the best score
      study = optuna.create_study(direction="maximize")

    elif classify_regress == 'regress':
      #mean_squared_error is >=0, with 0 being the best score
      study = optuna.create_study(direction="minimize")

    try:
      study.optimize(lambda trial: objective(trial, global_var_dict), n_trials=optuna_n_iter, timeout=optuna_timeout)
    except EarlyStoppingExceeded:
      pass
    
    trial = study.best_trial
    
    tuned_params = trial.params
    
    return tuned_params

  #__FunctionBlock: halting criteria support functions

  def __consolidate_multicolumn(self, df_traininfill, column, MLinfilltype, postprocess_dict):
    """
    #consolidates multicolumn infill representations to a single column of aggregated strings with header column
    #else returns received single column to same column but renamed to header column
    #this multicolumn representation is used to evaluate halting criteria for infilliterate
    """

    if MLinfilltype in {'multirt', '1010'}:
      
      categorylist = list(df_traininfill)
      
      #these have suffix included so won't have overlap with arbitrary column header 'asdf', but just in case
      tempcolumn = 'asdf'
      if tempcolumn in categorylist:
        while tempcolumn in categorylist:
          tempcolumn = tempcolumn + 'z'

      df_traininfill[tempcolumn] = ''
      
      for entry in categorylist:
        
        df_traininfill[tempcolumn] = df_traininfill[tempcolumn] + df_traininfill[entry].astype(str)
        
        del df_traininfill[entry]
        
      #now rename the target column
      df_traininfill.rename(columns = {tempcolumn : column}, inplace = True)
      
    # elif 'infill' in df_traininfill:
        
    #   df_traininfill.rename(columns = {'infill' : column}, inplace = True)
    
    return df_traininfill

  def __populate_halt_dict(self, df_infill_i, df_infill_iplus, MLinfilltype, halt_dict, iteration, column, df_traininfill_rowcount):
    """
    Appends entries to halt_dict associated with the current target column
    """

    if iteration > 0:
      
      #if set is categoric
      if MLinfilltype in {'singlct', 'binary', 'multirt', '1010', 'concurrent_ordl', 'concurrent_act'}:
        
        #this is a count of cases where imputations matched between iterations which signals that iterations are honing in on final form
        sumofinequal = int(pd.DataFrame(df_infill_iplus[column][:df_traininfill_rowcount] != df_infill_i[column][:df_traininfill_rowcount]).sum())
        
        #this is a count of number of imputations associated with this infill
        quantity = df_traininfill_rowcount
        
        ratio = sumofinequal / quantity
        
        categoric_tuple_df_row = pd.DataFrame({'sumofinequal':[sumofinequal], 
                                               'quantity':[quantity],
                                               'ratio':[ratio],
                                               'column':[column]})
        
        #now concat that row onto the categoric_tuple_df stored in halt_dict for this iteration
        halt_dict[iteration]['categoric_tuple_df'] = pd.concat([halt_dict[iteration]['categoric_tuple_df'], categoric_tuple_df_row], axis=0)
        
      #if set is numeric
      if MLinfilltype in {'numeric', 'integer', 'concurrent_nmbr'}:
        
        #this is the max value found in absolute value of deltas between iterations
        meanabsdelta = (df_infill_iplus[column][:df_traininfill_rowcount] - df_infill_i[column][:df_traininfill_rowcount]).abs().mean()
        
        meanabs = df_infill_iplus[column][:df_traininfill_rowcount].abs().mean()
        
        quantity = df_traininfill_rowcount
        
        numeric_tuple_df_row = pd.DataFrame({'meanabsdelta':[meanabsdelta], 
                                             'meanabs':[meanabs],
                                             'quantity':[quantity],
                                             'column':[column]})
        
        #now concat that row onto the numeric_tuple_df stored in halt_dict for this iteration
        halt_dict[iteration]['numeric_tuple_df'] = pd.concat([halt_dict[iteration]['numeric_tuple_df'], numeric_tuple_df_row], axis=0)
    
    return halt_dict

  def __calc_stop_result(self, halt_dict, iteration, ML_cmnd):
    """
    #calculates a result for stopping criteria
    #note that the numeric criteria was partly inspired by review of scikit-learn iterativeimputer stopping criteria
    #note that the categoric stopping criteria was partly inspired by review of MissForest stopping criteria
    #although there are some fundamental differences in place for each
    #receives tolerances in ML_cmnd as ML_cmnd['numeric_tol'] and ML_cmnd['categoric_tol']
    #and when these entries are not populated defaults to numeric_tol = 0.03, categoric_tol = 0.05
    #(these defaults are for the moment somewhat arbitrary and may be further refined in future update)
    
    #Please note that the stop_result is only returned as True 
    #if early stopping was activated by ML_cmnd['halt_iterate'] = True
    """

    #initialize
    stop_result = False
    
    if 'categoric_tol' in ML_cmnd:
      categoric_tol = ML_cmnd['categoric_tol']
    else:
      categoric_tol = 0.05
      
    if 'numeric_tol' in ML_cmnd:
      numeric_tol = ML_cmnd['numeric_tol']
    else:
      numeric_tol = 0.03
    
    if iteration > 0:
      
      #_(1)_
      
      #first calculate categoric result based on aggregation of all categoric imputations
      
      categoric_tuple_df = halt_dict[iteration]['categoric_tuple_df']

      if categoric_tuple_df.shape[0] == 0:
        categoric_result = True
      else:
        
        #result is based on whether sum(sumofinequal)/sum(quantity) < categoric_tol
        categoric_result = False
        
        sum_sumofinequal = categoric_tuple_df['sumofinequal'].sum()
        sum_quantity = categoric_tuple_df['quantity'].sum()

        if sum_sumofinequal / sum_quantity < categoric_tol:
          
          categoric_result = True

        # #validation printouts
        # print("categoric: sum_sumofinequal / sum_quantity")
        # print(sum_sumofinequal / sum_quantity)
        # print("categoric_tol = ", categoric_tol)
        # print("categoric_result = ", categoric_result)
        # print()
          
      #now calculate numeric result based on aggregation of all numeric imputations
      
      numeric_tuple_df = halt_dict[iteration]['numeric_tuple_df']
      
      if numeric_tuple_df.shape[0] == 0:
        numeric_result = True
      else:
        
        #result is based on weighted average of ratio meanabsdelta/meanabs < numeric_tol
        #where we'll derive quantity_times_ratio_sum and quantity_sum
        #and halt when quantity_times_ratio_sum / quantity_sum < numeric_tol
        
        #(weighted by quantity of imputations associated with the ratio)
        numeric_result = False
        
        #quantity_times_ratio_sum = quantity * ratio
        #ratio = meanabsdelta / meanabs
        
        #this is for an edge case where inferred infill is all 0, would interfere with division
        numeric_tuple_df = numeric_tuple_df.loc[numeric_tuple_df['meanabs'] != 0]

        quantity_times_ratio_sum = (numeric_tuple_df['quantity'] * \
                                   numeric_tuple_df['meanabsdelta'] / \
                                   numeric_tuple_df['meanabs']).sum()
        
        #for consistency of quantity basis we'll derive quantity_sum after extracting all zero infill edge case
        quantity_sum = numeric_tuple_df['quantity'].sum()
          
        if quantity_sum == 0:
          numeric_result = True
          
        elif quantity_times_ratio_sum / quantity_sum < numeric_tol:
          numeric_result = True

        # #validation printouts
        # print("numeric: quantity_times_ratio_sum / quantity_sum")
        # print(quantity_times_ratio_sum / quantity_sum)
        # print("numeric_tol = ", numeric_tol)
        # print("numeric_result = ", numeric_result)
        # print()

      #_(1)_
      stop_result = False
      
      if numeric_result is True and categoric_result is True:
        
        stop_result = True
          
      halt_dict[iteration]['numeric_result'] = numeric_result
      halt_dict[iteration]['categoric_result'] = categoric_result
      halt_dict[iteration]['stop_result'] = stop_result
      
      #check if user activated halt_iterate in ML_cmnd (early stopping for infilliterate)
      if 'halt_iterate' in ML_cmnd and ML_cmnd['halt_iterate'] is True:
        pass
      else:
        #stop_result recorded in halt_dict as evlauated but returned as False
        stop_result = False

    return stop_result, halt_dict

  #__FunctionBlock: ML infill leakage support

  def __check_for_leakage(self, ML_cmnd, postprocess_dict, masterNArows_train, postprocess_assigninfill_dict):
    """
    compare aggregated NArw activations from a target feature in a train set 
    to the surrounding features in a train set 
    and for cases where separate features share a high correlation of missing data 
    based on the shown formula
    ((Narw1 + Narw2) == 2).sum() / NArw1.sum() > tolerance
    we exclude those surrounding features from the imputation model basis for the target feature
    
    target features are those features which will have any returned columns serving as ML infill target
    
    the results are appended to any entries in user specified ML_cmnd['leakage_dict']
    for unidirectional exclusion in associated ML infill bases
    
    accepts parameters ML_cmnd, postprocess_dict, masterNArows_train, postprocess_assigninfill_dict
    each as in form after application of transformations and before infill
    and where MLinfill_targets is a list of targets in returned header convention for MLinfill imputation to be accessed from postprocess_assigninfill_dict
    
    note that this function is to be applied prior to _convert_leakage_dict which is prior to _convert_leakage_sets 
    
    note that masterNArows_train will have headers derived as origcolumn+'_NArows'
    and ML_cmnd['leakage_dict'] takes entries in form of {feature1 : {feature2}}
    where feature2 entries are exlcuded from feature1 basis
    
    note this operation can be deactivated by passing ML_cmnd['leakage_tolerance'] = 1 (or False)
    for increased sensitivity, set a lower leakage_tolerance threshold, accepts floats in range 0-1 inclusive
    """

    if 'leakage_tolerance' in ML_cmnd:
      leakage_tolerance = ML_cmnd['leakage_tolerance']
    else:
      leakage_tolerance = 0.85
      
    leakage_dict_derived = {}
      
    #evaluation not performed when leakage_tolerance == 1 or leakage_tolerance is False
    if leakage_tolerance < 1 and leakage_tolerance is not False:
      
      NArows_columns = list(masterNArows_train.columns)
      
      MLinfill_targets = postprocess_assigninfill_dict['MLinfill']
      
      #convert MLinfill_targets to input header convention
      MLinfill_targets = \
      self.__column_convert_support(MLinfill_targets, postprocess_dict, convert_to='input')
      
      #as received this includes label sets
      origcolumns = self.__autocopy(list(postprocess_dict['origcolumn']))
    
      #remove the label entry
      labelcolumn = False
      for entry in list(postprocess_dict['origcolumn']):
        if postprocess_dict['origcolumn'][entry]['type'] == 'label':
          origcolumns.remove(entry)

      #for input column header
      for targetcolumn in MLinfill_targets:
        
        targetcolumn_NArow = targetcolumn + '_NArows'

        for othercolumn in origcolumns:

          if othercolumn != targetcolumn:
            
            if targetcolumn not in leakage_dict_derived \
            or targetcolumn in leakage_dict_derived \
            and othercolumn not in leakage_dict_derived[targetcolumn]:

              othercolumn_NArow = othercolumn + '_NArows'

              #checking for ((Narw1 + Narw2) == 2).sum() / NArw1.sum() > tolerance
              temp_df = pd.DataFrame(masterNArows_train[targetcolumn_NArow].astype(int) + masterNArows_train[othercolumn_NArow].astype(int))

              numerator = temp_df[temp_df[list(temp_df)[0]] == 2].shape[0]
              denominator = masterNArows_train[masterNArows_train[targetcolumn_NArow] == True].shape[0]

              if denominator > 0:
                othercolumn_result = numerator / denominator
              else:
                othercolumn_result = 0

              #othercolumn_result will be a float potentially in range between 0 and 1 inclusive
              if othercolumn_result > leakage_tolerance:

                if targetcolumn not in leakage_dict_derived:
                  leakage_dict_derived.update({targetcolumn : set()})

                leakage_dict_derived[targetcolumn] = \
                leakage_dict_derived[targetcolumn] | {othercolumn}

    ML_cmnd.update({'leakage_dict_derived' : leakage_dict_derived})

    return ML_cmnd

  def __append_full_exclude(self, ML_cmnd, postprocess_dict):
    """
    #columns of MLinfilltype totalexclude
    #are appended onto list populated in ML_cmnd['full_exclude']
    #which has effect of excluding them from ML infill basis of other features
    """

    if 'full_exclude' in ML_cmnd:
      full_exclude = ML_cmnd['full_exclude']
    else:
      full_exclude = []
    
    excludemarker = False
    for column_dict_entry in postprocess_dict['column_dict']:
      
      treecategory = postprocess_dict['column_dict'][column_dict_entry]['category']
      MLinfilltype = postprocess_dict['process_dict'][treecategory]['MLinfilltype']
      
      if MLinfilltype == 'totalexclude':
        excludemarker = True
        full_exclude.append(column_dict_entry)
        
    if excludemarker is True:
      
      ML_cmnd['full_exclude'] = full_exclude
      
    return ML_cmnd

  def __convert_leakage_dict(self, ML_cmnd, postprocess_dict):
    """
    leakage_dict accepts entries in the form {feature1 : {feature2}}
    where entries to the feature2 set are unidirectionally excluded from feature1 basis for ML infill
    accepts features as column headers in either the input or returned convention with suffix
    (and potentially mixed between the two)
    
    _convert_leakage_dict is for converting leakage_dict to a single convention of returned header
    for both the keys and the value sets

    and includes the step of incorporating full_exclude columns, which are excluded from every other
    feature's ML infill basis
    
    it returns the received leakage_dict as leakage_dict_orig
    and a newly populated leakage_dict as leakage_dict
    which includes entries converted to returned column convention
    including a combination of those originating from leakage_dict_orig or leakage_dict_derived
    
    note this function is applied after _check_for_leakage
    note that after this consolidation an additional consolidation is conducted in _convert_leakage_sets
    to incorporate entries associated with leakage_sets
    
    (leakage_sets are for specifying bidirectional exclusions, leakage_dict is for specifying unidirectional, 
    the _check_for_leakage function identifies unidirectional and returns as leakage_dict_derived
    leakage_dict is converted to the final consolidated form inspected in _createMLinfillsets
    which includes specifications for both unidirectional and birdirectional)
    
    note that a final bit of leakage_dict prep is later performed in _convert_leakage_sets to scrub columnslist entries
    """

    if 'leakage_dict' in ML_cmnd:
      leakage_dict_orig = ML_cmnd['leakage_dict']
    else:
      ML_cmnd.update({'leakage_dict':{}})
      leakage_dict_orig = {}
    ML_cmnd.update({'leakage_dict_orig' : self.__autocopy(leakage_dict_orig)})
      
    if 'leakage_dict_derived' in ML_cmnd:
      leakage_dict_derived = ML_cmnd['leakage_dict_derived']
    else:
      leakage_dict_derived = {}

    #before extracting full_exclude columns
    #we'll add any columns returned from categories with MLinfilltype totalexclude to full_exclude
    ML_cmnd = self.__append_full_exclude(ML_cmnd, postprocess_dict)
      
    if 'full_exclude' in ML_cmnd:
      full_exclude = ML_cmnd['full_exclude']
    else:
      full_exclude = []
      
    #now for any full_exclude sets we'll populate for each column
    i=0
    for full_exclude_column in full_exclude:
      for origcolumn in postprocess_dict['origcolumn']:
        if i==0 and origcolumn not in leakage_dict_orig:
          leakage_dict_orig.update({origcolumn : {full_exclude_column}})
        else:
          leakage_dict_orig[origcolumn] = leakage_dict_orig[origcolumn] | {full_exclude_column}
      i+=1
    
    #we'll populate our consolidated entries in returned column convention in leakage_dict_converted
    leakage_dict_converted = {}
    
    #first we'll convert the leakage_dict_orig keys to returned convention and populate in our empty leakage_dict_converted
    for key in leakage_dict_orig:
      
      if key in postprocess_dict['origcolumn']:
        
        key_returned_list = postprocess_dict['origcolumn'][key]['columnkeylist']
        
        for key_returned in key_returned_list:
          
          leakage_dict_converted.update({key_returned : leakage_dict_orig[key]})
          
      elif key in postprocess_dict['column_dict']:
        
        leakage_dict_converted.update({key : leakage_dict_orig[key]})

      else:
        #scenario where leakage_dict included an incorrectly specified key not found in set
        pass
      
    #next we add keys and values associated with leakage_dict_derived which will be received in inputcolumn form
    for key in leakage_dict_derived:
      
      key_returned_list = postprocess_dict['origcolumn'][key]['columnkeylist']
      
      for key_returned in key_returned_list:
        
        if key_returned not in leakage_dict_converted:
          
          leakage_dict_converted.update({key_returned : leakage_dict_derived[key]})
          
        else:
          
          leakage_dict_converted[key_returned] = \
          leakage_dict_converted[key_returned] | leakage_dict_derived[key]
          
    #then we'll translate all leakage_dict_converted values to returned convention
    for key in leakage_dict_converted:
      
      translatedcolumns_list = \
      self.__column_convert_support(list(leakage_dict_converted[key]), postprocess_dict, convert_to='returned')
      
      leakage_dict_converted[key] = set(translatedcolumns_list)
    
    #the consolidated leakage_dict in returned column convention is returned as ML_cmnd['leakage_dict']
    ML_cmnd['leakage_dict'] = leakage_dict_converted
      
    return ML_cmnd

  def __convert_leakage_sets(self, ML_cmnd, postprocess_dict):
    """
    ML_cmnd accepts entries to ML_cmnd['leakage_sets'] as a list of input columns or a list of list of input columns
    each list of columns as can be considered an individual "leakage_set"
    (refering to them as a set even thought populated as lists, just going to go with it)
    leakage_sets are for specifying bidirectional ML infill basis exclusions

    note user can also pass column headers in leakage_sets as returned column headers to only exclude specific derived features
    in other words, leakage_set entries can be in either of ort mixed between input and returned header convention
    
    leakage sets are for purposes of specifying features that are to be excluded from each other's ML infill basis
    _convert_leakage_sets is for purposes of converting the received form into a more useful data structure
    mapping columns in the returned form with suffix appenders
    by converting to form of leakage_dict
    which will be received already populated with any entries from user specification or derived based on leakage tolerance
    
    leakage_dict = \
    {returnedcolumn : {set of returned columns to exclude from the key's basis}}
    
    note that we will only map returned columns not already included in the key returnedcolumn's columnslist
    as columnslist entries are already excluded from each other's ML infill basis
    
    returns ML_cmnd with edited entry ML_cmnd['leakage_dict']
    
    note this function is to be applied after transformations and before infill 
    so we'll have access to column_dict in postprocess_dict
    """

    #access leakage_sets from ML_cmnd
    if 'leakage_sets' in ML_cmnd:
      leakage_sets = ML_cmnd['leakage_sets']
    else:
      leakage_sets = []

    #access leakage_sets from ML_cmnd
    if 'leakage_dict' in ML_cmnd:
      leakage_dict = ML_cmnd['leakage_dict']
    else:
      leakage_dict = {}
    
    #if only a one tier set, embed in a list for common form
    if len(leakage_sets) > 0 and not isinstance(leakage_sets[0], list) or leakage_sets == []:
      leakage_sets = [leakage_sets]
    
    #convert leakage sets to equivent but with returned column header convention
    leakage_sets_returned = []
    for leakage_set in leakage_sets:
      
      #convert to an equivalent list of associated returned headers with suffix appenders
      #this will likely increase the number of entries
      leakage_set_returned = \
      self.__column_convert_support(leakage_set, postprocess_dict, convert_to='returned')
      
      leakage_sets_returned.append(leakage_set_returned)
      
    #now populate leakage_dict
    for leakage_set_returned in leakage_sets_returned:
      
      for leakage_set_returned_entry_1 in leakage_set_returned:
        
        if leakage_set_returned_entry_1 not in leakage_dict:
          
          leakage_dict.update({leakage_set_returned_entry_1 : set()})
          
        #we'll add any entries from leakage_set_returned not already populated
        leakage_dict[leakage_set_returned_entry_1] = \
        leakage_dict[leakage_set_returned_entry_1] | set(leakage_set_returned)
        
    #now scrub columnslist entries for each key of leakage_dict (columnslists are handled seperately in _createMLinfillsets)
    for leakage_dict_key in leakage_dict:
      leakage_dict[leakage_dict_key] = \
      leakage_dict[leakage_dict_key] - set(postprocess_dict['column_dict'][leakage_dict_key]['columnslist'])
      
    #great leakage_dict is now populated, return in ML_cmnd
    ML_cmnd.update({'leakage_dict' : leakage_dict})

    return ML_cmnd

  #__FunctionBlock: ML infill stochastic impute support

  def __stochastic_impute(self, ML_cmnd, df, column, postprocess_dict, df_train=False):
    """
    Master function for stochastic_impute
    Which may be applied based on either ML_cmnd['stochastic_impute_categoric'] or ML_cmnd['stochastic_impute_numeric']
    calls one of support functions _stochastic_impute_categoric or _stochastic_impute_numeric based on the column's MLinfilltype
    
    ML_cmnd is user passed dictionary to specify any parameter deviations from default and activate stochastic_impute
    column is one of the entries in target categorylist (this will only applied to each categorylist once)
    df is either df_traininfill or df_testinfill
    df_train is df_train as passed to infill with suffix appenders and only need be passed for train data in automunge
    """

    category = postprocess_dict['column_dict'][column]['category']
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']
    
    #if target is numeric (currently stochastic impute not supported for integer MLinfilltype)
    if MLinfilltype in {'numeric', 'concurrent_nmbr', 'integer'}:
      
      if 'stochastic_impute_numeric' in ML_cmnd and ML_cmnd['stochastic_impute_numeric'] is True:
        
        #test data case
        if df_train is False:
          maximum=False
          minimum=False
        #train data case
        else:
          maximum = df_train[column].max()
          minimum = df_train[column].min()
        
        df, postprocess_dict = \
        self.__stochastic_impute_numeric(ML_cmnd, df, column, postprocess_dict, maximum=maximum, minimum=minimum)
      
    #if target is categoric
    if MLinfilltype in {'singlct', 'binary', 'multirt', '1010', 'concurrent_ordl', 'concurrent_act'}:
      
      if 'stochastic_impute_categoric' in ML_cmnd and ML_cmnd['stochastic_impute_categoric'] is True:
      
        #train data case
        if df_train is not False:
          df_train = df_train[categorylist]
        
        df, postprocess_dict = \
        self.__stochastic_impute_categoric(ML_cmnd, df, column, postprocess_dict, df_unique=df_train)

    return df, postprocess_dict
    
  def __stochastic_impute_categoric(self, ML_cmnd, df, targetcolumn, postprocess_dict, df_unique=False):
    """
    Injects some stochasticity into categoric imputations derived from ML infill
    df is df_traininfill for train data or df_testinfill for test data
    which has a number of rows corrresponding to the number of imputations derived for this feature
    note that df may have one or more columns of imputations
    targetcolumn is one of entries from the categorylist (each categorylist will only be accessed once)
    for target feature in returned automunge data with suffix where imputations will be injected
    
    df_unique is needed for train data and should be passed as a copy of the train set feature set before imputations
    including all columns from the categorylist, as derived from the set df_train
    which we will internally have redundant rows consolidated (i.e. so each unique set of values between columns represented once)
    Note that df_unique needs to be passed for train data in automunge
    And for test data in automunge or postmunge we'll access values from postprocess_dict
    
    The noise injection will be conducted by selected a subset of df rows to target noise by flip_prob ratio
    and replace those target rows with a randomly sampled row from df_unique
   
    We will record any derived properties from automunge in a postprocess_dict entry 'stochastic_imputation_dict'
    To ensure a consistent basis applied in postmunge imputations, such as a consistent df_unique, as
    postprocess_dict['stochastic_imputation_dict'] = \
    {targetcolumn : {'type' : 'categoric',
                     'flip_prob' : flip_prob,
                     'df_unique' : df_unique,
                     }}
    
    Noise will default to a 0.03 injection ratio (flip_prob)
    Note that these defaults can be specified to deviate from the shown defaults as
    ML_cmnd['stochastic_impute_categoric_flip_prob'] = 0.03
    The replacement activation set in cases of noise injection
    Will be randomly drawn from a uniform problaility of one of the unique acitvation sets in df_train
    Note that this includes possibility of replacement with the same activation set based on random draw
    
    Note that noise sampling from distributions is supported by numpy.random
    
    Note that in some cases, such as I believe for ordinal data, df will have single column with header 'infill'
    And in other cases df will have zero, one or more columns with headers corresponding to categorylist entries
    This function will return df with consistent headers as received
    
    Note that noise injections will only be applied when user passes ML_cmnd['stochastic_impute_categoric'] = True

    Note that if the received encoding had a default infill based on a distinct activation set, 
    that set will be included in set of unique activaiton sets from df_unique
    """

    #(inspected in _stochastic_impute)
#     #Note that noise injections will only be applied when user passes ML_cmnd['stochastic_impute_numeric'] = True
#     if 'stochastic_impute_categoric' in ML_cmnd and ML_cmnd['stochastic_impute_categoric'] is True:     
    
    if 'stochastic_imputation_dict' not in postprocess_dict:
      postprocess_dict.update({'stochastic_imputation_dict' : {}})

    #test data case
    if targetcolumn in postprocess_dict['stochastic_imputation_dict']:

      #access parameter from entries recorded with train data
      flip_prob = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['flip_prob']
      df_unique = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['df_unique']
      weighted = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['weighted']
      weights = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['weights']

    #train data case (in train data case user needs to pass entries for df_unique)
    elif targetcolumn not in postprocess_dict['stochastic_imputation_dict']:

      if 'stochastic_impute_categoric_flip_prob' in ML_cmnd:
        flip_prob = ML_cmnd['stochastic_impute_categoric_flip_prob']
      else:
        flip_prob = 0.03

      if 'stochastic_impute_categoric_weighted' in ML_cmnd:
        weighted = ML_cmnd['stochastic_impute_categoric_weighted']
      else:
        weighted = True
        
      if weighted is True:
        #for weighted we'll create a temporary backup of df_unique used to derive weights
        df_unique_backup = df_unique.copy()

      #now consolidate redundant rows in df_unique before saving to stochastic_imputation_dict
      
      #first we derive a mask based on presence of duplicate rows
      mask = pd.DataFrame(df_unique.duplicated())

      #this operation inverts True and False in the mask for next operation
      mask = pd.Series(mask[list(mask)[0]].astype(int) - 1).abs().astype(bool)

      #now apply the mask to consolidate duplicate rows, this returns a dataframe with all unique rows
      #which will likely be much fewer rows than received df_unique 
      #since this is not applied to high cardinality sets (like hashing) based on their MLinfilltype
      df_unique = df_unique.iloc[mask.to_numpy()]

      #reset index in df_unique to a range index
      df_unique = df_unique.reset_index(drop=True)
      
      weights = []
      if weighted is True:
        #for each unique row in the consolidated df_unique calculate a weight
        #by count of that row in df_unique_backup
        #note this is applied in same order as unique_range fed to np.random.choice
        for i in range(df_unique.shape[0]):
          #to derive, we'll create a comparably shaped dataframe with all same row for comparison to df_unique_backup
          df_allonerow = pd.concat([df_unique[i:i+1]]*df_unique_backup.shape[0], axis=0)
          df_allonerow.index = df_unique_backup.index
          count = ((df_unique_backup == df_allonerow).sum(axis=1) == df_unique_backup.shape[1]).sum()
          weight = count / df_unique_backup.shape[0]
          weights.append(weight)
        del df_unique_backup

      #store parameters for use with test data
      postprocess_dict['stochastic_imputation_dict'].update({
        targetcolumn : {
          'type' : 'categoric',
          'flip_prob' : flip_prob,
          'df_unique' : df_unique,
          'weighted' : weighted,
          'weights' : weights,
        }
      })

    #now proceed with noise injections

    #for categoric imputations df may have zero, one, or more columns
    if len(list(df)) > 0 \
    and len(list(df)) == len(list(df_unique)) \
    and df.shape[0] > 0:

      #df later reverted back to orig_df_columns
      orig_df_columns = list(df)

      #I think df already has range index, this is just in case, later reverted back to orig_df_index
      orig_df_index = df.index
      df = df.reset_index(drop=True)

      #confirm that column headers of df are consistent to df_unique
      #I don't think they will be in different order but will check just in case
      #(it is ok if they are in different order as long as headers are same as per set comparison)
      #note we already confirmed number of columns is consistent in preceding if statement
      if list(df.columns) != list(df_unique.columns) and set(df.columns) != set(df_unique.columns):
        df.columns = df_unique.columns

      #we'll have support columns of DPod_column_temp1 = 1 and DPod_column_temp2 = 2
      #and returned set will delete support columns and retain original received column naming conventions
      #(using DPod naming as a shortcut so can repurpose code associated with DPod)
      #don't have to worry about suffix overlap since columns in df will be strings
      DPod_tempcolumn1 = 1
      DPod_tempcolumn2 = 2

      unique_count = df_unique.shape[0]
      unique_range = list(range(unique_count))

      #first we'll derive our sampled noise for injection

      #DPod_tempcolumn1 will return 1 for rows receiving injection and 0 elsewhere
      df[DPod_tempcolumn1] = pd.DataFrame(np.random.binomial(n=1, p=flip_prob, size=(df.shape[0])), index=df.index)
      
      if weighted is False:
        #DPod_tempcolumn2 will return a uniform random draw of integer sampled from unique_range for each row
        df[DPod_tempcolumn2] = pd.DataFrame(np.random.choice(unique_range, size=(df.shape[0])), index=df.index)
      elif weighted is True:
        df[DPod_tempcolumn2] = pd.DataFrame(np.random.choice(unique_range, p=weights, size=(df.shape[0])), index=df.index)

      #now we'll populate another dataframe df_unique2 with index translated 
      #so that rows are in order of sampled index numbers in df[DPod_tempcolumn2]
      #this also results in a number of rows matching df
      df_unique2 = df_unique.iloc[df[DPod_tempcolumn2]]
      #and then reset that index to range
      df_unique2 = df_unique2.reset_index(drop=True)

      #now we can carry the values from df_unique2 to replace entries in df associated with noise injection
      #here the columns in df_unique2 match the columns in df except df_unique2 does not include the tempcolumns
      for column in df_unique2:

        #setting to dataframe for single column case
        df = \
        self.__autowhere(df, column, df[DPod_tempcolumn1] == 1, df_unique2[column], specified='replacement')

      #delete support columns
      del df[DPod_tempcolumn1]
      del df[DPod_tempcolumn2]

      #recover columns and index to df in case they were renamed (such as may have taken place in a single column case)
      df.columns = orig_df_columns
      #recover index (not sure if this is neccesary just seems good practice)
      df.index = orig_df_index
      
    #returned data now has a randomly drawn activation set injected to a subset of imputations (per flip_prob ratio)
    return df, postprocess_dict

  def __stochastic_impute_numeric(self, ML_cmnd, df, targetcolumn, postprocess_dict, maximum=False, minimum=False):
    """
    Injects some stochasticity into numeric imputations derived from ML infill
    df is df_traininfill for train data or df_testinfill for test data
    which has a number of rows corrresponding to the number of imputations derived for this feature
    Note that maximum and minimum need to be specified for train data in automunge
    And for test data in automunge or postmunge we'll access values from postprocess_dict
    targetcolumn is the target feature in returned automunge data with suffix where imputations will be injected
    
    Since we only know that data is numeric and not what if any form of normalization
    We will receive as input max and min values of the training feature set corresponding to imputations
    And use that as boundaries for returned imputations with noise
    Sort of similar to noise injections in DPmm transform
    
    We will record any derived properties from automunge in a postprocess_dict entry 'stochastic_imputation_dict'
    To ensure a consistent basis applied in postmunge imputations, such as a consistent max/min, as
    postprocess_dict['stochastic_imputation_dict'] = \
    {targetcolumn : {'type' : 'numeric',
                     'maximum' : maximum, 
                     'minimum' : minimum, 
                     'mu' : mu,
                     'sigma' : sigma,
                     'flip_prob' : flip_prob,
                     'noisedistribution' : noisedistribution,
                     }}
    
    Noise will default to mean of 0 and scale of 0.03 and gaussian distribution with a 0.1 injection ratio
    And the noise will be injected to a min-max scaled representation of the imputations
    Inspired by DPmm, derived noise is capped at +/- midpoint of received max/min
    And imputations are converted to a min/max representation to apply similar formula to DPmm for scaled noise to avoid out of range
    And then inverted back to original representation resulting in noise scaling comensorate with original range
    Note that noise distribution may be sampled from the default of normal or laplace distributions with otherwise consistent parameter inputs
    
    Note that these defaults can be specified to deviate from the shown defaults as
    ML_cmnd['stochastic_impute_numeric_mu'] = 0
    ML_cmnd['stochastic_impute_numeric_sigma'] = 0.03
    ML_cmnd['stochastic_impute_numeric_flip_prob'] = 0.06
    ML_cmnd['stochastic_impute_numeric_noisedistribution'] = 'normal' (also accepts 'laplace', 'abs_normal', 'negabs_normal', 'abs_laplace', 'negabs_laplace')
    
    Note that noise injections will only be applied when user passes ML_cmnd['stochastic_impute_numeric'] = True
    
    Note that noise sampling from distributions is supported by numpy.random
    """

    #(inspected in _stochastic_impute)
#     #Note that noise injections will only be applied when user passes ML_cmnd['stochastic_impute_numeric'] = True
#     if 'stochastic_impute_numeric' in ML_cmnd and ML_cmnd['stochastic_impute_numeric'] is True:      

    if 'stochastic_imputation_dict' not in postprocess_dict:
      postprocess_dict.update({'stochastic_imputation_dict' : {}})

    MLinfilltype = postprocess_dict['process_dict'][postprocess_dict['column_dict'][targetcolumn]['category']]['MLinfilltype']

    #test data case
    if targetcolumn in postprocess_dict['stochastic_imputation_dict']:

      #access parameter from entries recorded with train data
      maximum = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['maximum']
      minimum = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['minimum']
      mu = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['mu']
      sigma = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['sigma']
      flip_prob = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['flip_prob']
      noisedistribution = postprocess_dict['stochastic_imputation_dict'][targetcolumn]['noisedistribution']

    #train data case (in train data case user needs to pass entries for maximum and minimum)
    elif targetcolumn not in postprocess_dict['stochastic_imputation_dict']:

      #access parameters from ML_cmnd if specified, else apply defaults
      if 'stochastic_impute_numeric_mu' in ML_cmnd:
        mu = ML_cmnd['stochastic_impute_numeric_mu']
      else:
        mu = 0

      if 'stochastic_impute_numeric_sigma' in ML_cmnd:
        sigma = ML_cmnd['stochastic_impute_numeric_sigma']
      else:
        sigma = 0.03

      if 'stochastic_impute_numeric_flip_prob' in ML_cmnd:
        flip_prob = ML_cmnd['stochastic_impute_numeric_flip_prob']
      else:
        flip_prob = 0.06

      if 'stochastic_impute_numeric_noisedistribution' in ML_cmnd:
        noisedistribution = ML_cmnd['stochastic_impute_numeric_noisedistribution']
      else:
        noisedistribution = 'laplace'

      #store parameters for use with test data
      postprocess_dict['stochastic_imputation_dict'].update({
        targetcolumn : {
          'type' : 'numeric',
          'maximum' : maximum, 
          'minimum' : minimum, 
          'mu' : mu,
          'sigma' : sigma,
          'flip_prob' : flip_prob,
          'noisedistribution' : noisedistribution,
        }
      })

    #now proceed with noise injections

    max_minus_min = maximum - minimum

    #for numeric imputations df will have one column
    if len(list(df)) > 0 \
    and df.shape[0] > 0 \
    and max_minus_min == max_minus_min \
    and max_minus_min != 0:

      #for numeric injections df will only have one column (which should be header 'infill' based on current convention)
      #note this is also true for infill to concurrent_nmbr MLinfilltype
      column = list(df)[0]

      #convert imputations to a min/max representation to be consistent with noise range
      df[column] = (df[column] - minimum) / max_minus_min

      #derive noise

      #we'll have support columns of DPmm_column = column + 'noise' and DPmm_column_temp1 = column + 'tmp1'
      #and returned set will delete column and DPmm_column_temp1 and rename DPmm_column to column
      #(using DPmm naming as a shortcut so can repurpose code associated with DPmm)
      #don't have to worry about suffix overlap since df_traininfill only has one column (even in concurrent_nmbr case)
      DPmm_column = column + 'noise'
      DPmm_column_temp1 = column + 'tmp1'

      #first we'll derive our sampled noise for injection (df.shape[0] corresponds to number of imputations)

      #the default will sample noise from a normal distribution, 
      #this will return both + and - values centered to mu and scaled to sigma
      if noisedistribution in {'normal', 'absnormal', 'negabsnormal'}:
        normal_samples = np.random.normal(loc=mu, scale=sigma, size=(df.shape[0]))
      #or if user specified laplace to ML_cmnd['stochastic_impute_numeric_noisedistribution']
      #(laplace has a higher prevalence of outliers, remember seeing discussions somewhere that may be preferred for differential privacy for instance)
      elif noisedistribution in {'laplace', 'abslaplace', 'negabslaplace'}:
        normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(df.shape[0]))

      if noisedistribution in {'absnormal', 'abslaplace'}:
        normal_samples = abs(normal_samples)
      if noisedistribution in {'negabsnormal', 'negabslaplace'}:
        normal_samples = (-1) * abs(normal_samples)

      #binomial samples are returned as 1 for rows of imputation set to be targets for injections, else 0
      binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(df.shape[0]))

      #we then combine the two, resulting in a zero value for entries without injection and a noise value elsewhere
      df[DPmm_column] = pd.DataFrame(normal_samples, index=df.index) * pd.DataFrame(binomial_samples, index=df.index)

      #cap outliers to ensure consistent returned range
      df = \
      self.__autowhere(df, DPmm_column, df[DPmm_column] < -0.5, -0.5, specified='replacement')
      df = \
      self.__autowhere(df, DPmm_column, df[DPmm_column] > 0.5, 0.5, specified='replacement')

      #support column to signal sign of noise, 0 is neg, 1 is pos
      df = \
      self.__autowhere(df, DPmm_column_temp1, df[DPmm_column] >= 0., 1, specified='replacement')

      #now inject noise, with scaled noise to maintain range 0-1
      #basically we're taking the input df[column] and adding a noise value which may be scaled based on where df[column] falls
      #where the (1 - df[DPmm_column_temp1]) multiplication is to turn on for negative noise
      #and the (df[DPmm_column_temp1]) multiplication is to turn on for positive noise
      #(so if df[column] <0.5, and neg noise, we scale noise to ensure can't result in returned value out of range, similarly for >0.5 and positive noise)
      #this formula is a little counterintuitive, it works because df[column] is in mnmx representation with a range 0-1 and noise is capped at +/- 0.5
      df = \
      self.__autowhere(df, 
                      DPmm_column, 
                      df[column] < 0.5, 
                      df[column] + \
                      (1 - df[DPmm_column_temp1]) * (df[DPmm_column] * df[column] / 0.5) + \
                      (df[DPmm_column_temp1]) * (df[DPmm_column]), \
                      specified='replacement')

      df = \
      self.__autowhere(df, 
                      DPmm_column, 
                      df[column] >= 0.5, 
                      df[column] + \
                      (1 - df[DPmm_column_temp1]) * (df[DPmm_column]) + \
                      (df[DPmm_column_temp1]) * (df[DPmm_column] * (1 - df[column]) / 0.5), \
                      specified='replacement')

      #remove support columns
      del df[column]
      del df[DPmm_column_temp1]

      #rename DPmm_column to column
      df.rename(columns = {DPmm_column : column}, inplace = True)

      #now invert the min/max scaling
      df[column] = df[column] * max_minus_min + minimum

      if MLinfilltype == 'integer':
        df[column] = df[column].round()
      
    #returned data now has stochastic noise injected to a subset of imputations (per flip_prob ratio)
    return df, postprocess_dict

  #__FunctionBlock: ML infill autoML training, inference, and support

  def __assemble_autoMLer(self):
    """
    #populates the "autoMLer" data structure that supports application of autoML for ML infill
    #first tier is platform e.g. 'randomforest', 'autoML_1', 'auto_ML2'
    #(currently just randomforest supported, intent is to build in support for a few autoML platforms)
    #second tier is application i.e. 'classification', 'regression'
    #third tier is action i.e. 'train', 'predict'
    #third tier is populated with associated functions, which follow convention
    
    #train:
    #model = train_function(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)
    
    #predict:
    #infill = predict_function(ML_cmnd, model, df_train_fillfeatures, printstatus)

    #the intent is to incorproate some additional autoML options here in future extension
    
    #note that binary encoded sets use onehotclassification by way of 1010->text conversion in predictinfill function
    """

    autoMLer = {}
    
    autoMLer.update({'randomforest' : {'booleanclassification'  : {'train'   : self._train_randomforest_classifier,
                                                                   'predict' : self._predict_randomforest_classifier},
                                       'ordinalclassification'  : {'train'   : self._train_randomforest_classifier,
                                                                   'predict' : self._predict_randomforest_classifier},
                                       'onehotclassification'   : {'train'   : self._train_randomforest_classifier,
                                                                   'predict' : self._predict_randomforest_classifier},
                                       'regression'             : {'train'   : self._train_randomforest_regressor,
                                                                   'predict' : self._predict_randomforest_regressor}},
                     'customML'     : {'booleanclassification'  : {'train'   : self._train_customML_classifier,
                                                                   'predict' : self._predict_customML_classifier},
                                       'ordinalclassification'  : {'train'   : self._train_customML_classifier,
                                                                   'predict' : self._predict_customML_classifier},
                                       'onehotclassification'   : {'train'   : self._train_customML_classifier,
                                                                   'predict' : self._predict_customML_classifier},
                                       'regression'             : {'train'   : self._train_customML_regressor,
                                                                   'predict' : self._predict_customML_regressor}},
                     'flaml'        : {'booleanclassification'  : {'train'   : self._train_flaml_classifier,
                                                                   'predict' : self._predict_flaml_classifier},
                                       'ordinalclassification'  : {'train'   : self._train_flaml_classifier,
                                                                   'predict' : self._predict_flaml_classifier},
                                       'onehotclassification'   : {'train'   : self._train_flaml_classifier,
                                                                   'predict' : self._predict_flaml_classifier},
                                       'regression'             : {'train'   : self._train_flaml_regressor,
                                                                   'predict' : self._predict_flaml_regressor}},
                     'catboost'     : {'booleanclassification'  : {'train'   : self._train_catboost_classifier,
                                                                   'predict' : self._predict_catboost_classifier},
                                       'ordinalclassification'  : {'train'   : self._train_catboost_classifier,
                                                                   'predict' : self._predict_catboost_classifier},
                                       'onehotclassification'   : {'train'   : self._train_catboost_classifier,
                                                                   'predict' : self._predict_catboost_classifier},
                                       'regression'             : {'train'   : self._train_catboost_regressor,
                                                                   'predict' : self._predict_catboost_regressor}},
                     'xgboost'      : {'booleanclassification'  : {'train'   : self._train_xgboost_classifier_boolean, \
                                                                   'predict' : self._predict_xgboost_classifier}, \
                                       'ordinalclassification'  : {'train'   : self._train_xgboost_classifier, \
                                                                   'predict' : self._predict_xgboost_classifier}, \
                                       'onehotclassification'   : {'train'   : self._train_xgboost_classifier, \
                                                                   'predict' : self._predict_xgboost_classifier}, \
                                       'regression'             : {'train'   : self._train_xgboost_regressor, \
                                                                   'predict' : self._predict_xgboost_regressor}}})
    
    return autoMLer

  def __autoMLer_cleanup(self, postprocess_dict, ML_cmnd):
    """
    strikes entries in the returned postprocess_dict['autoMLer']
    that won't be inspected in postmunge
    """

    if 'autoML_type' not in ML_cmnd:
      ML_cmnd.update({'autoML_type' : 'randomforest'})
      
    autoMLtype = ML_cmnd['autoML_type']
    
    autoMLer_keys_to_delete = list(postprocess_dict['autoMLer'])
    
    autoMLer_keys_to_delete.remove(autoMLtype)
    
    for autoMLer_key_to_delete in autoMLer_keys_to_delete:

      if autoMLer_key_to_delete in postprocess_dict['autoMLer']:
      
        del postprocess_dict['autoMLer'][autoMLer_key_to_delete]
    
    #for customML scenario, we'll only store the inference function so user doesn't have to reintialize trainer for postmunge
    #tradeoff is that postmunge feature importance will need to apply a different model type
    if autoMLtype == 'customML':
      #modeltarget is e.g. booleanclassification, ordinalclassification, etc
      for modeltarget in postprocess_dict['autoMLer'][autoMLtype]:
        del postprocess_dict['autoMLer'][autoMLtype][modeltarget]['train']

    #we'll also strike any populated customML training functions passed to ML_cmnd['customML']
    if 'customML' in ML_cmnd:
      if 'customML_Classifier_train' in ML_cmnd['customML']:
        ML_cmnd['customML']['customML_Classifier_train'] = True
      if 'customML_Regressor_train' in ML_cmnd['customML']:
        ML_cmnd['customML']['customML_Regressor_train'] = True
        
    return postprocess_dict, ML_cmnd

  def __populateMLinfilldefaults(self, randomseed):
    '''
    populates a dictionary with default values for ML infill,
    currently based on Random Forest Regressor and Random Forest Classifier 
    (Each based on ScikitLearn default values)
  
    note that n_estimators set at 100 (default for version 0.22)

    note that we apply our own random seed by default instead of defering to scikit
    (where randomseed is randomized if not specified)
    '''
  
    MLinfilldefaults = {'RandomForestClassifier':{}, 'RandomForestRegressor':{}}
    
    MLinfilldefaults['RandomForestClassifier'].update({'random_state':randomseed})
  
    MLinfilldefaults['RandomForestRegressor'].update({'random_state':randomseed})

    return MLinfilldefaults

  def __initRandomForestClassifier(self, ML_cmnd, MLinfilldefaults):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a RandomForestClassifier model
    '''

    #populate ML_cmnd if stuff not already present
    if 'MLinfill_cmnd' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_cmnd':{}})
    if 'RandomForestClassifier' not in ML_cmnd['MLinfill_cmnd']:
      ML_cmnd['MLinfill_cmnd'].update({'RandomForestClassifier':{}})

    model_params = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']
    model_params.update(MLinfilldefaults['RandomForestClassifier'])

    #do other stuff?

    #then initialize RandomForestClassifier model
    model = RandomForestClassifier(
                                  **model_params
                                  )

    return model

  def __initRandomForestRegressor(self, ML_cmnd, MLinfilldefaults):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a RandomForestRegressor model
    '''

    #populate ML_cmnd if stuff not already present
    if 'MLinfill_cmnd' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_cmnd':{}})
    if 'RandomForestRegressor' not in ML_cmnd['MLinfill_cmnd']:
      ML_cmnd['MLinfill_cmnd'].update({'RandomForestRegressor':{}})

    model_params = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']
    model_params.update(MLinfilldefaults['RandomForestRegressor'])

    #do other stuff?

    #then initialize RandomForestRegressor model 
    model = RandomForestRegressor(
                                  **model_params,
                                 )

    return model

  def _train_randomforest_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #performs tuning if appropriate based on ML_cmnd
    #initializes model
    #trains model
    
    #uses scikit-learn random forest models
    
    #where tuning is activated by passing parameters to the model as lists or distributions instead of distinct values
    #and uses grid search or random search based on ML_cmnd
    #see also ML_cmnd documentation in read me
    
    #determination of whether parameters passed as targets for tuning is by inspect_ML_cmnd function
    #and if tuning applied aggregation of entries distinguishing tuning targets use assemble_param_sets function
    
    #in short, in addition to parameters passed to model
    #ML_cmnd also accepts arguments for hyperparam_tuner and randomCV_n_iter
    #where hyperparam_tuner can be one of {False, 'gridCV', 'randomCV'}
    #and randomCV_n_iter can be passed as an integer when hyperparam_tuner passed as randomCV
    
    #model initialization makes use of initRandomForestClassifier function
    #and default values for Random Forest Classifer are initialized with populateMLinfilldefaults
    """

    #for single column convert to a series
    if df_train_filllabel.shape[1] == 1:
      df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]
    
    #initialize defaults dictionary, these are the default parameters for random forest model initialization
    MLinfilldefaults = \
    self.__populateMLinfilldefaults(randomseed)
    
    #ML_cmnd accepts specification for type of tuner when hyperparaemter tuning applied, else defaults to gridCV
    MLinfill_tuner = ML_cmnd['hyperparam_tuner']
      
    #if randomCV tuner applied, number of iterations accepted as randomCV_n_iter, else defaults to 100
    randomCV_n_iter = ML_cmnd['randomCV_n_iter']
    
    autoML_type = ML_cmnd['autoML_type']
    MLinfill_alg = 'RandomForestClassifier'
    
    #tune marker tells us if user passed some parameters as a list for hyperparameter tuning
    tune_marker = self.__inspect_ML_cmnd(ML_cmnd, autoML_type, MLinfill_alg)
    
    if tune_marker is True:
    
      #static_params are user passed parameters that won't be tuned, 
      #tune_params are user passed params (passed as list or range) that will be tuned
      static_params, tune_params = self.__assemble_param_sets(ML_cmnd, autoML_type, MLinfill_alg)
    
      #we'll create a temp ML_cmnd to initialize a tuning model
      temp_ML_cmnd = {'MLinfill_cmnd':{MLinfill_alg : static_params}}
      
      #then we'll initialize a tuning model
      #note that this populates the parameters to be tuned with defaults
      #my understanding is that scikit gridsearch still allows tuning for parameters
      #that were previously initialized in the model
      tuning_model = self.__initRandomForestClassifier(temp_ML_cmnd, MLinfilldefaults)
    
      #for now we'll default to grid scoring of ‘accuracy’
      #I've heard that F1 score is a better general default, but not sure how it handles edge cases
      #need to do a little more investsigation on this point
      #(the problem with f1 is if folds split doesn't have fully represented activations triggers printouts)
      grid_scoring = 'accuracy'
      # grid_scoring = 'f1_weighted'
      
      #now we'll initialize a grid search
      if MLinfill_tuner == 'gridCV':
        tuned_model = GridSearchCV(tuning_model, cv=5, iid='deprecated', \
                                   param_grid = tune_params, scoring = grid_scoring)
      elif MLinfill_tuner == 'randomCV':
        tuned_model = RandomizedSearchCV(tuning_model, cv=5, iid='deprecated', \
                                     param_distributions = tune_params, scoring = grid_scoring, \
                                     n_iter = randomCV_n_iter)
      else:
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: randomforest hyperparam_tuner currently only supports 'gridCV' or 'randomCV'.")      
        self.__autoprint('warning', '')
      
      #now we'll run a fit on the grid search
      #for now won't pass any fit parameters (which means omitting the scikit option for sample_weight)
      fit_params = {}
      tuned_model.fit(df_train_filltrain, df_train_filllabel, **fit_params)    
    
      #acess the tuned parameters based on the tuning operation
      tuned_params = tuned_model.best_params_    

      # if printstatus is True:
      self.__autoprint('debug', "tuned parameters:")
      self.__autoprint('debug', tuned_params)
      self.__autoprint('debug', "")

      return tuned_model, postprocess_dict

    else:
      
      model = self.__initRandomForestClassifier(ML_cmnd, MLinfilldefaults)

      model.fit(df_train_filltrain, df_train_filllabel)

      return model, postprocess_dict

  def _predict_randomforest_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in train_randomforest_classifier
    #for random forest
    #returns infill predictions
    """

    infill = model.predict(fillfeatures)
    
    return infill

  def _train_randomforest_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #performs tuning if appropriate based on ML_cmnd
    #initializes model
    #trains model
    
    #uses scikit-learn random forest models
    
    #where tuning is activated by passing parameters to the model as lists or distributions instead of distinct values
    #and uses grid search or random search based on ML_cmnd
    #see also ML_cmnd documentation in read me
    
    #determination of whether parameters passed as targets for tuning is by inspect_ML_cmnd function
    #and if tuning applied aggregation of entries distinguishing tuning targets use assemble_param_sets function
    
    #in short, in addition to parameters passed to model
    #ML_cmnd also accepts arguments for hyperparam_tuner and randomCV_n_iter
    #where hyperparam_tuner can be one of {False, 'gridCV', 'randomCV'}
    #and randomCV_n_iter can be passed as an integer when hyperparam_tuner passed as randomCV
    
    #model initialization makes use of initRandomForestRegressor function
    #and default values for Random Forest Regressor are initialized with populateMLinfilldefaults
    """

    #for single column convert to a series
    df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]
    
    #initialize defaults dictionary, these are the default parameters for random forest model initialization
    MLinfilldefaults = \
    self.__populateMLinfilldefaults(randomseed)
    
    #ML_cmnd accepts specification for type of tuner when hyperparaemter tuning applied, else defaults to gridCV
    MLinfill_tuner = ML_cmnd['hyperparam_tuner']
      
    #if randomCV tuner applied, number of iterations accepted as randomCV_n_iter, else defaults to 100
    randomCV_n_iter = ML_cmnd['randomCV_n_iter']
    
    autoML_type = ML_cmnd['autoML_type']
    MLinfill_alg = 'RandomForestRegressor'
    
    #tune marker tells us if user passed some parameters as a list for hyperparameter tuning
    tune_marker = self.__inspect_ML_cmnd(ML_cmnd, autoML_type, MLinfill_alg)
    
    if tune_marker is True:
    
      #static_params are user passed parameters that won't be tuned, 
      #tune_params are user passed params (passed as list or range) that will be tuned
      static_params, tune_params = self.__assemble_param_sets(ML_cmnd, autoML_type, MLinfill_alg)
    
      #we'll create a temp ML_cmnd to initialize a tuning model
      temp_ML_cmnd = {'MLinfill_cmnd':{MLinfill_alg : static_params}}
      
      #then we'll initialize a tuning model
      #note that this populates the parameters to be tuned with defaults
      #my understanding is that scikit gridsearch still allows tuning for parameters
      #that were previously initialized in the model
      tuning_model = self.__initRandomForestRegressor(temp_ML_cmnd, MLinfilldefaults)
    
      #for now we'll default to grid scoring of ‘neg_mean_squared_error’
      #am not positive this is best default this is worth some further investigation when get a chance
      grid_scoring = 'neg_mean_squared_error'
      
      #now we'll initialize a grid search
      if MLinfill_tuner == 'gridCV':
        tuned_model = GridSearchCV(tuning_model, cv=5, iid='deprecated', \
                                   param_grid = tune_params, scoring = grid_scoring)
      elif MLinfill_tuner == 'randomCV':
        tuned_model = RandomizedSearchCV(tuning_model, cv=5, iid='deprecated', \
                                     param_distributions = tune_params, scoring = grid_scoring, \
                                     n_iter = randomCV_n_iter)
      else:
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: randomforest hyperparam_tuner currently only supports 'gridCV' or 'randomCV'.")
        self.__autoprint('warning', '')
      
      #now we'll run a fit on the grid search
      #for now won't pass any fit parameters (which means omitting the scikit option for sample_weight)
      fit_params = {}
      tuned_model.fit(df_train_filltrain, df_train_filllabel, **fit_params)    
    
      #acess the tuned parameters based on the tuning operation
      tuned_params = tuned_model.best_params_    

      # if printstatus is True:
      self.__autoprint('debug', "tuned parameters:")
      self.__autoprint('debug', tuned_params)
      self.__autoprint('debug', "")
        
      return tuned_model, postprocess_dict

    else:
      
      model = self.__initRandomForestRegressor(ML_cmnd, MLinfilldefaults)

      model.fit(df_train_filltrain, df_train_filllabel)

      return model, postprocess_dict

  def _predict_randomforest_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in train_randomforest_classifier
    #for random forest
    #returns infill predictions
    """

    infill = model.predict(fillfeatures)
    
    return infill

  def _train_autogluon_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    this option no longer supported as of 7.72
    retaining function naming space for backward compatibility of prior populated postprocess_dict referencing this function
    """

    print("Please note autogluon ml infill option no longer supported as of 7.72")
    return False, postprocess_dict

  def _train_autogluon_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    this option no longer supported as of 7.72
    retaining function naming space for backward compatibility of prior populated postprocess_dict referencing this function
    """
    print("Please note autogluon ml infill option no longer supported as of 7.72")
    return False, postprocess_dict

  def _predict_autogluon_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    this option no longer supported as of 7.72
    retaining function naming space for backward compatibility of prior populated postprocess_dict referencing this function
    """

    print("Please note autogluon ml infill option no longer supported as of 7.72")
    return np.zeros(shape=(1,len(categorylist)))

  def _predict_autogluon_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    this option no longer supported as of 7.72
    retaining function naming space for backward compatibility of prior populated postprocess_dict referencing this function
    """

    print("Please note autogluon ml infill option no longer supported as of 7.72")
    return np.zeros(shape=(1,len(categorylist)))

  def _train_flaml_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #Trains a model for ML infill using flaml classifier
    #accepts parameters to fit operation as ML_cmnd['MLinfill_cmnd']['flaml_classifier_fit']
    #converts multi column one hot sets to ordinal (no string conversion required)
    """

    # #user can conduct this import externally to speed up this function
    # module = 'flaml.automl'
    # if module not in sys.modules:
    from flaml import AutoML
    
    try:
    # if True is True:

      df_train_filllabel = df_train_filllabel.copy()

      #column headers matter for convert_onehot_to_singlecolumn methods, reset as integers
      df_train_filllabel.columns = list(range(len(list(df_train_filllabel.columns))))
      df_train_filllabel = df_train_filllabel.reset_index(drop=True)

      ag_label_column = list(df_train_filllabel.columns)

      #flaml accepts single column labels, string conversion not required since designating task type as classification
      if len(ag_label_column) == 1:
        ag_label_column = ag_label_column[0]

      else:
        df_train_filllabel, ML_cmnd = self.__convert_onehot_to_singlecolumn(df_train_filllabel, ML_cmnd, stringtype=False)
        ag_label_column = list(df_train_filllabel.columns)[0]

      #convert to a Series
      df_train_filllabel = df_train_filllabel[ag_label_column]

      #user can pass parameters to flaml fit operation in ML_cmnd['MLinfill_cmnd']['flaml_classifier_fit']
      fit_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'flaml_classifier_fit' in ML_cmnd['MLinfill_cmnd']:
          fit_params = ML_cmnd['MLinfill_cmnd']['flaml_classifier_fit']

      #we'll have a default parameter to set task type as classificaiton
      #user can override in ML_cmnd['MLinfill_cmnd']['flaml_classifier_fit']
      default_fit_params = {'task' : 'classification', 'verbose' : 0}

      #now incorproate user passed parameters
      default_fit_params.update(fit_params)

      train_nunique = int(df_train_filllabel.nunique())
      train_rows = int(df_train_filllabel.shape[0])

      if train_nunique < 2 or train_nunique > 0.75 * train_rows:
        model = False

      else:

        #initialize model
        model = AutoML()

        #train the model without validation set
        model.fit(
          df_train_filltrain, df_train_filllabel, **default_fit_params
        )

      return model, postprocess_dict
    
    except:
      return False, postprocess_dict

  def _predict_flaml_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in flaml_classifier
    #returns infill predictions

    #the categorylist parameter is used to handle an edge case
    #note that in some cases the passed categorylist may be a proxy list of equivalent length
    #such as a range of integers
    """

    # #user can conduct this import externally to speed up this function
    # module = 'flaml.automl'
    # if module not in sys.modules:
    from flaml import AutoML
    
    if model is not False:
      
      infill = model.predict(fillfeatures)

      if len(categorylist) > 1:

        infill = self.__convert_singlecolumn_to_onehot(infill, ML_cmnd, categorylist)

      return infill
    
    else:

      infill = np.zeros(shape=(1,len(categorylist)))
      
      return infill

  def _train_flaml_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #Trains a model for ML infill using flaml regressor
    #accepts parameters to fit operation as ML_cmnd['MLinfill_cmnd']['flaml_regressor_fit']
    """

    # #user can conduct this import externally to speed up this function
    # module = 'flaml.automl'
    # if module not in sys.modules:
    from flaml import AutoML
    
    try:
    # if True is True:

      #user can pass parameters to flaml fit operation in ML_cmnd['MLinfill_cmnd']['flaml_regressor_fit']
      fit_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'flaml_regressor_fit' in ML_cmnd['MLinfill_cmnd']:
          fit_params = ML_cmnd['MLinfill_cmnd']['flaml_regressor_fit']

      #we'll have a default parameter to set task type as regression
      #user can override in ML_cmnd['MLinfill_cmnd']['flaml_classifier_fit']
      default_fit_params = {'task' : 'regression', 'verbose' : 0}

      #now incorproate user passed parameters
      default_fit_params.update(fit_params)

      train_nunique = int(df_train_filllabel.nunique())

      #convert to a Series
      df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]

      if train_nunique < 2:
        model = False

      else:

        #initialize model
        model = AutoML()

        #train the model without validation set
        model.fit(
          df_train_filltrain, df_train_filllabel, **default_fit_params
        )

      return model, postprocess_dict
    
    except:
      return False, postprocess_dict

  def _predict_flaml_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in flaml_regressor
    #returns infill predictions
    """

    # #user can conduct this import externally to speed up this function
    # module = 'flaml.automl'
    # if module not in sys.modules:
    from flaml import AutoML
    
    if model is not False:
      
      infill = model.predict(fillfeatures)

      return infill
    
    else:

      infill = np.zeros(shape=(1,len(categorylist)))
      
      return infill

  def _train_catboost_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #Trains a model for ML infill using catboost classifier
    #accepts parameters to model initialization as ML_cmnd['MLinfill_cmnd']['catboost_classifier_model']
    #accepts parameters to fit operation as ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']
    #defaults to no early stopping with 0% validation set, can be turned on by passing e.g.
    #ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']['eval_ratio'] = 0.15
    #note that early stopping may cause issues in ML infill when all instances of label carried into validation set
    """

    # #user can conduct this import externally to speed up this function
    # module = 'catboost'
    # if module not in sys.modules:
    from catboost import CatBoostClassifier
    
    try:

      df_train_filllabel = df_train_filllabel.copy()

      #catboost takes specification of categoric columns
      columntypes = self.__populate_columntype_report(postprocess_dict, list(df_train_filltrain))
      categorical_features_indices = \
      columntypes['boolean'] + columntypes['ordinal'] \
      + columntypes['onehot'] + columntypes['binary']

      #column headers matter for convert_onehot_to_singlecolumn methods, reset as integers
      df_train_filllabel.columns = list(range(len(list(df_train_filllabel.columns))))
      df_train_filllabel = df_train_filllabel.reset_index(drop=True)

      ag_label_column = list(df_train_filllabel.columns)

      #catboost accepts single column labels, these both convert to string for recognition of classification
      if len(ag_label_column) == 1:
        ag_label_column = ag_label_column[0]
        df_train_filllabel[ag_label_column] = df_train_filllabel[ag_label_column].astype(str)

      else:
        df_train_filllabel, ML_cmnd = self.__convert_onehot_to_singlecolumn(df_train_filllabel, ML_cmnd, stringtype=True)
        ag_label_column = list(df_train_filllabel.columns)[0]

      #convert single column to series
      df_train_filllabel = df_train_filllabel[ag_label_column]

      #user can pass parameters to catboost model initialization in ML_cmnd['MLinfill_cmnd']['catboost_classifier_model']
      model_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'catboost_classifier_model' in ML_cmnd['MLinfill_cmnd']:
          model_params = ML_cmnd['MLinfill_cmnd']['catboost_classifier_model']

      default_model_params = {'random_seed'   : randomseed, \
                              'logging_level' : 'Silent'}

      #these are parameters for early stopping, we'll remove them if no eval_ratio
      default_model_params.update({
          'od_type': 'Iter',
          'od_wait': 40
      })

      #now incorproate user passed parameters
      default_model_params.update(model_params)

      #user can pass parameters to catboost fit operation in ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']
      fit_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'catboost_classifier_fit' in ML_cmnd['MLinfill_cmnd']:
          fit_params = ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']

      #we'll have a custom parameter to carve out a validation set for early stopping defaulting to 0.15
      #user can override in ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']
      default_fit_params = {'eval_ratio' : 0}

      #now incorproate user passed parameters
      default_fit_params.update(fit_params)

      if 'eval_ratio' in default_fit_params:
        eval_ratio = default_fit_params['eval_ratio']
        del default_fit_params['eval_ratio']
      else:
        eval_ratio = 0

      if eval_ratio > 0 and eval_ratio < 1:
        
        #extract validation sets
        df_train_filltrain, df_train_filltrain_val = \
        self.__df_split(df_train_filltrain, eval_ratio, True, randomseed)

        df_train_filllabel, df_train_filllabel_val = \
        self.__df_split(df_train_filllabel, eval_ratio, True, randomseed)
        
        train_nunique = int(df_train_filllabel.nunique())
        train_rows = int(df_train_filllabel.shape[0])

        #catboost needs >1 label and takes a long time to train with all unique labels which are edge cases
        #0.75 is a heuristic 
        if train_nunique < 2 or train_nunique > 0.75 * train_rows:
          model = False

        else:
          #initialize model
          model = CatBoostClassifier(
            **default_model_params
          )

          #train the model with validation set
          model.fit(
            df_train_filltrain, df_train_filllabel, \
            eval_set=(df_train_filltrain_val, df_train_filllabel_val),
            cat_features = categorical_features_indices, \
            **default_fit_params
          )

      else:

        train_nunique = int(df_train_filllabel.nunique())
        train_rows = int(df_train_filllabel.shape[0])
        
        if train_nunique < 2 or train_nunique > 0.75 * train_rows:
          model = False
        
        else:

          #remove early stop params since no validation set
          for entry in {'od_type', 'od_wait'}:
            if entry in default_model_params:
              del default_model_params[entry]

          #initialize model
          model = CatBoostClassifier(
            **default_model_params
          )

          #train the model without validation set
          model.fit(
            df_train_filltrain, df_train_filllabel, cat_features = categorical_features_indices, **default_fit_params
          )

      return model, postprocess_dict
    
    except ValueError:
      return False, postprocess_dict

  def _predict_catboost_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in catboost_classifier
    #returns infill predictions

    #the categorylist parameter is used to handle an edge case
    #note that in some cases the passed categorylist may be a proxy list of equivalent length
    #such as a range of integers
    """

    # #user can conduct this import externally to speed up this function
    # module = 'catboost'
    # if module not in sys.modules:
    from catboost import CatBoostClassifier
    
    if model is not False:
      
      infill = model.predict(fillfeatures)

      if len(categorylist) > 1:

        infill = self.__convert_singlecolumn_to_onehot(infill, ML_cmnd, categorylist)

      return infill
    
    else:

      infill = np.zeros(shape=(1,len(categorylist)))
      
      return infill

  def _train_catboost_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    """
    #Trains a model for ML infill using catboost regressor
    #accepts parameters to model initialization as ML_cmnd['MLinfill_cmnd']['catboost_regressor_model']
    #accepts parameters to fit operation as ML_cmnd['MLinfill_cmnd']['catboost_regressor_fit']
    #defaults to early stopping with 15% validation set, can be turned off by passing 
    #ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']['eval_ratio'] = 0
    """

    # #user can conduct this import externally to speed up this function
    # module = 'catboost'
    # if module not in sys.modules:
    from catboost import CatBoostRegressor
    
    try:

      #convert single column to series
      df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]

      #catboost takes specification of categoric columns
      columntypes = self.__populate_columntype_report(postprocess_dict, list(df_train_filltrain))
      categorical_features_indices = \
      columntypes['boolean'] + columntypes['ordinal'] \
      + columntypes['onehot'] + columntypes['binary']

      #user can pass parameters to catboost model initialization in ML_cmnd['MLinfill_cmnd']['catboost_classifier_model']
      model_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'catboost_regressor_model' in ML_cmnd['MLinfill_cmnd']:
          model_params = ML_cmnd['MLinfill_cmnd']['catboost_regressor_model']

      default_model_params = {'random_seed'   : randomseed, \
                              'logging_level' : 'Silent'}

      #these are parameters for early stopping, we'll remove them if no eval_ratio
      default_model_params.update({
          'od_type': 'Iter',
          'od_wait': 40
      })

      #now incorproate user passed parameters
      default_model_params.update(model_params)

      #user can pass parameters to catboost fit operation in ML_cmnd['MLinfill_cmnd']['catboost_regressor_fit']
      fit_params = {}
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'catboost_regressor_fit' in ML_cmnd['MLinfill_cmnd']:
          fit_params = ML_cmnd['MLinfill_cmnd']['catboost_regressor_fit']

      #we'll have a custom parameter to carve out a validation set for early stopping defaulting to 0.15
      #user can override in ML_cmnd['MLinfill_cmnd']['catboost_classifier_fit']
      default_fit_params = {'eval_ratio' : 0.15}

      #now incorproate user passed parameters
      default_fit_params.update(fit_params)

      if 'eval_ratio' in default_fit_params:
        eval_ratio = default_fit_params['eval_ratio']
        del default_fit_params['eval_ratio']
      else:
        eval_ratio = 0

      if eval_ratio > 0 and eval_ratio < 1:
        
        #extract validation sets
        df_train_filltrain, df_train_filltrain_val = \
        self.__df_split(df_train_filltrain, eval_ratio, True, randomseed)

        df_train_filllabel, df_train_filllabel_val = \
        self.__df_split(df_train_filllabel, eval_ratio, True, randomseed)

        train_nunique = int(df_train_filllabel.nunique())
        
        if train_nunique < 2:
          model = False

        else:

          #initialize model
          model = CatBoostRegressor(
            **default_model_params
          )

          #train the model with validation set
          model.fit(
            df_train_filltrain, df_train_filllabel, \
            eval_set=(df_train_filltrain_val, df_train_filllabel_val),
            cat_features = categorical_features_indices, \
            **default_fit_params
          )

      else:

        train_nunique = int(df_train_filllabel.nunique())
        
        if train_nunique < 2:
          model = False

        else:

          #remove early stop params since no validation set
          for entry in {'od_type', 'od_wait'}:
            if entry in default_model_params:
              del default_model_params[entry]

          #initialize model
          model = CatBoostRegressor(
            **default_model_params
          )

          #train the model without validation set
          model.fit(
            df_train_filltrain, df_train_filllabel, cat_features = categorical_features_indices, **default_fit_params
          )

      return model, postprocess_dict
    
    except ValueError:
      return False, postprocess_dict

  def _predict_catboost_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in catboost_regressor
    #returns infill predictions
    """

    # #user can conduct this import externally to speed up this function
    # module = 'catboost'
    # if module not in sys.modules:
    from catboost import CatBoostRegressor
    
    if model is not False:
      
      infill = model.predict(fillfeatures)

      return infill
    
    else:

      infill = np.zeros(shape=(1,len(categorylist)))
      
      return infill

  def _train_xgboost_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    modeltype = 'classification'
    return self.__train_xgboost(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype)

  def _train_xgboost_classifier_boolean(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    modeltype = 'boolean'
    return self.__train_xgboost(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype)

  def _train_xgboost_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    modeltype = 'regression'
    return self.__train_xgboost(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype)
  
  def __train_xgboost(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype='regression'):
    """
    this function adapted from the customML training function
    and supports hyperparameter tuning by passing ML_cmnd['hyperparam_tuner'] = 'optuna_XG1' 
    which also supports setting number of trials and max time via ML_cmnd['optuna_n_iter'] and ['optuna_timeout']

    for classificaiton scenario, this function accepts both ordinal and one hot label target forms (1010 is converted to onehot externally)
    onehot is converted to ordinal and provide labels as a fully represented sequential range integer set from 0 to max encoding

    modeltype accepted as classification, boolean, and regression
    where boolean differs from classificaiton by setting different xgboost objective in tuning
    """

    #xgboost tuning with optuna benefits from distinguishing between binary classification and multi target classificaiton
    #booleantype is marker for this purpose and primary use is for passing entry to optuna tuner
    booleantype = False
    if modeltype == 'boolean':
      booleantype = True
      modeltype = 'classification'
    
    # columntype_report = self.__populate_columntype_report(postprocess_dict, list(df_train_filltrain))
    
    #featurekey is a unique concatinated string of all column headers serving as basis for this model
    featureskey = ''
    for entry in df_train_filltrain.columns:
      featureskey += entry
    ML_cmnd['customML_inference_support'].update({featureskey : {'seq_ordinal_marker':True}})

    df_train_filllabel = df_train_filllabel.copy()
    
    #column headers matter for convert_onehot_to_singlecolumn methods, reset as integers
    df_train_filllabel.columns = list(range(len(list(df_train_filllabel.columns))))
    df_train_filllabel = df_train_filllabel.reset_index(drop=True)
    
    df_train_filltrain = df_train_filltrain.reset_index(drop=True)
    
    ML_label_columns = list(df_train_filllabel.columns)

    if len(ML_label_columns) == 1:
      #this will be ML_label_column = integer 0
      ML_label_column = ML_label_columns[0]

    else:
      #note this scenario only occurs for classification
      #returns a single column with str(int) entries encoding each distinct activation set
      #(moved the featureskey methods to __sequential_range_integer)
      df_train_filllabel, ML_cmnd = self.__convert_onehot_to_singlecolumn(df_train_filllabel, ML_cmnd, stringtype=False)
      ML_label_column = list(df_train_filllabel.columns)[0]

    if modeltype == 'classification':
      #convention is that regression will return labels with header of integer 0, classification with header of integer 1
      df_train_filllabel = df_train_filllabel.rename(columns = {ML_label_column:1})
      ML_label_column = 1

      #convert to fully represented sequential ranged integer as str(int) type
      df_train_filllabel, ML_cmnd = \
      self.__sequential_range_integer(df_train_filllabel, ML_cmnd, stringtype = True, featureskey=featureskey)

    #convert single column to a series
    if df_train_filllabel.shape[1] == 1:
      df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]
    
    #note that we know that ML_label_column won't overlap with df_train_filltrain headers
    #because df_train_filltrain headers are strings in suffix convention which include an underscore character
    #and labels header is integer 0
    #so if user wants to concatinate labels onto training set in their function it is ok
    
    #access any user passed parameters
    commands = {}
    if modeltype == 'classification':
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'xgboost_Classifier' in ML_cmnd['MLinfill_cmnd']:
          commands = ML_cmnd['MLinfill_cmnd']['xgboost_Classifier']
    if modeltype == 'regression':
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'xgboost_Regressor' in ML_cmnd['MLinfill_cmnd']:
          commands = ML_cmnd['MLinfill_cmnd']['xgboost_Regressor']
    
    #train the model
    model = False
    if modeltype == 'classification':

      if df_train_filllabel.nunique() <= 1:
        model = False

      else:

        df_train_filllabel = df_train_filllabel.astype(int)

        #classify_regress will be basis of optuna tuner
        classify_regress = 'classify'
        #for binary classification using different xgboost objective basis
        if booleantype == True:
          classify_regress = 'boolean'
        
        tuned_params = {"use_label_encoder" : False}
        if ML_cmnd['hyperparam_tuner'] == 'optuna_XG1':
          tuned_params = \
          self.__optuna_XG1(df_train_filltrain, df_train_filllabel, 0.25, ML_cmnd, randomseed, classify_regress)
        else:
          #otherwise set any settings based on ML_cmnd['xgboost_gpu_id']
          #noting that specific commands passed through ML_cmnd['MLinfill_cmnd'] still take precedence
          if ML_cmnd['xgboost_gpu_id'] is not False:
            tuned_params.update({'tree_method' : 'gpu_hist',
                                'gpu_id'      : ML_cmnd['xgboost_gpu_id'] })

        #now update with any commands passed through ML_cmnd['MLinfill_cmnd']
        tuned_params.update(commands)

        from xgboost import XGBClassifier
        model = XGBClassifier(**tuned_params)

        #train the model without validation set
        model.fit(
          df_train_filltrain, df_train_filllabel,
        )
      
    elif modeltype == 'regression':

      tuned_params = {}
      if ML_cmnd['hyperparam_tuner'] == 'optuna_XG1':
        tuned_params = \
        self.__optuna_XG1(df_train_filltrain, df_train_filllabel, 0.25, ML_cmnd, randomseed, 'regress')
      
      tuned_params.update(commands)

      from xgboost import XGBRegressor
      model = XGBRegressor(**tuned_params)

      #train the model without validation set
      model.fit(
        df_train_filltrain, df_train_filllabel,
      )
    
    postprocess_dict['customML_inference_support'].update(ML_cmnd['customML_inference_support'])
    
    return model, postprocess_dict

  def _predict_xgboost_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    modeltype = 'classification'
    return self.__predict_xgboost(ML_cmnd, model, fillfeatures, printstatus, categorylist, modeltype)

  def _predict_xgboost_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    modeltype = 'regression'
    return self.__predict_xgboost(ML_cmnd, model, fillfeatures, printstatus, categorylist, modeltype)

  def __predict_xgboost(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[], modeltype='regression'):
    """
    this function adapted from the customML predict function, 
    and makes use of the xgboost default inference function to run inference
    """

    if model is not False:

      if modeltype == 'classification':
        from xgboost import XGBClassifier
      elif modeltype == 'regression':
        from xgboost import XGBRegressor
      
      #featurekey is a unique concatinated string of all column headers serving as basis for this model
      featureskey = ''
      for entry in fillfeatures.columns:
        featureskey += entry
      
      fillfeatures = fillfeatures.reset_index(drop=True)
      
      defaulttype = 'xgboost'
      commands = {}
      
      infill = self.__call_default_function(defaulttype, modeltype, fillfeatures, model, commands, ML_cmnd)
              
      #infill is expected as a single column, as either a pandas dataframe, pandas series, or numpy array
      #we'll convert to a flattened array for common form
      if type(infill) == type(pd.DataFrame()) or type(infill) == type(pd.Series([1])):
        infill = infill.to_numpy()
        
      infill = infill.ravel()
              
      if modeltype == 'classification':
        
        #returned values from inference are accepted as either type int or type str(int)
        #categorylist is a list of headers corresponding to onehot form, where if this is a 1010 set will be list before converting back to binarized
        if len(categorylist) > 1:
          
          #this inverts any consolidations to sequential ranged integer from training
          infill = self.__sequential_range_integer_invert(infill, ML_cmnd, featureskey=featureskey)
          
          #this will return a onehot encoded array with 0/1 integer entries
          #note the featureskey method is now performed in __sequential_range_integer_invert
          infill = self.__convert_singlecolumn_to_onehot(infill, ML_cmnd, categorylist)
        
        else:
          
          #else for single column case if entries are str(int) we'll convert to int
          #so this could result in a boolean integer set or ordinal integer set depending on label composition
          infill = infill.astype(int)

    elif model is False:
      
      #note that infill is not inserted when model is False
      infill = np.zeros(shape=(1,len(categorylist)))
      
    return infill

  #__FunctionBlock: customML support functions

  def _train_customML_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    modeltype = 'classification'
    return self.__train_customML(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype)

  def _train_customML_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict):
    modeltype = 'regression'
    return self.__train_customML(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype)
  
  def __train_customML(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus, postprocess_dict, modeltype='regression'):
    """
    #wrapper for custom defined autoMLer training functions 
    #either for classification or regression, which this function distinguishes with modeltype parameter
    
    #which custom defined functions are accepted in the form
    #def customML_train_template(labels, features, columntype_report, commands, randomseed):
    #  return model
    
    #where labels for classification are passed to the custom function 
    #as a pandas dataframe with single column with header of integer 1 with str(int) entries
    #and if user prefers labels as integers instead of string can apply in their function: labels = labels.astype(int)
    
    #and labels for regression are passed to the custom function 
    #as a pandas dataframe with single column with header of integer 0 with continuous float entries
    
    #and features is received as a pandas dataframe numerically encoded, 
    #with categoric entries as integers, and headers matching the returned suffix convention

    #columntype_report is a dictionary reporting properties of the columns found in features
    #a list of categoric features is available as columntype_report['all_categoric']
    #a list of of numeric features is available as columntype_report['all_numeric']
    #and columntype_report also contains more granular information such as feature set groupings and types

    #commands is received as a dictionary as passed by user 
    #for classifier in ML_cmnd['MLinfill_cmnd']['customClassifier]
    #or for regression in ML_cmnd['MLinfill_cmnd']['customRegressor]
    #randomseed is the randomseed associated with the automunge(.) call
    #the returned model is saved in postprocess_dict
    #and accessed to impute missing data in automunge and again in postmunge
    #if model training not successful user can return model as False
    
    #note that pandas is available as pd and numpy as np
    
    #we'll have convention that if custom function returns a ValueError
    #will not halt operation and instead just return model as False 
    #meaning imputaitons will defer to the defaultinfill applied with transformation function
    
    #any required imports can either be conducted externally when defining the function
    #or internal to the template
    
    #the custom_autoMLer_train_template will be passed to an automunge call in ML_cmnd as
    #ML_cmnd = {'autoML_type':'customML',
    #           'MLinfill_cmnd':{'customML_Classifier':{},
    #                            'customML_Regressor':{}},
    #           'customML':{'customML_Classifier_train'  :function, 
    #                       'customML_Classifier_predict':function, 
    #                       'customML_Regressor_train'   :function, 
    #                       'customML_Regressor_predict' :function}}
    """

    columntype_report = self.__populate_columntype_report(postprocess_dict, list(df_train_filltrain))
    
    #featurekey is a unique concatinated string of all column headers serving as basis for this model
    featureskey = ''
    for entry in df_train_filltrain.columns:
      featureskey += entry
    ML_cmnd['customML_inference_support'].update({featureskey : {'seq_ordinal_marker':True}})

    df_train_filllabel = df_train_filllabel.copy()
    
    #column headers matter for convert_onehot_to_singlecolumn methods, reset as integers
    df_train_filllabel.columns = list(range(len(list(df_train_filllabel.columns))))
    df_train_filllabel = df_train_filllabel.reset_index(drop=True)
    
    df_train_filltrain = df_train_filltrain.reset_index(drop=True)
    
    ML_label_columns = list(df_train_filllabel.columns)

    if len(ML_label_columns) == 1:
      #this will be ML_label_column = integer 0
      ML_label_column = ML_label_columns[0]

      if modeltype == 'classification':
        df_train_filllabel[ML_label_column] = df_train_filllabel[ML_label_column]

    else:
      #note this scenario only occurs for classification
      #returns a single column with int entries
      #(moved the featureskey methods to __sequential_range_integer)
      df_train_filllabel, ML_cmnd = self.__convert_onehot_to_singlecolumn(df_train_filllabel, ML_cmnd, stringtype=False)
      ML_label_column = list(df_train_filllabel.columns)[0]

    if modeltype == 'classification':
      #convention is that regression will return labels with header of integer 0, classification with header of integer 1
      df_train_filllabel = df_train_filllabel.rename(columns = {ML_label_column:1})
      ML_label_column = 1

      #convert to fully represented sequential ranged integer as str(int) type
      df_train_filllabel, ML_cmnd = \
      self.__sequential_range_integer(df_train_filllabel, ML_cmnd, stringtype = True, featureskey=featureskey)

    #convert single column to a series
    if df_train_filllabel.shape[1] == 1:
      df_train_filllabel = df_train_filllabel[df_train_filllabel.columns[0]]
    
    #note that we know that ML_label_column won't overlap with df_train_filltrain headers
    #because df_train_filltrain headers are strings in suffix convention which include an underscore character
    #and labels header is integer 0
    #so if user wants to concatinate labels onto training set in their function it is ok
    
    #access any user passed parameters
    commands = {}
    if modeltype == 'classification':
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'customML_Classifier' in ML_cmnd['MLinfill_cmnd']:
          commands = ML_cmnd['MLinfill_cmnd']['customML_Classifier']
    if modeltype == 'regression':
      if 'MLinfill_cmnd' in ML_cmnd:
        if 'customML_Regressor' in ML_cmnd['MLinfill_cmnd']:
          commands = ML_cmnd['MLinfill_cmnd']['customML_Regressor']
    
    #train the model
    model = False
    if modeltype == 'classification':
      function_address = 'customML_Classifier_train'
    elif modeltype == 'regression':
      function_address = 'customML_Regressor_train'

    if 'customML' in ML_cmnd:
      if function_address in ML_cmnd['customML']:
        if callable(ML_cmnd['customML'][function_address]):
          # try:
          model = \
          ML_cmnd['customML'][function_address](df_train_filllabel, 
                                                df_train_filltrain, 
                                                columntype_report,
                                                commands, 
                                                randomseed)
          # except ValueError:
          #   pass
    
    postprocess_dict['customML_inference_support'].update(ML_cmnd['customML_inference_support'])
    
    return model, postprocess_dict

  def _predict_customML_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    modeltype = 'classification'
    return self.__predict_customML(ML_cmnd, model, fillfeatures, printstatus, categorylist, modeltype)

  def _predict_customML_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    modeltype = 'regression'
    return self.__predict_customML(ML_cmnd, model, fillfeatures, printstatus, categorylist, modeltype)

  def __predict_customML(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[], modeltype='regression'):
    """
    #wrapper for custom defined autoMLer inference functions 
    #either for classification or regression, which this function distinguishes with modeltype parameter
    
    #which custom defined functions are accepted in the form
    #def customML_predict_template(features, model):
    #  return infill
    
    #where features is a pandas dataframe matching the form of features passed to the corresponding training operation
    #and features will have at least one row
    #model is the model returned forom the corresponding training operation
    #(the model == False scenario won't call the custom function)
    
    #and the expectation is that infill will be derived by passing features to a model inference operation
    #and infill should be returned as either a single column pandas dataframe (column header is ignored)
    #or infill can also be returned as single column numpy array
    
    #where for the regression application the infill entries should be returned as float type
    #and for the classification application the infill entries can be returned as either int type or str(int) type
    #and more granular data type management will be conducted externally
    
    #note that pandas is available as pd and numpy as np
    #if imports were performed internal to customML_train_template they will need to be reinitilized in customML_predict_template
    
    #defaulttype is associated with use of internal default inference fucntions as alternate to user defined
    #and is accessed when user passes to ML_cmnd a string for the custon predict functions
           'customML' : {'customML_Classifier_train'  : customML_train_classifier, 
                         'customML_Classifier_predict': '(defaulttype)', 
                         'customML_Regressor_train'   : customML_train_regressor, 
                         'customML_Regressor_predict' : '(defaulttype)'}}
    #where the string '(defaulttype)' may be one of
    #{'tensorflow', 'xgboost', 'catboost', 'flaml', 'randomforest'}
    """

    if model is not False:
      
      #featurekey is a unique concatinated string of all column headers serving as basis for this model
      featureskey = ''
      for entry in fillfeatures.columns:
        featureskey += entry
      
      fillfeatures = fillfeatures.reset_index(drop=True)

      #access any user passed parameters
      commands = {}
      if modeltype == 'classification':
        if 'MLinfill_cmnd' in ML_cmnd:
          if 'customML_Classifier' in ML_cmnd['MLinfill_cmnd']:
            commands = ML_cmnd['MLinfill_cmnd']['customML_Classifier']
      if modeltype == 'regression':
        if 'MLinfill_cmnd' in ML_cmnd:
          if 'customML_Regressor' in ML_cmnd['MLinfill_cmnd']:
            commands = ML_cmnd['MLinfill_cmnd']['customML_Regressor']
      
      if modeltype == 'classification':
        function_address = 'customML_Classifier_predict'
      elif modeltype == 'regression':
        function_address = 'customML_Regressor_predict'
      
      #ML_cmnd['customML'][function_address] will either be populated with a function or a string
      #when populated with a string it is passed to defaulttype and used to access one of internal defined inference functions
      defaulttype = False
      if not callable(ML_cmnd['customML'][function_address]):
        defaulttype = ML_cmnd['customML'][function_address]
        
      #defaulttype is associated with use of internal default inference functions as alternate to user defined
      if defaulttype in {'tensorflow', 'xgboost', 'catboost', 
                         'flaml', 'randomforest'}:
        
        infill = self.__call_default_function(defaulttype, modeltype, fillfeatures, model, commands, ML_cmnd)
        
      #otherwise this is scenario when a string defaulttype was not defined, meaning we call the custom function
      elif defaulttype is False:
        
        if 'customML' in ML_cmnd:
          if function_address in ML_cmnd['customML']:
            if callable(ML_cmnd['customML'][function_address]):
              # try:
              infill = \
              ML_cmnd['customML'][function_address](fillfeatures, 
                                                    model,
                                                    commands)
              # except ValueError:
              #   infill = np.zeros(shape=(fillfeatures.shape[0],1))
              
      #infill is expected as a single column, as either a pandas dataframe, pandas series, or numpy array
      #we'll convert to a flattened array for common form
      if type(infill) == type(pd.DataFrame()) or type(infill) == type(pd.Series([1])):
        infill = infill.to_numpy()
        
      infill = infill.ravel()
              
      if modeltype == 'classification':
        
        #returned values from inference are accepted as either type int or type str(int)
        #categorylist is a list of headers corresponding to onehot form, where if this is a 1010 set will be list before converting back to binarized
        if len(categorylist) > 1:

          #this inverts any consolidations to sequential ranged integer from training
          infill = self.__sequential_range_integer_invert(infill, ML_cmnd, featureskey=featureskey)
          
          #this will return a onehot encoded array with 0/1 integer entries
          #note the featureskey method is now performed in __sequential_range_integer_invert
          infill = self.__convert_singlecolumn_to_onehot(infill, ML_cmnd, categorylist)

        else:
          
          #else for single column case if entries are str(int) we'll convert to int
          #so this could result in a boolean integer set or ordinal integer set depending on label composition
          infill = infill.astype(int)
        
    elif model is False:
      
      #note that infill is not inserted when model is False
      infill = np.zeros(shape=(1,len(categorylist)))
      
    return infill

  def __call_default_function(self, defaulttype, modeltype, fillfeatures, model, commands, ML_cmnd):
    """
    support function called in __predict_customML
    associated with cases where user passed ML_cmnd['customML'][function_address]
    as a string instead of a function
    this function uses that string (received as defaulttype) 
    to access one of the internal defined inference functions 
    which is applied to derive infill
    modeltype is expected as one of {'classification', 'regression'}
    """
    
    if defaulttype == 'tensorflow':
      if modeltype == 'classification':
        try:
          infill = \
          self.__customML_tensorflow_defaultpredict_classification(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))
      if modeltype == 'regression':
        try:
          infill = \
          self.__customML_tensorflow_defaultpredict_regression(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))

    if defaulttype == 'xgboost':
      if modeltype in {'classification', 'regression'}:
        try:
          infill = \
          self.__customML_xgboost_defaultpredict(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))

    if defaulttype == 'catboost':
      if modeltype in {'classification', 'regression'}:
        try:
          infill = \
          self.__customML_catboost_defaultpredict(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))

    if defaulttype == 'flaml':
      if modeltype in {'classification', 'regression'}:
        try:
          infill = \
          self.__customML_flaml_defaultpredict(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))

    if defaulttype == 'randomforest':
      if modeltype in {'classification', 'regression'}:
        try:
          infill = \
          self.__customML_randomforest_defaultpredict(fillfeatures, model, commands, ML_cmnd)
        except ValueError:
          infill = np.zeros(shape=(fillfeatures.shape[0],1))
          
    return infill

  #__FunctionBlock: customML default inference functions

  def __customML_tensorflow_defaultpredict_classification(self, features, model, commands, ML_cmnd):
    """
    #based on tensorflow, including conversion to tensor
    """

    #need to validate

    # #user can conduct this import externally to speed up this function
    # module = 'tensorflow'
    # if module not in sys.modules:
    import tensorflow as tf
    
    features = tf.convert_to_tensor(features.to_numpy())
    
    infill = model.predict(features)
    
    if len(infill.shape) == 0:
      pass
    
    elif len(infill.shape) == 1 or len(infill.shape) > 1 and infill.shape[1] == 1:
      #this works for binary classificaiton
      infill = np.rint(infill)
    
    else:
      #for multi label classificiton, assumes infill.shape[1] > 1 and len(infill.shape) == 2:
      
      #convert max entry in row to 1, other entries to 0
      infill2 = np.where(infill == np.max(infill, axis=1).reshape(-1,1), 1, 0)

      #for rows where the original derivation had all zeros recover that form
      infill2 = np.where(infill.sum(axis=1).reshape(-1,1) == 0, 0, infill2)
      
      #this translates to a single column with entries correpsonding to column index number of activation +1
      #as str(int) type and entry and '0' used as register for rows without activation
      #e.g. first column activations populated as '1', second as '2', etc. No activations populated as '0'
      #for rows where 1 is populated in every column (as would be case when all entries in row are nonzero and equal), 
      #the rightmost column is treated as the activation
      
      infill, ML_cmnd = self.__convert_onehot_to_singlecolumn(pd.DataFrame(infill2), ML_cmnd)
    
    return infill

  def __customML_tensorflow_defaultpredict_regression(self, features, model, commands, ML_cmnd):
    """
    #based on tensorflow, including conversion to tensor
    """

    #need to validate

    # #user can conduct this import externally to speed up this function
    # module = 'tensorflow'
    # if module not in sys.modules:
    import tensorflow as tf
    
    features = tf.convert_to_tensor(features.to_numpy())
    
    infill = model.predict(features)
    
    return infill

  def __customML_xgboost_defaultpredict(self, features, model, commands, ML_cmnd):
    """
    #based on xgboost XGBClassifier or XGBRegressor
    """

    # #user can conduct this import externally to speed up this function
    # module = 'xgboost'
    # if module not in sys.modules:
    from xgboost import XGBClassifier
    from xgboost import XGBRegressor
      
    infill = model.predict(features)

    return infill

  def __customML_catboost_defaultpredict(self, features, model, commands, ML_cmnd):
    """
    #based on catboost CatBoostClassifier or CatBoostRegressor
    """

    # #user can conduct this import externally to speed up this function
    # module = 'catboost'
    # if module not in sys.modules:
    from catboost import CatBoostClassifier
    from catboost import CatBoostRegressor
      
    infill = model.predict(features)

    return infill

  def __customML_flaml_defaultpredict(self, features, model, commands, ML_cmnd):
    """
    #based on flaml AutoML
    """

    # #user can conduct this import externally to speed up this function
    # module = 'flaml.automl'
    # if module not in sys.modules:
    from flaml import AutoML
      
    infill = model.predict(features)

    return infill

  def __customML_randomforest_defaultpredict(self, features, model, commands, ML_cmnd):
    """
    #scikit random forest classifier and regressor already imported
    """

    infill = model.predict(features)

    return infill

  #__FunctionBlock: data translations support

  def __convert_onehot_to_singlecolumn(self, df, ML_cmnd, stringtype = True):
    """
    #support function for autoML libraries that don't accept multicolumn labels
    #converts onehot encoded sets to single column
    #with entries corresponding to the column header
    #for cases where a row did not have an entry (such as all zeros)
    #we'll populate with -1
    #which since these are dervied from a numpy set won't overlap with headers
    
    #(featureskey workflow now takes place in __sequential_range_integer)
    """

    df2 = pd.DataFrame({-1:[-1]*df.shape[0]}, index=df.index)
    
    for column in df:
      df2 = \
      self.__autowhere(df2, -1, df[column]==1, column, specified='replacement')
      
    df2 = df2.rename(columns = {-1:'labels'})

    #this shifts register forward by 1, is associated with the -1 convention for onehot rows without activation
    #after this register shift entries are always non-negative
    df2 += 1

    #(previously featureskey workflow took place here)
      
    #this converts the integers to type str(int)
    if stringtype is True:
      df2['labels'] = df2['labels'].astype(int).astype(str)
        
    return df2, ML_cmnd

  def __sequential_range_integer(self, df2, ML_cmnd, stringtype = True, featureskey = ''):
    """
    #when featureskey is passed as a non empty string the conversion to sequential integers occurs
    
    #expects df2 as a single column ordinal encoded labels which may have potential gaps in encoding space
    #returns df compressed to a fully represented sequential range integer 
    
    #df2 is a dataframe with header of the integer 1
    """

    #the ordinal encoding may have gaps in the set of sequential ranged integers
    #some learning libraries need a fully represented set of integers from 0-max
    #so this option translates from the gapped representation to sequential fully represented
    if featureskey != '':
      
      seq_ordinal_dict = {}
      inverse_seq_ordinal_dict = {}
      original_labels = sorted(list(df2[1].unique()))
      
      #this tests for fully represented encoding space
      if len(set(df2[1].astype(int).unique()) - set(range(len(original_labels)))) != 0:

        sequential_labels = list(range(len(original_labels)))

        #'seq_ordinal_dict' : {original_label : sequential_label}
        seq_ordinal_dict = dict(zip(original_labels, sequential_labels))
        
        #'inverse_seq_ordinal_dict' : {sequential_label : original_label}
        inverse_seq_ordinal_dict = {value:key for key,value in seq_ordinal_dict.items()}
      
      if featureskey in ML_cmnd['customML_inference_support']:

        ML_cmnd['customML_inference_support'][featureskey].update({
          'seq_ordinal_dict' : seq_ordinal_dict,
          'inverse_seq_ordinal_dict' : inverse_seq_ordinal_dict,
        })
        
      #now apply the conversion to fully represented sequential ranged integers
      df2[1] = df2[1].replace(seq_ordinal_dict)
      
    if stringtype is True:
      
      df2[1] = df2[1].astype(str)
      
    return df2, ML_cmnd

  def __sequential_range_integer_invert(self, df, ML_cmnd, featureskey=''):
    """
    inverts to conversion to fully represented sequential ranged integer 
    that for training was performed in __sequential_range_integer
    expects df as a single column numpy array
    with entries of either int or str(int)
    and header of integer 0
    """

    df = pd.DataFrame(df)

    df[0] = df[0].astype(int)
    
    #if a sequential convert was applied for training it is now inverted
    #we have convention that featurekey='' signals no translation performed
    if featureskey != '':
      
      #this function called in both automunge and postmunge, 'automungeversion' present in postmunge
      if 'automungeversion' not in ML_cmnd \
      or 'automungeversion' in ML_cmnd and float(ML_cmnd['automungeversion']) >= 7.29:
        if featureskey in ML_cmnd['customML_inference_support']:

          inverse_seq_ordinal_dict = \
          ML_cmnd['customML_inference_support'][featureskey]['inverse_seq_ordinal_dict']
        
          df[0] = df[0].astype(int).replace(inverse_seq_ordinal_dict)
    
    return df

  def __convert_singlecolumn_to_onehot(self, df, ML_cmnd, columnslist):
    """
    #support function for autoML libraries that don't accept multicolumn labels
    #converts single column encoded sets back to onehot
    #with entries corresponding to the column header
    #where the entries will be
    #for cases where a row did not have an entry (such as all zeros)
    #we'll populate with -1
    #which since these are dervied from a numpy set won't overlap with headers

    #(featureskey workflow now takes place in __sequential_range_integer_invert)
    """

    df = pd.DataFrame(df)

    if 0 in df.columns:
      df[0] = df[0].astype(int)
      df = df.rename(columns = {0:'labels'})

    #(previously featureskey workflow took place here)

    if 'automungeversion' not in ML_cmnd \
    or 'automungeversion' in ML_cmnd and float(ML_cmnd['automungeversion']) >= 7.26:
      #this shifts the register back by 1, and results in -1 convention for onehot rows without activation
      df -= 1
    
    df2 = pd.DataFrame(np.zeros(shape = (df.shape[0], len(columnslist))), index=df.index)
    
    df2.columns = list(range(len(columnslist)))
    
    df = pd.concat([df, df2], axis=1)
    
    del df2
    
    for entry in list(range(len(columnslist))):

      df = \
      self.__autowhere(df, entry, df['labels'] == entry, 1, specified = 'replacement')
      
    del df['labels']

    df = df.to_numpy()
    
    return df

  def __convert_1010_to_onehot(self, df_array):  
    """
    takes as input dataframe encoded in 1010 format
    and translates to a one-hot encoding equivalent
    with number of columns based on 2^n where n is number of 1010 columns
    and returns a single activation in each row of the onehot form
    and in some cases returns columns with no activaitons
    """

    received_column_count = df_array.shape[1]
    
    df_onehot = pd.DataFrame(index=df_array.index)

    #initialize a column to store encodings
    df_onehot['-1'] = ''

    #populate column to store encodings 
    #this populates a set of strings composed of 1 and 0 characters
    for column in df_array.columns:
      df_onehot['-1'] = \
      df_onehot['-1'] + df_array[column].astype(int).astype(str)

    #create list of columns for the encoding with binary encodings
    #this will be full list of range of values based on number of 1010 columns
    #postprocess_textsupport  support function needs string headers
    #this relies on convention that received columns with suffix appenders have '_' included to ensure no overlap
    textcolumns = list(range(2**received_column_count))
    textcolumns = ['-1_' + str(format(item, f"0{received_column_count}b")) for item in textcolumns]

    df_onehot = \
    self.__onehot_support(df_onehot, '-1', scenario=2, activations_list = textcolumns)

    return df_onehot
  
  def __convert_onehot_to_1010(self, np_onehot):
    """
    takes as input numpy array encoded in one-hot format
    and translates to a 1010 encoding equivalent
    based on assumption that order of columns consistent per 
    convention of convert_1010_to_onehot(.)

    please note that this assumes the onehot form will have activations in each row
    which is consistent with our use for ML infill
    and otherwise populates rows without activations in the all zero bucket

    this is applied in context of infill conversions, 
    so sequential range integer index is ok (doesn't need to match index of another dataframe)
    """

    #create list of binary encodings corresponding to the onehot array
    #assumes consistent order of columns from convert_1010_to_onehot basis
    columnslist = list(range(np_onehot.shape[1]))
    columnslist = \
    [str(format(item, f"0{int(np.ceil(np.log2(np_onehot.shape[1])))}b")) for item in columnslist]

    #convert to dataframe with columnslist as column headers
    df_array = pd.DataFrame(np_onehot, columns = columnslist)

    #create new column to store encodings
    # df_array['1010'] = 0
    df_array.loc[:, '1010'] = 0

    #copy columns headers to activated cells, others are 0
    for column in df_array:

      if column != '1010':

        df_array[column].replace(1, column, inplace=True)

        df_array = \
        self.__autowhere(df_array, '1010', df_array[column] != 0, df_array[column], specified='replacement')

        del df_array[column]

    nbrcolumns = int(np.ceil(np.log2(np_onehot.shape[1])))
  
    #replace zeros with infill partition (a string of zeros of lenth nmbrcolumns)
    #note this corresponds to the default infill encoding for '1010'
    infill_plug = '0' * nbrcolumns

    df_array = \
    self.__autowhere(df_array, '1010', df_array.eq(0).all(1), infill_plug, specified='replacement')

    _1010_columns = []
    for i in range(nbrcolumns):
      _1010_columns.append('1010_'+str(i))

    df_array['1010'] = df_array['1010'].astype(str)

    #now let's store the encoding
    i=0
    for _1010_column in _1010_columns:

      df_array[_1010_column] = df_array['1010'].str.slice(i,i+1).astype(np.int8)

      i+=1

    del df_array['1010']

    np_1010 = df_array.to_numpy()

    return np_1010

  def __onehot_support(self, df, column, scenario=0, activations_list = []):
    """
    Receives a dataframe df and target column column and a scenario id
    Converts a single column of entries that may include numeric, string, and nan
    Into a one hot encoding returned in a seperate dataframe df2 with a matched index
    With columns in order of increasing numeric and then increasing string
    Without column or activation for nan
    This function returns a similar order of columns as would pd.get_dummies
    Although the sorted order of boolean entries is different
    The convention of a matched index is comparable
    Part of the reason for creating this function is so will have ability to experiment with variations
    For potential use in different transformation function scenarios
    
    scenario == 1 is for cases where activations_list is specified
    where activations_list is a list of entries that may be found in column
    and df2 is returned with columns matching activations_list with activations correpsonding to those entries
    with columns in the order of activations_list
    
    scenario == 2 is for cases where a list of returned columns is specified as activation_list
    structured as column + '_' + activation
    where first we'll derive similar to scenario 0 but without sorting
    and then rename columns to consistent form as activation_list and align composition and order
    """

    #current configuration is scenario 0
    if scenario == 0:

      #this is unique list in order of first occurance in set
      unique_list = list(df[column].unique())

      #to sort the unique list we'll seperate between numeric and string and sort each seperately

      #first populate a sorted list of numeric entries from the unique_list
      unique_list_numeric = [(isinstance(x, (int, float)), x) for x in unique_list]
      unique_list_numeric_two = []
      for type_tuple in unique_list_numeric:
        if type_tuple[0] and not isinstance(type_tuple[1], bool):
          unique_list_numeric_two.append(type_tuple[1])
      unique_list_numeric_two = sorted(unique_list_numeric_two)

      #now populate a sorted list of string entries from the unique_list
      unique_list_string = [(isinstance(x, (str)), x) for x in unique_list]
      unique_list_string_two = []
      for type_tuple in unique_list_string:
        if type_tuple[0]:
          unique_list_string_two.append(type_tuple[1])
      unique_list_string_two = sorted(unique_list_string_two)
      
      #now populate a sorted list of boolean entries
      unique_list_bool = [(isinstance(x, (bool)), x) for x in unique_list]
      unique_list_bool_two = []
      for type_tuple in unique_list_bool:
        if type_tuple[0]:
          unique_list_bool_two.append(type_tuple[1])
      unique_list_bool_two = sorted(unique_list_bool_two)

      #then combine the sorted numeric and sorted string
      unique_list = unique_list_numeric_two + unique_list_string_two + unique_list_bool_two

      #then apply the one-hot encoding
      #initalize df2
      df2 = pd.DataFrame(index=df.index)
      for entry in unique_list:
        #omit nan
        if entry == entry:
          #populate column of activations in df2
          df2 = \
          self.__autowhere(df2, entry, df[column] == entry, 1, specified='replacement')
    
    #current configuration is scenario 0
    if scenario == 1:
      
      df2 = pd.DataFrame(index=df.index)

      for activation in activations_list:
        if activation == activation:
          df2 = \
          self.__autowhere(df2, activation, df[column] == activation, 1, specified='replacement')
        else:
          df2 = \
          self.__autowhere(df2, activation, df[column].isna(), 1, specified='replacement')
          
    if scenario == 2:
      
      #this applies similar to scenario 0 but without the sorting
      unique_list = list(df[column].unique())
      
      #convert to string for this scenario
      unique_list = [str(x) for x in unique_list]
      
      #then apply the one-hot encoding
      #initalize df2
      df2 = pd.DataFrame(index=df.index)
      for entry in unique_list:
        #omit nan
        if entry == entry:
          #populate column of activations in df2
          df2 = \
          self.__autowhere(df2, entry, df[column].astype(str) == entry, 1, specified='replacement')
      
      #now translate columns to form of activation_list, i.e. column + '_' + str(activaiton)
      current_columns = list(df2.columns)
      returned_form_list = []
      translate_dict = {}
      for current_column in current_columns:
        returned_form = str(column) + '_' + str(current_column)
        returned_form_list.append(returned_form)
        translate_dict.update({current_column : returned_form})
        
      df2.rename(columns = translate_dict, inplace = True)
      
      #now populate any missing columns
      for activation_column in activations_list:
        if activation_column not in df2:
          # df2[activation_column] = 0
          # df2 = pd.concat([df2, pd.DataFrame({activation_column:np.zeros((df2.shape[0]))}, index=df2.index)], axis=1)
          df2.loc[:,activation_column] = 0
          
      #now reorder columns, this also drops columns not found in activation_list
      df2 = df2[activations_list]
    
    return df2

  #__FunctionBlock: oversampling support

  def __LabelSetGenerator(self, df, column, label):
    '''
    #LabelSetGenerator
    #takes as input dataframe for test set, label column name, and label
    #returns a dataframe set of all rows which included that label in the column
    '''

    df = df[df[column] == label]

    return df

  def __LabelFrequencyLevelizer(self, train_df, labels_df, \
                                postprocess_dict):
    """
    #LabelFrequencyLevelizer(.)
    #takes as input dataframes for train set, labels, and label category
    #combines them to single df, then creates sets for each label category
    #such as to add on multiples of each set to achieve near levelized
    #frequency of label occurence in training set (increases the size
    #of the training set by redundant inclusion of rows with lower frequency
    #labels.) Returns train_df, labels_df, trainID_df.
    #for now have convention that MLinfilltypes of 1010 or concurrent_act
    #not yet supported (future extension)

    #note that the methods take into account the MLinfilltype 
    #of the lblctgy entry (tree category) associated with label set root category
    """

    #find origcateogry of am_labels from FSpostprocess_dict
    labelcolumnkey = list(labels_df)[0]

    if labelcolumnkey in postprocess_dict['column_dict']:
      consolidatedcase = False
      origcolumn = postprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
      origcategory = postprocess_dict['column_dict'][labelcolumnkey]['origcategory']
      #find labelctgy from process_dict based on this origcategory
      labelscategory = postprocess_dict['process_dict'][origcategory]['labelctgy']
      #here we're inspecting MLinfilltype based on the labelctgy tree category
      MLinfilltype = postprocess_dict['process_dict'][labelscategory]['MLinfilltype']
    #else means the labels were prepared by a categoric consolidation
    else:
      consolidatedcase = True
      Binary_specification = postprocess_dict['labels_Binary_dict'][0]['Binary_specification']
      if Binary_specification in {True}:
        labelscategory = '1010'
        MLinfilltype = '1010'
      if Binary_specification in {'ordinal'}:
        labelscategory = 'ord3'
        MLinfilltype = 'singlct'
      if Binary_specification in {'onht'}:
        labelscategory = 'onht'
        MLinfilltype = 'multirt'

    #columns_labels may be reset for numeric labels, labels is fixed
    columns_labels = list(labels_df)
    labels = list(labels_df)
    #labels.sort()

    #markers to support numeric labels supplmented by bins
    multirt_append = False
    singlct_append = False

    if labels != []:

      setnameslist = []
      setlengthlist = []
      multiplierlist = []

      #this is the MLinfilltype of the labelctgy
      if MLinfilltype in {'numeric', 'integer'}:

        columns_labels = []
        for label in labels_df.columns:

          #the MLinfilltypes below are associated with tree categories for when labels are returned in multiple configurations

          #here we're checking if the column is a numeric set with aggregated bins
          #we'll only apply levelizer to one of a multirt or singlct set
          #whichever shows up first in the list of label columns
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][label]['category']]['MLinfilltype'] \
          in {'multirt'} \
          and singlct_append is False:
            multirt_append = True
            columns_labels.append(label)

          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][label]['category']]['MLinfilltype'] \
          in {'singlct', 'binary'} \
          and multirt_append is False:
            singlct_append = True
            columns_labels.append(label)

      #this is the MLinfilltype of the labelctgy
      if MLinfilltype in {'multirt'} \
      or MLinfilltype in {'numeric', 'integer'} and multirt_append is True:
        if columns_labels != []:
          
          if consolidatedcase is False:
            #we'll only apply to first multirt set in labels
            if multirt_append is False:
              for label in labels:
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][label]['category']]['MLinfilltype'] \
                in {'multirt'}:
                  columns_labels = postprocess_dict['column_dict'][label]['categorylist']
                  break
          elif consolidatedcase is True:
            columns_labels = postprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns']

          #note for. label smoothing activation values won't be 1, recomend supplementing transform with onehot set
          level_activation = 1

          i=0
          #for label in labels:
          for label in columns_labels:

            column = columns_labels[i]
            #derive set of labels dataframe for counting length
            df = self.__LabelSetGenerator(labels_df, column, level_activation)

            #append length onto list
            setlength = df.shape[0]

            #setlengthlist = setlengthlist.append(setlength)
            setlengthlist.append(setlength)

            i+=1

          #length of biggest label set
          maxlength = max(setlengthlist)

          #set counter to 0
          i = 0
          #for label in labels:
          for label in columns_labels:

            #derive multiplier to levelize label frequency
            setlength = setlengthlist[i]
            if setlength > 0:
              labelmultiplier = int(round(maxlength / setlength)) - 1
            else:
              labelmultiplier = 0
            #append multiplier onto list
            #multiplierlist = multiplierlist.append(labelmultiplier)
            multiplierlist.append(labelmultiplier)
            #increment counter
            i+=1

          #concatinate labels onto train set
          train_df = pd.concat([train_df, labels_df], axis=1)

          #reset counter
          i=0
          #for loop through labels

          #for label in labels:
          for label in columns_labels:

            #create train subset corresponding to label
            column = columns_labels[i]
            df = self.__LabelSetGenerator(train_df, column, level_activation)

            #set j counter to 0
            j = 0
            #concatinate an additional copy of the label set multiplier times
            while j < multiplierlist[i]:
              train_df = pd.concat([train_df, df], axis=0)
              #train_df = train_df.reset_index()
              j+=1

            i+=1

          #now seperate the labels df from the train df
          labels_df = train_df[labels]
          #now delete the labels column from train set
          train_df = train_df.drop(labels, axis=1)

      if MLinfilltype in {'singlct', 'binary'} \
      or MLinfilltype in {'numeric', 'integer'} and singlct_append is True:

        singlctcolumn = False

        if len(labels) == 1:
          singlctcolumn = labels[0]
        else:
          if consolidatedcase is False:
            for labelcolumn in labels:
              #levelizing based on first singlct column found in labels set
              if postprocess_dict['process_dict'][postprocess_dict['column_dict'][labelcolumn]['category']]['MLinfilltype'] \
              in {'singlct'}:
                singlctcolumn = labelcolumn
          if consolidatedcase is True:
            columns_labels = postprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns']

        uniquevalues = list(labels_df[singlctcolumn].unique())

        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:

          #value = 

          #derive set of labels dataframe for counting length
          df = self.__LabelSetGenerator(labels_df, singlctcolumn, label)

          #append length onto list
          setlength = df.shape[0]
          #setlengthlist = setlengthlist.append(setlength)
          setlengthlist.append(setlength)

        #length of biggest label set
        maxlength = max(setlengthlist)
        #set counter to 0
        i = 0
        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:
          #derive multiplier to levelize label frequency
          setlength = setlengthlist[i]
          if setlength > 0:

            labelmultiplier = int(round(maxlength / setlength)) - 1
          else:
            labelmultiplier = 0
          #append multiplier onto list
          #multiplierlist = multiplierlist.append(labelmultiplier)
          multiplierlist.append(labelmultiplier)
          #increment counter
          i+=1

        #concatinate labels onto train set
        train_df = pd.concat([train_df, labels_df], axis=1)

        #reset counter
        i=0
        #for loop through labels
        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:

          #create train subset corresponding to label
          df = self.__LabelSetGenerator(train_df, singlctcolumn, label)

          #set j counter to 0
          j = 0
          #concatinate an additional copy of the label set multiplier times
          while j < multiplierlist[i]:
            train_df = pd.concat([train_df, df], axis=0)
            #train_df = train_df.reset_index()
            j+=1

          i+=1

        #now seperate the labels df from the train df
        labels_df = pd.DataFrame(train_df[labels].copy())
        #now delete the labels column from train set
        for labelcolumn in labels:
          del train_df[labelcolumn]

    return train_df, labels_df

  #__FunctionBlock: automunge feature importance

  def __trainFSmodel(self, am_subset, am_labels, randomseed, \
                   process_dict, FSpostprocess_dict, labelctgy, FSML_cmnd, printstatus, 
                  logger,  FSautoMLer=False):
    """
    This function is used in feature importance evaluation
    And serves to translate a feature importance model to the conventions of ML infill
    So that we can use a common model training operation for both use cases with _predictinfill
    Thus ML_cmnd model training parameters, such as hyperparmeter tuning, are included
    Although basis exclusions are handled preceding this function call, 
    and include inspection of ML_cmnd['full_exclude'] and cases of MLinfilltype = 'totalexclude'
    since feature importance does not apply same approach for partitioning sets as ML infill
    """

    if len(list(am_labels)) > 0:

      if FSautoMLer is False:
        FSautoMLer = FSpostprocess_dict['autoMLer']

      df_train_fillfeatures_plug = pd.DataFrame(am_subset[:][:1].copy())
      df_test_fillfeatures_plug = pd.DataFrame(am_subset[:][:1].copy())
      categorylist = FSpostprocess_dict['column_dict'][list(am_labels)[0]]['categorylist']

      _infilla, _infillb, FSmodel, FSpostprocess_dict = \
      self.__predictinfill(categorylist[0], labelctgy, am_subset, am_labels, \
                         df_train_fillfeatures_plug, df_test_fillfeatures_plug, \
                         randomseed, FSpostprocess_dict, FSML_cmnd, FSautoMLer, printstatus, logger, \
                         categorylist = categorylist)

      del _infilla, _infillb
      
    else:
      
      FSmodel = False
    
    return FSmodel, FSpostprocess_dict
  
  def __createFSsets(self, am_subset, column, columnslist, randomseed):
    '''
    very simply shuffles rows of columns from columnslist with randomseed
    then returns the resulting dataframe
    
    hat tip for permutation method from "Beware Default Random Forest Importances"
    by Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard
    '''

    shuffleset = am_subset.copy()
    
    for clcolumn in columnslist:

      if clcolumn in shuffleset:
        
        #uses support function
        shuffleset = self.__df_shuffle_series(shuffleset, clcolumn, randomseed)
      
    return shuffleset

  def __createFSsets2(self, am_subset, column, columnslist, randomseed):
    '''
    similar to createFSsets except performed such as to only leave one column from
    the columnslist untouched and shuffle the rest 
    '''

    shuffleset2 = am_subset.copy()
    
    for clcolumn in columnslist:

      if clcolumn in shuffleset2:
        
        if clcolumn != column:
              
          #uses support function
          shuffleset2 = self.__df_shuffle_series(shuffleset2, clcolumn, randomseed)
    
    return shuffleset2

  def __shuffleaccuracy(self, np_shuffleset, np_labels, FSmodel, randomseed, label_categorylist, \
                      process_dict, labelctgy, postprocess_dict):
    '''
    measures accuracy of predictions of shuffleset (which had permutation method)
    against the model trained on the unshuffled set

    np_shuffleset and np_labels are now recast as pandas dataframe, leaving the "np" in place for convenience
    '''

    ML_cmnd = postprocess_dict['ML_cmnd']

    autoMLer = postprocess_dict['autoMLer']
    
    categorylist_for_predict = label_categorylist

    printstatus_for_predict = postprocess_dict['printstatus']

    #if autoML_type not specified than we'll apply default (randomforest)
    #note this is only a temporary update to ML_cmnd and is not returned from function call
    if 'autoML_type' not in postprocess_dict['ML_cmnd']:
      postprocess_dict['ML_cmnd'].update({'autoML_type' : 'randomforest'})
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = postprocess_dict['ML_cmnd']['autoML_type']

    labelscategory = labelctgy
    MLinfilltype = process_dict[labelscategory]['MLinfilltype']
    
    if MLinfilltype in {'numeric', 'concurrent_nmbr', 'integer'}:
      ML_application = 'regression'
    elif MLinfilltype in {'singlct', 'concurrent_ordl'}:
      ML_application = 'ordinalclassification'
    elif MLinfilltype in {'binary', 'concurrent_act'}:
      ML_application = 'booleanclassification'
    elif MLinfilltype in {'multirt', '1010'}:
      ML_application = 'onehotclassification'
    
    if MLinfilltype in {'numeric', 'concurrent_nmbr', 'integer'}:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #this absolute translation is to align with mean_squared_log_error use for performance metric
      #which is specific to feature importance

      #just in case this returned any negative predictions
      np_predictions = np.absolute(np_predictions)
      #and we're trying to generalize here so will go ahead and apply to labels
      np_labels = np.absolute(np_labels)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      #columnaccuracy = mean_squared_error(np_labels, np_predictions)
      #columnaccuracy = mean_squared_log_error(np_labels, np_predictions)
      columnaccuracy = 1 - mean_squared_log_error(np_labels, np_predictions)
      
    if MLinfilltype in {'singlct', 'binary', 'concurrent_ordl', 'concurrent_act'}:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      columnaccuracy = accuracy_score(np_labels, np_predictions)
      
    if MLinfilltype in {'multirt'}:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      columnaccuracy = accuracy_score(np_labels, np_predictions)

    if MLinfilltype in {'1010'}:
      
      np_labels = \
      self.__convert_1010_to_onehot(np_labels)
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      columnaccuracy = accuracy_score(np_labels, np_predictions)
        
    #I think this will clear some memory
    del np_labels, np_shuffleset
    
    return columnaccuracy
  
  def __assemblemadethecut(self, FScolumn_dict, featurethreshold, featureselection, \
                         am_subset_columns, FSprocess_dict):
    '''
    takes as input the FScolumn_dict and the passed automunge argument featurethreshold
    and a list of the columns from automunge application in featureselection
    and uses to assemble a list of columns that made it through the feature
    selection process
    
    returns list madethecut
    '''

    #create empty dataframe for sorting purposes
    FSsupport_df = pd.DataFrame(columns=['FS_column', 'metric', 'category'])
    
    #Future extension:
    #FSsupport_df = pd.DataFrame(columns=['FS_column', 'metric', 'metric2', 'category'])
    
    #add rows to the dataframe for each column
    for key in FScolumn_dict:
      
      column_df = pd.DataFrame([[key, FScolumn_dict[key]['metric'], FScolumn_dict[key]['category']]], \
                               columns=['FS_column', 'metric', 'category'])
  
      FSsupport_df = pd.concat([FSsupport_df, column_df], axis=0)
    
    #sort the rows by metric (from large to small, not that higher metric implies
    #more predictive power associated with that column's feature)
    #(note that NaN rows will have NaN values at bottom of list)
    FSsupport_df = FSsupport_df.sort_values(['metric'], ascending=False)
    
    #create list of candidate entries for madethecut
    candidates = list(FSsupport_df['FS_column'])
    
    #count the total number of rows
    totalrowcount =  FSsupport_df.shape[0]
    #count ranked rows
    metriccount = totalrowcount
    
    #create list of NArws
    #candidateNArws = candidates[-NaNcount:]
#     candidateNArws = list(FSsupport_df[FSsupport_df['category']=='NArw']['FS_column'])
    candidateNArws = list()
    
    #create list of feature rows
    #candidatefeaturerows = candidates[:-NaNcount]
#     candidatefeaturerows = list(FSsupport_df[FSsupport_df['category']!='NArw']['FS_column'])
    candidatefeaturerows = list(FSsupport_df['FS_column'])
    
#     #calculate the number of features we'll keep using the ratio passed from automunge
#     numbermakingcut = int(metriccount * featurepct)
      
    if featureselection is True:

      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = len(FSsupport_df)
    
    if featureselection == 'pct':

      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = int(metriccount * featurethreshold)
      
    if featureselection == 'metric':
      
      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = len(FSsupport_df[FSsupport_df['metric'] >= featurethreshold])
      
    if featureselection == 'report':
      #just a plug vlaue
      numbermakingcut = 1
      
    #generate list of rows making the cut
    madethecut = candidatefeaturerows[:numbermakingcut]
    
    #this is to retain full sets if any 1010 sets returned
    madethecut_copy = madethecut.copy()
    for entry in madethecut_copy:
      if FSprocess_dict[FScolumn_dict[entry]['category']]['MLinfilltype'] == '1010':
        if not set(FScolumn_dict[entry]['categorylist']).issubset(set(madethecut)):
          for entry2 in FScolumn_dict[entry]['categorylist']:
            if entry2 not in madethecut:
              madethecut.append(entry2)
    
    return madethecut

  def __featureselect(self, df_train, labels_column, trainID_column, \
                    powertransform, binstransform, randomseed, \
                    numbercategoryheuristic, assigncat, transformdict, \
                    processdict, featurethreshold, featureselection, \
                    ML_cmnd, process_dict, valpercent, printstatus, \
                    NArw_marker, assignparam, \
                    entropy_seeds, random_generator, sampling_dict, logger):
    """
    featureselect is a function called within automunge() that applies methods
    to evaluate predictive power of derived features towards a downstream model
    such as to trim the branches of the transform tree.
    
    The function returns a list of column names that "made the cut" so that
    automunge() can then remove extraneous branches.
    """

    #now we'll use automunge() to prepare the subset for feature evaluation
    #note the passed arguments, these are all intentional (no MLinfill applied,
    #primary goal here is to produce a processed dataframe for df_subset
    #with corresponding labels)
    
    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Begin Feature Importance evaluation")
    self.__autoprint('info', "")

    FSML_cmnd = self.__autocopy(ML_cmnd)

    FS_PCAn_components = False
    FS_Binary = False

    FS_assignparam = self.__autocopy(assignparam)

    totalvalidation = 0
    if isinstance(valpercent, float):
      totalvalidation = valpercent
    elif isinstance(valpercent, tuple):
      totalvalidation = valpercent[1] - valpercent[0]

    if totalvalidation == 0:
      totalvalidation = 0.2

    #following automunge call will return validation results in original bucket
    #which we'll store in featureselection_validations
    orig_validations_dict = self.__autocopy(logger_dict['validations'])
    logger_dict['validations'] = {'suffixoverlap_results':{}}

    am_train, _1, am_labels, \
    am_validation1, _2, am_validationlabels1, \
    _3, _4, _5, \
    FSpostprocess_dict = \
    self.automunge(df_train, df_test = False, labels_column = labels_column, trainID_column = trainID_column, \
                  testID_column = False, valpercent = totalvalidation, \
                  shuffletrain = True, TrainLabelFreqLevel = False, powertransform = powertransform, \
                  binstransform = binstransform, MLinfill = False, infilliterate=1, randomseed = randomseed, \
                  excl_suffix = True, \
                  numbercategoryheuristic = numbercategoryheuristic, pandasoutput = 'dataframe', NArw_marker = NArw_marker, \
                  featureselection = False, \
                  PCAn_components = FS_PCAn_components, \
                  Binary = FS_Binary, \
                  ML_cmnd = FSML_cmnd, assigncat = assigncat, \
                  assigninfill = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[], \
                                  'adjinfill':[], 'meaninfill':[], 'medianinfill':[]}, \
                  assignparam = FS_assignparam, entropy_seeds = entropy_seeds, random_generator = random_generator, sampling_dict = sampling_dict, \
                  transformdict = transformdict, processdict = processdict, printstatus=printstatus, logger=logger)

    #record validation results from automunge call internal to featureselect
    #logger_dict validation results stored in logger_dict['featureselection_validations']
    logger_dict['featureselection_validations'] = self.__autocopy(logger_dict['validations'])
    logger_dict['validations'] = {'suffixoverlap_results':{}}
    logger_dict['validations'] = self.__autocopy(orig_validations_dict)

    #automunge will have struck any customML training functions for different reason, reintroduce for training feture importance model
    if 'customML' in ML_cmnd and 'customML_Classifier_train' in ML_cmnd['customML'] \
    and 'customML' in FSpostprocess_dict['autoMLer']:
      FSpostprocess_dict['ML_cmnd']['customML']['customML_Classifier_train'] = ML_cmnd['customML']['customML_Classifier_train']
      FSpostprocess_dict['ML_cmnd']['customML']['customML_Regressor_train'] = ML_cmnd['customML']['customML_Regressor_train']

      temp_autoMLer = self.__assemble_autoMLer()
      for modeltarget in temp_autoMLer['customML']:
        FSpostprocess_dict['autoMLer']['customML'][modeltarget]['train'] = \
        temp_autoMLer['customML'][modeltarget]['train']

    #capture any ML_cmnd preprocessing returned from automunge
    FSML_cmnd = FSpostprocess_dict['ML_cmnd']

    #__

    #we'll remove columns from ML_cmnd['full_exclude'] or with MLinfilltype == 'totalexclude'
    #using a simpler method than applied in ML infill for this purpose since we're only training one model
    full_exclude_specified = []
    if 'full_exclude' in FSML_cmnd:
      full_exclude_specified = FSML_cmnd['full_exclude']
      
    totalexclude_MLinfilltype = []
    for column_dict_entry in FSpostprocess_dict['column_dict']:
      column_dict_entry_category = FSpostprocess_dict['column_dict'][column_dict_entry]['category']
      column_dict_entry_category_MLinfilltype = FSpostprocess_dict['process_dict'][column_dict_entry_category]['MLinfilltype']
      if column_dict_entry_category_MLinfilltype == 'totalexclude':
        totalexclude_MLinfilltype.append(column_dict_entry)
        
    nonnumeric_columns = full_exclude_specified + totalexclude_MLinfilltype
    #convert to returtned header format
    nonnumeric_columns = self.__column_convert_support(nonnumeric_columns, FSpostprocess_dict, convert_to='returned')

    nonnumeric_columns = list(set(nonnumeric_columns) & set(am_train))
    
    #now drop any potentially nonnumeric columns
    am_train = am_train.drop(nonnumeric_columns, axis=1)
    am_validation1 = am_validation1.drop(nonnumeric_columns, axis=1)

    #__

    #this is the returned process_dict
    #(remember "processdict" is what we pass to automunge() call, "process_dict" is what is 
    #assembled inside automunge, there is a difference)
    FSprocess_dict = FSpostprocess_dict['process_dict']

    if am_labels.empty is True:
      FSmodel = False

      baseaccuracy = False

      logger_dict['validations'].update({'FS_numeric_data_result': False})
      logger_dict['validations'].update({'FS_all_valid_entries_result': False})

      returned_label_set_for_featureselect_valresult = True
      logger_dict['validations'].update({'returned_label_set_for_featureselect_valresult' : returned_label_set_for_featureselect_valresult})

      logger = self.__logger_append(logger)
      
      #printout display progress
      # if printstatus != 'silent':
      self.__autoprint('warning', "_______________")
      self.__autoprint('warning', "No labels returned from automunge(.), Feature Importance halted")
      self.__autoprint('warning', "")

    #if am_labels is not an empty set
    if am_labels.empty is False:

      #find origcateogry of am_labels from FSpostprocess_dict
      labelcolumnkey = list(am_labels)[0]
      if labelcolumnkey in FSpostprocess_dict['column_dict']:
        origcolumn = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
        origcategory = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcategory']

        #find labelctgy from process_dict based on this origcategory
        labelctgy = FSprocess_dict[origcategory]['labelctgy']

        am_categorylist = []

        for am_label_column in am_labels.columns:

          if am_label_column in FSpostprocess_dict['column_dict'] \
          and FSpostprocess_dict['column_dict'][am_label_column]['category'] == labelctgy:

            am_categorylist = FSpostprocess_dict['column_dict'][am_label_column]['categorylist']

            #we'll follow convention that if target label category MLinfilltype is concurrent
            #we'll arbitrarily take the first column and use that as target
            if FSpostprocess_dict['process_dict'][labelctgy]['MLinfilltype'] \
            in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

              am_categorylist = [am_categorylist[0]]

            break
            
      #same as if labelcolumnkey not in FSpostprocess_dict['column_dict']:
      #only other scenario for labels is column originated from categoric consolidation
      elif len(FSpostprocess_dict['labels_Binary_dict']) > 0:
        
        #for categoric consolidated labels, we are going to edit the column_dict and process_dict
        #as a hack to enable __trainFSmodel support
        #noting that in so doing there is a remote edge case of process_dict edit interfering with user overwrite to processdict
        #so we have some additional accomodation there as well
        
        am_label_column = FSpostprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns'][0]
        
        am_categorylist = FSpostprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns']
        
        labelctgy = FSpostprocess_dict['labels_Binary_dict'][0]['Btype']
          
        #since process_dict only returns entries applied for transformations, there is chance a Binary transform isn't included
        #so as part of the hack we'll add the associated Binary transform for purposes of inspecting the MLinfilltype
        #FSpostprocess_dict['process_dict'] will only be inspected for MLinfilltype in __trainFSmodel and __shuffleaccuracy
        
        #in cases where labelctgy is already an entry in process_dict we'll add suffix until unique
        remote_overwrite_edge_case_marker = False
        if labelctgy in FSpostprocess_dict['process_dict']:
          remote_overwrite_edge_case_marker = True
        
        if labelctgy == 'ord3':
          
          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in FSpostprocess_dict['process_dict']:
              labelctgy += 'B'
          
          FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'singlct'}})
          
        if labelctgy == 'onht':
          
          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in FSpostprocess_dict['process_dict']:
              labelctgy += 'B'
          
          FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'multirt'}})
          
        if labelctgy == '1010':
          
          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in FSpostprocess_dict['process_dict']:
              labelctgy += 'B'
          
          FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : '1010'}})
        
        #this is a hack to support __trainFSmodel with categoric consolidations
        FSpostprocess_dict['column_dict'].update({am_label_column : {'categorylist' : am_categorylist,
                                                                     'columnslist' : list(am_labels), 
                                                                     'category' : labelctgy,
                                                                     'origcategory' : labelctgy}})
          
      #____________________
          
      if len(am_categorylist) == 0:

        labelctgy_not_found_in_familytree_valresult = True
        logger_dict['validations'].update({'labelctgy_not_found_in_familytree_valresult' : labelctgy_not_found_in_familytree_valresult})
        logger = self.__logger_append(logger)

        # if printstatus != 'silent':
        #this is a remote edge case, printout added for troubleshooting support
        self.__autoprint('warning', "Label root category processdict entry contained a labelctgy entry not found in family tree")
        self.__autoprint('warning', "Feature Selection model training will not run without valid labelgctgy processdict entry")
        self.__autoprint('warning', '')

      elif len(am_categorylist) == 1:
        am_labels = pd.DataFrame(am_labels[am_categorylist[0]])
        am_validationlabels1 = pd.DataFrame(am_validationlabels1[am_categorylist[0]])

      else:
        am_labels = am_labels[am_categorylist]
        am_validationlabels1 = am_validationlabels1[am_categorylist]

      #if there's a bug occuring after this point it might mean the labelctgy wasn't
      #properly populated in the process_dict for the root category assigned to the labels
      #again the labelctgy entry to process_dict represents for labels returned in 
      #multiple configurations the trasnofrmation category whose returned set will be
      #used to train the feature selection model

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Training feature importance evaluation model")
      self.__autoprint('debug', "")

      #first validate that data is all valid numeric
      FS_numeric_data_result, FS_all_valid_entries_result = \
      self.__validate_allvalidnumeric(am_train, printstatus)

      logger_dict['validations'].update({'FS_numeric_data_result': FS_numeric_data_result})
      logger_dict['validations'].update({'FS_all_valid_entries_result': FS_all_valid_entries_result})
      logger = self.__logger_append(logger)

      #apply function trainFSmodel

      #'customML_inference_support' for feature selection is a little tricky since it is being used in different context than ML infill
      FSpostprocess_dict['customML_inference_support'] = {}

      FSmodel, FSpostprocess_dict = \
      self.__trainFSmodel(am_train, am_labels, randomseed, \
                        FSprocess_dict, FSpostprocess_dict, labelctgy, FSML_cmnd, \
                        printstatus, logger)

      #now carry the 'customML_inference_support' from FSpostprocess_dict to FSpostprocess_dict['ML_cmnd']
      #this means that can't currently do ML infill on automunge call to prepare feature selction data as would overwrite
      #which has been our basis anyway'
      #basically the inference functions are inspecting customML_inference_support in ML_cmnd
      FSpostprocess_dict['ML_cmnd']['customML_inference_support'] = FSpostprocess_dict['customML_inference_support']
      
      if FSmodel is False:
        
        FScolumn_dict = {}
        
        FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

        baseaccuracy = False
        
        #printout display progress
        # if printstatus != 'silent':
        self.__autoprint('warning', "_______________")
        self.__autoprint('warning', "No model returned from training, Feature Importance halted")
        self.__autoprint('warning', "")
        
        featureselect_trained_model_valresult = True
        logger_dict['validations'].update({'featureselect_trained_model_valresult' : featureselect_trained_model_valresult})
        logger = self.__logger_append(logger)
      
      elif FSmodel is not False:

        #update v2.11 baseaccuracy should be based on validation set
        baseaccuracy = self.__shuffleaccuracy(am_validation1, am_validationlabels1, \
                                            FSmodel, randomseed, am_categorylist, \
                                            FSprocess_dict, labelctgy, FSpostprocess_dict)

        # if printstatus is True:
        self.__autoprint('debug', "Base Accuracy of feature importance model:")
        self.__autoprint('debug', baseaccuracy)
        self.__autoprint('debug', '')

        #get list of columns
        am_train_columns = list(am_train)

        #initialize dictionary FScolumn_dict = {}
        FScolumn_dict = {}
        
        FS_origcolumns = list(FSpostprocess_dict['origcolumn'])
        
        #assemble FScolumn_dict to support the feature evaluation
        for column in am_train_columns:
          
          #Binary and PCA consolidated columns don't get evaluated
          if column in FSpostprocess_dict['column_dict']:

            #pull categorylist, category, columnslist
            categorylist = FSpostprocess_dict['column_dict'][column]['categorylist']
            category = FSpostprocess_dict['column_dict'][column]['category']
            columnslist = FSpostprocess_dict['column_dict'][column]['columnslist']
            origcolumn = FSpostprocess_dict['column_dict'][column]['origcolumn']

            #create entry to FScolumn_dict
            FScolumn_dict.update({column : {'categorylist' : categorylist, \
                                            'category' : category, \
                                            'columnslist' : columnslist, \
                                            'origcolumn' : origcolumn, \
                                            'FScomplete' : False, \
                                            'shuffleaccuracy' : None, \
                                            'shuffleaccuracy2' : None, \
                                            'baseaccuracy' : baseaccuracy, \
                                            'metric' : None, \
                                            'metric2' : None}})
          
        #this is for assemblemadethecut
        FSprocess_dict = FSpostprocess_dict['process_dict']

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Evaluating feature importances")
        self.__autoprint('debug', "")

        #perform feature evaluation on each column
        for column in am_train_columns:

          if column not in nonnumeric_columns and column in FScolumn_dict:

            if FScolumn_dict[column]['FScomplete'] is False:

              columnslist = FScolumn_dict[column]['columnslist']

              #create set with columns shuffle from columnslist
              #shuffleset = self.__createFSsets(am_train, column, categorylist, randomseed)
              #shuffleset = self.__createFSsets(am_train, column, columnslist, randomseed)
              shuffleset = self.__createFSsets(am_validation1, column, columnslist, randomseed)

              #determine resulting accuracy after shuffle
              columnaccuracy = self.__shuffleaccuracy(shuffleset, am_validationlabels1, \
                                                    FSmodel, randomseed, am_categorylist, \
                                                    FSprocess_dict, labelctgy, FSpostprocess_dict)

              #I think this will clear some memory
              del shuffleset

              #category accuracy penalty metric
              metric = baseaccuracy - columnaccuracy
              #metric2 = baseaccuracy - columnaccuracy2

              #save accuracy to FScolumn_dict and set FScomplete to True
              #(for each column in the categorylist)
              #for categorycolumn in FSpostprocess_dict['column_dict'][column]['categorylist']:
              for categorycolumn in FSpostprocess_dict['column_dict'][column]['columnslist']:

                if categorycolumn not in nonnumeric_columns:

                  FScolumn_dict[categorycolumn]['FScomplete'] = True
                  FScolumn_dict[categorycolumn]['shuffleaccuracy'] = columnaccuracy
                  FScolumn_dict[categorycolumn]['metric'] = metric
                  #FScolumn_dict[categorycolumn]['shuffleaccuracy2'] = columnaccuracy2
                  #FScolumn_dict[categorycolumn]['metric2'] = metric2

            columnslist = FScolumn_dict[column]['columnslist']

            #create second set with all but one columns shuffled from columnslist
            #this will allow us to compare the relative importance between columns
            #derived from the same parent
            #shuffleset2 = self.__createFSsets2(am_train, column, columnslist, randomseed)
            shuffleset2 = self.__createFSsets2(am_validation1, column, columnslist, randomseed)

            #determine resulting accuracy after shuffle
            columnaccuracy2 = self.__shuffleaccuracy(shuffleset2, am_validationlabels1, \
                                                  FSmodel, randomseed, am_categorylist, \
                                                  FSprocess_dict, labelctgy, FSpostprocess_dict)

            metric2 = baseaccuracy - columnaccuracy2

            FScolumn_dict[column]['shuffleaccuracy2'] = columnaccuracy2
            FScolumn_dict[column]['metric2'] = metric2
        
        madethecut = self.__assemblemadethecut(FScolumn_dict, featurethreshold, \
                                          featureselection, am_train_columns, FSprocess_dict)
  
  #if the only column left in madethecut from origin column is a NArw, delete from the set
  #(this is going to lean on the column ID string naming conventions)
  #couldn't get this to work, this functionality a future extension
#     trimfrommtc = []
#     for traincolumn in list(df_train):
#       if (traincolumn + '_') not in [checkmtc[:(len(traincolumn)+1)] for checkmtc in madethecut]:
#         for mtc in madethecut:
#           #if mtc originated from traincolumn
#           if mtc[:(len(traincolumn)+1)] == traincolumn + '_':
#             #count the number of same instance in madethecut set
#             madethecut_trim = [mdc_trim[:(len(traincolumn)+1)] for mdc_trim in madethecut]
#             if madethecut_trim.count(mtc[:(len(traincolumn)+1)]) == 1 \
#             and mtc[-5:] == '_NArw':
#               trimfrommtc = trimfrommtc + [mtc]
#     madethecut = list(set(madethecut).difference(set(trimfrommtc)))
      
  #apply function madethecut(FScolumn_dict, featurepct)
  #return madethecut
  #where featurepct is the percent of features that we intend to keep
  #(might want to make this a passed argument from automunge)
  
      #I think this will clear some memory
      del am_train, _1, am_labels, am_validation1, _2, \
      am_validationlabels1, \
      _3, _4, _5,  \
      FSpostprocess_dict

      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Feature Importance results:")
      self.__autoprint('debug', "")

      #to inspect values returned in featureimportance object one could run
      # if printstatus is True:
      for keys,values in FScolumn_dict.items():
        self.__autoprint('debug', keys)
        self.__autoprint('debug', 'metric = ', values['metric'])
        self.__autoprint('debug', 'metric2 = ', values['metric2'])
        self.__autoprint('debug', "")

    FS_sorted = {'baseaccuracy' : baseaccuracy, \
                 'metric_key':{}, \
                 'column_key':{}, \
                 'metric2_key':{}, \
                 'metric2_column_key':{}}
    
    #first we'll handle first metric based on source column
    for FS_origcolumn in FS_origcolumns:
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric'] in FS_sorted['metric_key']:
            if isinstance(FS_sorted['metric_key'][FScolumn_dict[key]['metric']], list):
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
            else:
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']] = \
              [FS_sorted['metric_key'][FScolumn_dict[key]['metric']]]
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
          else:
            FS_sorted['metric_key'].update({FScolumn_dict[key]['metric'] : [FS_origcolumn]})
          break
      
    FS_sorted['metric_key'] = dict(sorted(FS_sorted['metric_key'].items(), reverse=True))
    
    for key in FS_sorted['metric_key']:
      for entry in FS_sorted['metric_key'][key]:
        entry_index = FS_sorted['metric_key'][key].index(entry)
        FS_sorted['column_key'].update({FS_sorted['metric_key'][key][entry_index] : key})
    
    #now for metric2 based on derived columns relative importance, note sorted in other order
    for FS_origcolumn in FS_origcolumns:
      FS_sorted['metric2_key'].update({FS_origcolumn : {}})
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric2'] in FS_sorted['metric2_key'][FS_origcolumn]:
            if isinstance(FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']], list):
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
            else:
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']] = \
              [FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']]]
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
          else:
            FS_sorted['metric2_key'][FS_origcolumn].update({FScolumn_dict[key]['metric2'] : [key]})
    
    for key in FS_sorted['metric2_key']:
      FS_sorted['metric2_key'][key] = dict(sorted(FS_sorted['metric2_key'][key].items(), reverse=False))
    
    for key1 in FS_sorted['metric2_key']:
      FS_sorted['metric2_column_key'].update({key1 : {}})
      for key2 in FS_sorted['metric2_key'][key1]:
        for entry in FS_sorted['metric2_key'][key1][key2]:
          entry_index = FS_sorted['metric2_key'][key1][key2].index(entry)
          FS_sorted['metric2_column_key'][key1].update({FS_sorted['metric2_key'][key1][key2][entry_index] : key2})
    
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', '')
    self.__autoprint('info', "______________________")
    self.__autoprint('info', "sorted metric results:")
    self.__autoprint('info', '')
    for keys,values in FS_sorted['metric_key'].items():
      for entry in values:
        self.__autoprint('info', entry)
        self.__autoprint('info', keys)
        self.__autoprint('info', '')
    self.__autoprint('info', "______________________")
    self.__autoprint('info', "sorted metric2 results:")
    self.__autoprint('info', '')
    for key in FS_sorted['metric2_key']:
      self.__autoprint('info', "for source column: ", key)
      for keys,values in FS_sorted['metric2_key'][key].items():
        for entry in values:
          self.__autoprint('info', entry)
          self.__autoprint('info', keys)
          self.__autoprint('info', '')
      self.__autoprint('info', '')
    
    if FSmodel is False:
      
      madethecut = []
      FScolumn_dict = {}
    
    #printout display progress
    # if printstatus is True:
    self.__autoprint('debug', "_______________")
    self.__autoprint('debug', "Feature Importance evaluation complete")
    self.__autoprint('debug', "")
    
    return madethecut, FSmodel, FScolumn_dict, FS_sorted

  #__FunctionBlock: assigninfill support functions
  
  def __assemblepostprocess_assigninfill(self, assigninfill, infillcolumns_list, 
                                       columns_train, postprocess_dict, MLinfill):
    """
    #this function converts user passed assigninfill
    #into a collection of post-transform column assignments
    #to various infill methods for application in apply infill functions
    
    #assigninfill is as passed by user
    #(note assigninfill previously had validations performed in check_assigninfill)
    #infillcolumns_list is all dervied columns in train set
    #columns_train is all source columns in train set
    #postprocess_dict is how data shared between functions
    #MLinfill is boolean marker for default MLinfill applciation
    
    #The convention is that unspecified columns are cast to
    #stndrdinfill or MLinfill based on MLinfill parameter
    #The other convention is that user may assign column headers
    #both with and without suffix appenders
    #the source column headers are converted to set of dervied column headers
    #and then any user specified derived columns w/ suffix take precendence over the converted ones
    
    #so workflow is as follows
    #- received spec'd assigninfill (which may include both pre-suf and w/-suf)
    #- aggregate just spec'd w/-suf into mirror spec assigninfill_withsuffix
    #- aggregate spec'd pre-suf into mirror spec assigninfill_sourcecolumn
    #- identify unspecified source columns missing from assigninfill_sourcecolumn,
    #add as new category assigninfill_sourcecolumn['unspecified']
    #- convert source columns to dervied columns from assigninfill_sourcecolumn
    #to assigninfill_sourcecolumn_converted
    #- if assigninfill_sourcecolumn_converted doesn't yet have entries for stdrdinfill or MLinfill, create
    #- based on MLinfill, copy entries from assigninfill_sourcecolumn_converted['unspecified']
    #into either 'stdrdinfill' or 'MLinfill'
    #(we'll keep 'unspecified' entry in case might be of use down the road)
    #- for duplicates between entries to assigninfill_withsuffix and assigninfill_sourcecolumn_converted
    #assigninfill_withsuffix takes precedence
    #so cycle through and if duplicates found remove from assigninfill_sourcecolumn_converted
    #- combine assigninfill_withsuffix and assigninfill_sourcecolumn_converted into
    #the returned set postprocess_assigninfill_dict
    #- insert any missing keys needed for apply_am_infill
    #- return postprocess_assigninfill_dict
    """

    #this just makes searching a little more efficient
    infillcolumns_list = set(infillcolumns_list)
    columns_train = set(columns_train)
    
    #received spec'd assigninfill may include both pre-suffix and w/-suffix

    #instead of converting all specs to returned form, we'll segregate returned suffix columns from returned input columns
    #and then convert the input columns to the returned column convention
    #and for cases of redundant specification, between specified input and specified returned convention, the returned one takes precedent
    
    #- aggregate just spec'd w/-suf into mirror spec assigninfill_withsuffix
    assigninfill_withsuffix = {}
    for key in assigninfill:
      assigninfill_withsuffix.update({key:[]})
      for entry in assigninfill[key]:
        if entry in infillcolumns_list:
          assigninfill_withsuffix[key].append(entry)
          
    #- aggregate spec'd pre-suf into mirror spec assigninfill_sourcecolumn
    assigninfill_sourcecolumn = {}
    for key in assigninfill:
      assigninfill_sourcecolumn.update({key:[]})
      for entry in assigninfill[key]:
        if entry in columns_train:
          assigninfill_sourcecolumn[key].append(entry)
          
    #- identify unspecified source columns missing from assigninfill_sourcecolumn,
    #add as new category assigninfill_sourcecolumn['unspecified']
    specd_sourcecolumns = []
    for key in assigninfill_sourcecolumn:
      specd_sourcecolumns += assigninfill_sourcecolumn[key]
    unspecd_sourcecolumns = list(set(columns_train) - set(specd_sourcecolumns))
    assigninfill_sourcecolumn.update({'unspecified':unspecd_sourcecolumns})
    
    #- convert source columns to dervied columns from assigninfill_sourcecolumn
    #to assigninfill_sourcecolumn_converted
    assigninfill_sourcecolumn_converted = {}
    for key in assigninfill_sourcecolumn:
      assigninfill_sourcecolumn_converted.update({key:[]})
      for entry in assigninfill_sourcecolumn[key]:
        #accessing dervied columns from source column, 
        #adding as entries to assigninfill_sourcecolumn_converted[key]
        assigninfill_sourcecolumn_converted[key] += postprocess_dict['origcolumn'][entry]['columnkeylist']
        
    #- if assigninfill_sourcecolumn_converted doesn't yet have entries for stdrdinfill or MLinfill, create
    if 'stdrdinfill' not in assigninfill_sourcecolumn_converted:
      assigninfill_sourcecolumn_converted.update({'stdrdinfill':[]})
    if 'MLinfill' not in assigninfill_sourcecolumn_converted:
      assigninfill_sourcecolumn_converted.update({'MLinfill':[]})
    
    #- based on MLinfill, copy entries from assigninfill_sourcecolumn_converted['unspecified']
    #into either 'stdrdinfill' or 'MLinfill'
    #(we'll keep 'unspecified' entry in case might be of use down the road)
    if MLinfill is True:
      assigninfill_sourcecolumn_converted['MLinfill'] += assigninfill_sourcecolumn_converted['unspecified']
    else:
      assigninfill_sourcecolumn_converted['stdrdinfill'] += assigninfill_sourcecolumn_converted['unspecified']
      
    #- for duplicates between entries to assigninfill_withsuffix and assigninfill_sourcecolumn_converted
    #assigninfill_withsuffix takes precedence
    #so cycle through and if duplicates found remove from assigninfill_sourcecolumn_converted
    all_specd_withsuffix = []
    for key in assigninfill_withsuffix:
      all_specd_withsuffix += assigninfill_withsuffix[key]
    for key in assigninfill_sourcecolumn_converted:
      assigninfill_sourcecolumn_converted_key_copy = self.__autocopy(assigninfill_sourcecolumn_converted[key])
      for entry in assigninfill_sourcecolumn_converted_key_copy:
        if entry in all_specd_withsuffix:
          assigninfill_sourcecolumn_converted[key].remove(entry)
          
    #- combine assigninfill_withsuffix and assigninfill_sourcecolumn_converted into
    #the returned set postprocess_assigninfill_dict
    
    #first let's make sure they have equivalent keys
    for key1 in assigninfill_withsuffix:
      if key1 not in assigninfill_sourcecolumn_converted:
        assigninfill_sourcecolumn_converted.update({key1:[]})
    for key2 in assigninfill_sourcecolumn_converted:
      if key2 not in assigninfill_withsuffix:
        assigninfill_withsuffix.update({key2:[]})
    
    #ok now populate 
    postprocess_assigninfill_dict = {}
    
    for key in assigninfill_sourcecolumn_converted:
      postprocess_assigninfill_dict.update({key: assigninfill_withsuffix[key] + assigninfill_sourcecolumn_converted[key]})
    
    #- insert any missing keys needed for apply_am_infill
    if 'stdrdinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['stdrdinfill'] = []
    
    if 'zeroinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['zeroinfill'] = []

    if 'negzeroinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['negzeroinfill'] = []

    if 'oneinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['oneinfill'] = []
      
    if 'naninfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['naninfill'] = []

    if 'adjinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['adjinfill'] = []

    if 'interpinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['interpinfill'] = []

    if 'medianinfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['medianinfill'] = []

    if 'meaninfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['meaninfill'] = []

    if 'modeinfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['modeinfill'] = []
      
    if 'lcinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['lcinfill'] = []
      
    if 'MLinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['MLinfill'] = []
    
    #- return postprocess_assigninfill_dict
    return postprocess_assigninfill_dict

  def __assemble_sorted_columns_by_NaN_dict(self, masterNArows_train, list_df_train, postprocess_dict):
    """
    #assembles a dictionary with returned columns as key and missing data count as value
    #used in apply_am_infill
    """

    sorted_columns_by_NaN_dict = {}

    #assemble dictionary
    for returned_column in list_df_train:
      #if next line returns an error it might be because you incorrectly specified an inplace_option in process_dict
      orig_column = postprocess_dict['column_dict'][returned_column]['origcolumn']
      nancount = masterNArows_train[orig_column + '_NArows'].sum()
      sorted_columns_by_NaN_dict.update({returned_column : nancount})

    #now sort by values
    #first sort by column header to ensure grouping coherence originating from same source column
    sorted_columns_by_NaN_dict = \
    dict(sorted(sorted_columns_by_NaN_dict.items(), key=lambda item: item[0]))
      
    #then sort by nan count from most to least
    sorted_columns_by_NaN_dict = \
    dict(sorted(sorted_columns_by_NaN_dict.items(), reverse=True, key=lambda item: item[1]))

    #return a list
    sorted_columns_by_NaN_list = \
    list(sorted_columns_by_NaN_dict)

    return sorted_columns_by_NaN_list
  
  def __apply_am_infill(self, df_train, df_test, postprocess_assigninfill_dict, \
                      postprocess_dict, infilliterate, printstatus, infillcolumns_list, \
                      masterNArows_train, masterNArows_test, randomseed, ML_cmnd, logger):
    """
    #Modularizes the application of infill to train and test sets
    #note that this works in both sequential and parallel conventions of for loops
    #although may return slightly different values between the two cases
    """

    sorted_columns_by_NaN_list = \
    self.__assemble_sorted_columns_by_NaN_dict(masterNArows_train, list(df_train), postprocess_dict)

    #infilliterate allows ML infill sets to run multiple times
    #as may be beneficial if set had a high proportion of infill for instance
    iteration = 0
    if infilliterate == 0:
      infilliterate = 1
      
    #if we're uysing this method we'll have some extra printouts
    if infilliterate > 1:
      print_infilliterate = True
    else:
      print_infilliterate = False
      
    #initialize validation results
    infill_validations = {}

    #initialize early stopping support entries
    halt_dict = {}
    stop_count = infilliterate
      
    while iteration < infilliterate:
      
      #resent MLinfill infillcomplete markers to False
      if iteration > 0:
        for key in postprocess_assigninfill_dict['MLinfill']:
          postprocess_dict['column_dict'][key]['infillcomplete'] = False
      
      # if printstatus is True:
      if print_infilliterate is True:
        self.__autoprint('debug', "______")
        self.__autoprint('debug', "ML infill infilliterate iteration: ", iteration + 1)
        self.__autoprint('debug', " ")

      #initalize halt_dict entry, iteration is the current integer >=0 associated with infilliterate
      halt_dict_entry = \
      {iteration : {
        'stop_result' : False, 
        'numeric_result' : False,
        'categoric_result' : False,
        'categoric_tuple_df' : pd.DataFrame({'sumofinequal':[], 'quantity':[], 'ratio':[], 'column':[]}),
        'numeric_tuple_df' : pd.DataFrame({'meanabsdelta':[], 'meanabs':[], 'quantity':[], 'column':[]}),
      }}

      halt_dict.update(halt_dict_entry)

      #df_infill_i is associated with the previous iteration and df_infill_iplus is associated with this iteration
      if iteration > 0:
        #access df_infill_iplus from previous iteration
        df_infill_i = df_infill_iplus.copy()
      else:
        df_infill_i = pd.DataFrame()

      #df_infill_iplus will be populated with a single column of infill entries for each set
      #note entries will be in top rows with remainder padded out with NaN for disparate entry count between columns
      #(we'll have access to the entry count in halt_dict when populating results)
      df_infill_iplus = pd.DataFrame()
          
      #columns sorted by number of missing entries from most to least
      for column in sorted_columns_by_NaN_list:
          
        if column in postprocess_dict['column_dict']:
          
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
          not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

            if iteration == 0:
              
              #stndrdinfill 
              #printouts are ommitted, this is referring to cases of no MLinfill 
              #and no assigninfill other than stndrdinfill
              #(so infill defers to defaultinfill from processing function)
                  
              #zeroinfill
              if column in postprocess_assigninfill_dict['zeroinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: zeroinfill")
                self.__autoprint('debug', "")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.__zeroinfillfunction(df_train, column, postprocess_dict, \
                                        masterNArows_train)

                df_test = \
                self.__zeroinfillfunction(df_test, column, postprocess_dict, \
                                        masterNArows_test)

              #negzeroinfill
              if column in postprocess_assigninfill_dict['negzeroinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: negzeroinfill")
                self.__autoprint('debug', "")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.__negzeroinfillfunction(df_train, column, postprocess_dict, \
                                        masterNArows_train)

                df_test = \
                self.__negzeroinfillfunction(df_test, column, postprocess_dict, \
                                        masterNArows_test)

              #oneinfill
              if column in postprocess_assigninfill_dict['oneinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: oneinfill")
                self.__autoprint('debug', "")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.__oneinfillfunction(df_train, column, postprocess_dict, \
                                       masterNArows_train)

                df_test = \
                self.__oneinfillfunction(df_test, column, postprocess_dict, \
                                       masterNArows_test)

              #adjinfill
              if column in postprocess_assigninfill_dict['adjinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: adjinfill")
                self.__autoprint('debug', "")

                df_train = \
                self.__adjinfillfunction(df_train, column, postprocess_dict, \
                                       masterNArows_train)

                df_test = \
                self.__adjinfillfunction(df_test, column, postprocess_dict, \
                                       masterNArows_test)

              #interpinfill
              if column in postprocess_assigninfill_dict['interpinfill']:
                
                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: interpinfill")
                self.__autoprint('debug', "")
                
                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True
                
                if incompatible_MLinfilltype is False:
                
                  df_train = \
                  self.__interpinfillfunction(df_train, column, postprocess_dict, \
                                              masterNArows_train)

                  df_test = \
                  self.__interpinfillfunction(df_test, column, postprocess_dict, \
                                              masterNArows_test)

              #medianinfill
              if column in postprocess_assigninfill_dict['medianinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: medianinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                if (categorylistlength == 1) \
                and incompatible_MLinfilltype is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  df_train, infillvalue = \
                  self.__train_medianinfillfunction(df_train, column, postprocess_dict, \
                                                  masterNArows_train)

                  postprocess_dict['column_dict'][column].update({'assigninfill_infillvalue':infillvalue})

                  df_test = \
                  self.__test_medianinfillfunction(df_test, column, postprocess_dict, \
                                                 masterNArows_test, infillvalue)
              
              #meaninfill
              if column in postprocess_assigninfill_dict['meaninfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: meaninfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                #if (column not in excludetransformscolumns) \
                if (categorylistlength == 1) \
                and incompatible_MLinfilltype is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  df_train, infillvalue = \
                  self.__train_meaninfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column].update({'assigninfill_infillvalue':infillvalue})

                  df_test = \
                  self.__test_meaninfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)
              
              #modeinfill
              if column in postprocess_assigninfill_dict['modeinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: modeinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {}:
                # in {'boolexclude', 'concurrent_nmbr'}:
                  incompatible_MLinfilltype = True

                if incompatible_MLinfilltype is False:

                  df_train, infillvalue = \
                  self.__train_modeinfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column].update({'assigninfill_infillvalue':infillvalue})

                  df_test = \
                  self.__test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)
              
              #lcinfill:
              if column in postprocess_assigninfill_dict['lcinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: lcinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False

                #seems reasonable to exclude concurrent_nmbr from mode
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {}:
                  incompatible_MLinfilltype = True

                if incompatible_MLinfilltype is False:

                  df_train, infillvalue = \
                  self.__train_lcinfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column].update({'assigninfill_infillvalue':infillvalue})

                  #repurpose modeinfillfunction for test, only difference is the passed infillvalue
                  df_test = \
                  self.__test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)

            #MLinfill:
            if column in postprocess_assigninfill_dict['MLinfill']:

              #printout display progress
              # if printstatus is True:
              self.__autoprint('debug', "infill to column: ", column)
              self.__autoprint('debug', "     infill type: MLinfill")
              self.__autoprint('debug', "")

              infill_validations = \
              self.__check_ML_infill(printstatus, df_train, column, postprocess_dict, logger, infill_validations)
                
              #added a returned df_traininfill, convention is df_traininfill will be False when infillcomplete is True to ensure only one entry per categorylist
              df_train, df_test, postprocess_dict, df_traininfill = \
              self.__MLinfillfunction(df_train, df_test, column, postprocess_dict, \
                                    masterNArows_train, masterNArows_test, randomseed, ML_cmnd, \
                                    printstatus, logger)
              
              MLinfilltype = postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype']
              
              if df_traininfill is not False:
                
                df_traininfill_rowcount = df_traininfill.shape[0]

                #consolidate multicolumn infill into a single column of strings with header column
                #convention is df_traininfill will have headers consistent with target columns
                #note that for concurrent_nmbr and concurrent_act MLinfilltype this will be a single column
                #this is not further applied as infill, it's just used in this form to evaluate halting criteria
                df_traininfill = \
                self.__consolidate_multicolumn(df_traininfill, column, MLinfilltype, postprocess_dict)
                
                #now concat that single column infill representation onto df_infill_iplus
                df_infill_iplus = pd.concat([df_infill_iplus, df_traininfill], axis=1)
                
                #this will append additional entries onto the halt_dict
                halt_dict = \
                self.__populate_halt_dict(df_infill_i, df_infill_iplus, MLinfilltype, halt_dict, iteration, column, df_traininfill_rowcount)
                
      for columnname in df_train.columns:
        postprocess_dict['column_dict'][columnname]['infillcomplete'] = False
      
      #calc stop_result
      stop_result, halt_dict = \
      self.__calc_stop_result(halt_dict, iteration, ML_cmnd)
      
      if stop_result is True:
        
        # if printstatus is True:
        if print_infilliterate is True:
          self.__autoprint('debug', '')
          self.__autoprint('debug', "ML infill infilliterate halted at iteration: ", iteration + 1)
          self.__autoprint('debug', "______")
          self.__autoprint('debug', " ")

        #stop_count will be the infilliterate threshold applied in postmunge, defaults to infilliterate value when stopping not reached
        stop_count = iteration + 1

        #setting iteration to infilliterate will halt iterations once this round of columns is complete
        iteration = infilliterate
      
      else:
        iteration += 1

    #naninfill is performed after ML infill to avoid interference
    for column in postprocess_assigninfill_dict['naninfill']:
        
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "infill to column: ", column)
        self.__autoprint('debug', "     infill type: naninfill")
        self.__autoprint('debug', "")

        df_train = \
        self.__naninfillfunction(df_train, column, postprocess_dict, \
                                masterNArows_train)

        df_test = \
        self.__naninfillfunction(df_test, column, postprocess_dict, \
                                masterNArows_test)
    
    return df_train, df_test, postprocess_dict, infill_validations, sorted_columns_by_NaN_list, stop_count
  
  def __apply_pm_infill(self, df_test, postprocess_assigninfill_dict, \
                      postprocess_dict, printstatus, infillcolumns_list, \
                      masterNArows_test, logger):
    """
    #Modularizes the application of infill to test sets
    """

    sorted_columns_by_NaN_list = postprocess_dict['sorted_columns_by_NaN_list']
    
    #infilliterate allows ML infill sets to run multiple times
    #as may be bneficial if set had a high number of infill for instance
    iteration = 0

    #note that stop_count may be less than the original infilliterate
    #in cases when early stopping was evaluated based on ML_cmnd['halt_iterate'] 
    infilliterate = postprocess_dict['stop_count']
    
    #if we're uysing this method we'll have some extra printouts
    if infilliterate > 1:
      print_infilliterate = True
    else:
      print_infilliterate = False
    
    #just the convention
    if infilliterate == 0:
      infilliterate = 1

    infill_validations = {}
    
    while iteration < infilliterate:
      
      #resent MLinfill infillcomplete markers to False
#       if iteration > 0:
      for key in postprocess_assigninfill_dict['MLinfill']:
        postprocess_dict['column_dict'][key]['infillcomplete'] = False
      
      # if printstatus is True:
      if print_infilliterate is True:
        self.__autoprint('debug', "______")
        self.__autoprint('debug', "ML infill infilliterate iteration: ", iteration + 1)
        self.__autoprint('debug', " ")
    
      #columns sorted by nan count in train data from automunge
      for column in sorted_columns_by_NaN_list:

        if column in postprocess_dict['column_dict']:
        
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
          not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

            if iteration == 0:

              #stndrdinfill 
              #printouts are ommitted, this is referring to cases of no MLinfill 
              #and no assigninfill other than stndrdinfill
              #(so infill defers to defaultinfill from processing function)

              #zeroinfill:
              if column in postprocess_assigninfill_dict['zeroinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: zeroinfill")
                self.__autoprint('debug', "")

                df_test = \
                self.__zeroinfillfunction(df_test, column, postprocess_dict, \
                                        masterNArows_test)

              #negzeroinfill:
              if column in postprocess_assigninfill_dict['negzeroinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: negzeroinfill")
                self.__autoprint('debug', "")

                df_test = \
                self.__negzeroinfillfunction(df_test, column, postprocess_dict, \
                                        masterNArows_test)
                
              #oneinfill:
              if column in postprocess_assigninfill_dict['oneinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: oneinfill")
                self.__autoprint('debug', "")

                df_test = \
                self.__oneinfillfunction(df_test, column, postprocess_dict, \
                                      masterNArows_test)
                
              #adjinfill:
              if column in postprocess_assigninfill_dict['adjinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: adjinfill")
                self.__autoprint('debug', "")

                df_test = \
                self.__adjinfillfunction(df_test, column, postprocess_dict, \
                                      masterNArows_test)

              #interpinfill
              #first line backward compatibility preceding 8.25
              if 'interpinfill' in postprocess_assigninfill_dict \
              and column in postprocess_assigninfill_dict['interpinfill']:
                
                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: interpinfill")
                self.__autoprint('debug', "")
                
                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True
                
                if incompatible_MLinfilltype is False:

                  df_test = \
                  self.__interpinfillfunction(df_test, column, postprocess_dict, \
                                              masterNArows_test)
                
              #medianinfill
              if column in postprocess_assigninfill_dict['medianinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: medianinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                #if (column not in excludetransformscolumns) \
                if (categorylistlength == 1) \
                and incompatible_MLinfilltype is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  infillvalue = postprocess_dict['column_dict'][column]['assigninfill_infillvalue']

                  df_test = \
                  self.__test_medianinfillfunction(df_test, column, postprocess_dict, \
                                                masterNArows_test, infillvalue)

              #meaninfill:
              if column in postprocess_assigninfill_dict['meaninfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: meaninfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {'multirt', 'singlct', 'binary', '1010', 'concurrent_ordl', 'concurrent_act'}:
                  incompatible_MLinfilltype = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                #if (column not in excludetransformscolumns) \
                if (categorylistlength == 1) \
                and incompatible_MLinfilltype is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  infillvalue = postprocess_dict['column_dict'][column]['assigninfill_infillvalue']

                  df_test = \
                  self.__test_meaninfillfunction(df_test, column, postprocess_dict, \
                                              masterNArows_test, infillvalue)

              #modeinfill:
              if column in postprocess_assigninfill_dict['modeinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: modeinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False

                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {}:
                  incompatible_MLinfilltype = True

                if incompatible_MLinfilltype is False:

                  infillvalue = postprocess_dict['column_dict'][column]['assigninfill_infillvalue']

                  df_test = \
                  self.__test_modeinfillfunction(df_test, column, postprocess_dict, \
                                                masterNArows_test, infillvalue)
                  
              #lcinfill:
              if column in postprocess_assigninfill_dict['lcinfill']:

                #printout display progress
                # if printstatus is True:
                self.__autoprint('debug', "infill to column: ", column)
                self.__autoprint('debug', "     infill type: lcinfill")
                self.__autoprint('debug', "")

                #check if column is incompatible_MLinfilltype
                incompatible_MLinfilltype = False

                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in {}:
                  incompatible_MLinfilltype = True

                if incompatible_MLinfilltype is False:

                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  infillvalue = postprocess_dict['column_dict'][column]['assigninfill_infillvalue']

                  #repurpose modeinfillfunction for test, only difference is the passed infillvalue
                  df_test = \
                  self.__test_modeinfillfunction(df_test, column, postprocess_dict, \
                                                masterNArows_test, infillvalue)

            #MLinfill:
            if column in postprocess_assigninfill_dict['MLinfill']:

              #printout display progress
              # if printstatus is True:
              self.__autoprint('debug', "infill to column: ", column)
              self.__autoprint('debug', "     infill type: MLinfill")
              self.__autoprint('debug', "")

              df_test, postprocess_dict = \
              self.__postMLinfillfunction (df_test, column, postprocess_dict, \
                                        masterNArows_test, printstatus, logger)

      for columnname in df_test.columns:
        postprocess_dict['column_dict'][columnname]['infillcomplete'] = False
      
      iteration += 1

    #naninfill is performed after ML infill to avoid interference
    for column in postprocess_assigninfill_dict['naninfill']:
      
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:
              
        #naninfill
        if column in postprocess_assigninfill_dict['naninfill']:

          #printout display progress
          # if printstatus is True:
          self.__autoprint('debug', "infill to column: ", column)
          self.__autoprint('debug', "     infill type: naninfill")
          self.__autoprint('debug', "")

          df_test = \
          self.__naninfillfunction(df_test, column, postprocess_dict, \
                                masterNArows_test)
      
    return df_test, infill_validations

  def __zeroinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)
    
    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def __oneinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def __negzeroinfillfunction(self, df, column, postprocess_dict, \
                             masterNArows):
    
    # #copy the datatype to ensure returned set is consistent
    # df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all neg zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    infill = \
    self.__autowhere(infill, column, infill[column] == 1, -0., specified='replacement')
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)

    #for negzeroinfill we'll convert to float in case __insertinfill returns as pandas object dtype
    #note that negzero requires float data type to represent
    #note this may result in float sets not converted by floatprecision parameter when negzero infill applied to a categoric feature, 
    #so recomend reserving negzeroinfill to continuous numeric features
    df[column] = \
    df[column].astype(float)

    #reset data type to ensure returned data is consistent with what was passed
    #(we won't do this for naninfill since nan may be different data type)
    # df[column] = \
    # df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df
    
  def __naninfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    infill = \
    self.__autowhere(infill, column, infill[column] == 1, np.nan, specified='replacement')
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)

    #reset data type to ensure returned data is consistent with what was passed
    #(we won't do this for naninfill since nan may be different data type)
    # df[column] = \
    # df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def __adjinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all nan with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, np.nan)
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)
    
    #this is hack
    df[column] = df[column].replace('nan', np.nan)
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[column] = df[column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[column] = df[column].fillna(method='bfill')
    
    #and final edge case if all cells were subject to infill we'll just insert 0
    df[column] = df[column].fillna(value=0)
    
    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df

  def __interpinfillfunction(self, df, column, postprocess_dict, \
                             masterNArows):

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    infill = \
    self.__autowhere(infill, column, infill[column] == 1, np.nan, specified='replacement')
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[categorylist][:0]).copy()

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)
    
    #everything preceding here was consistent with naninfill
    #having reverted default infill to nan, can now apply pandas method
    #we'll default to linear, if users want other scnearios let me know
    for categorylist_entry in categorylist:
      
      df[categorylist_entry] = pd.to_numeric(df[categorylist_entry], errors='coerce').interpolate(method='linear', 
                                                                                                  axis=0, 
                                                                                                  limit=None, 
                                                                                                  inplace=False,
                                                                                                  limit_direction='forward',
                                                                                                  limit_area=None,
                                                                                                  downcast=None,
                                                                                                 )
      
      #we'll follow with a adj just in case first or last row had a nan
      df[categorylist_entry] = df[categorylist_entry].fillna(method='ffill')
      df[categorylist_entry] = df[categorylist_entry].fillna(method='bfill')

      #reset data type to ensure returned data is consistent with what was passed
      #(naninfill preceding interpolate may have changed data type)
      df[categorylist_entry] = \
      df[categorylist_entry].astype({column:df_temp_dtype[categorylist_entry].dtypes})
    
    return df

  def __train_medianinfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #create df without rows that were subject to infill to dervie median
    tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
    #remove rows that were subject to infill
    tempdf = tempdf[tempdf[NArw_columnname] != 1]
    #calculate median of remaining rows
    median = tempdf[column].median()
    
    #edge case
    if median != median:
      median = 0
    
    del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, median)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df, median

  def __test_medianinfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows, median):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    median = median

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, median)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def __train_meaninfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #create df without rows that were subject to infill to dervie median
    tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
    #remove rows that were subject to infill
    tempdf = tempdf[tempdf[NArw_columnname] != 1]
    #calculate median of remaining rows
    mean = tempdf[column].mean()
    
    #edge case
    if mean != mean:
      mean = 0
    
    del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, mean)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df, mean

  def __test_meaninfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows, mean):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    mean = mean

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, mean)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def __train_modeinfillfunction(self, df, column, postprocess_dict, \
                               masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #deriving a mode for one-hot encoded sets requires a different approach
    if len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in {'multirt'}:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      tempdf_mode_dict = {}
      
      #since this is one hot encoded we can count activations in a column with sum
      for tempcolumn in tempdf.columns:
        tempdf_mode_dict.update({tempdf[tempcolumn].sum() : tempcolumn})
      
      #create a list of those sums then sort to grab the mode column
      tempdf_mode_dict_keys = list(tempdf_mode_dict)
      tempdf_mode_dict_keys = sorted(tempdf_mode_dict_keys)
      mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[-1]]
      
      if mode_column == column:
        mode = 1
      else:
        mode = 0
        
      del tempdf
      del tempdf_mode_dict
      del tempdf_mode_dict_keys
      del mode_column
      
    #deriving a mode for multi-column binary encoded sets requires a different approach
    elif len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in {'1010'}:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      #initialize a column to store encodings
      tempdf['onehot'] = ''

      #populate column to store aggregated encodings 
      for tempdf_column in tempdf.columns:
        if tempdf_column != 'onehot':
          tempdf['onehot'] = \
          tempdf['onehot'] + tempdf[tempdf_column].astype(int).astype(str)

      #find mode of the aggregation
      binary_mode = tempdf['onehot'].mode()
      
      if len(binary_mode) > 0:
        binary_mode = binary_mode[0]
      else:
        binary_mode = tempdf['onehot'].iat[0]
      
#       if len(binary_mode) < 1:
#         binary_mode = 0
      
      #remove rows other than mode
      tempdf = tempdf[tempdf['onehot'] == binary_mode]
      
      #mode is the current columns value associated with that mode
      mode = binary_mode[categorylist.index(column)]
      
      del tempdf
      del binary_mode
    
    #else if columns were not multi-column
    #note this scenario also includes 'concurrent_act' MLinfilltype
    else:
    
      #create df without rows that were subject to infill to dervie mode
      tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      #calculate mode of remaining rows
      mode = tempdf[column].mode()
      
      if len(mode) > 0:
        mode = mode[0]
      else:
        mode = 0

      del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df, mode

  def __test_modeinfillfunction(self, df, column, postprocess_dict, \
                              masterNArows, mode):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    mode = mode

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df

  def __train_lcinfillfunction(self, df, column, postprocess_dict, \
                             masterNArows):
    """
    #comparable to modeinfill function but uses least common value instead of most common value
    """

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #deriving a mode for one-hot encoded sets requires a different approach
    if len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in {'multirt'}:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      tempdf_mode_dict = {}
      
      #since this is one hot encoded we can count activations in a column with sum
      for tempcolumn in tempdf.columns:
        tempdf_mode_dict.update({tempdf[tempcolumn].sum() : tempcolumn})
      
      #create a list of those sums then sort to grab the mode column
      tempdf_mode_dict_keys = list(tempdf_mode_dict)
      tempdf_mode_dict_keys = sorted(tempdf_mode_dict_keys)
      #mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[-1]]
      mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[0]]
      
      if mode_column == column:
        mode = 1
      else:
        mode = 0
        
      del tempdf
      del tempdf_mode_dict
      del tempdf_mode_dict_keys
      del mode_column
      
    #deriving a mode for multi-column binary encoded sets requires a different approach
    elif len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in {'1010'}:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      #initialize a column to store encodings
      #note this arbitrary column won't overlap with any
      #because categorylist all have suffixes with '_' character
      #however just as good practice will build in an overlap mitigation
      tempstring1 = 'onehot'
      if tempstring1 in tempdf:
        while tempstring1 in tempdf:
          tempstring1 += 'z'
      
      tempdf[tempstring1] = ''

      #populate column to store aggregated encodings 
      for tempdf_column in tempdf.columns:
        if tempdf_column != tempstring1:
          tempdf[tempstring1] = \
          tempdf[tempstring1] + tempdf[tempdf_column].astype(int).astype(str)

      #find mode of the aggregation
      
      tempstring2 = 'asdf'
      if tempstring2 in tempdf:
        while tempstring2 in tempdf:
          tempstring2 += 'z'
      #binary_mode = tempdf['onehot'].mode()
      #note lcinfill applied after encodings so don't need to worry about edge case for bytes entries for value_counts sorting
      mode_valuecounts_list = pd.DataFrame(tempdf[tempstring1].value_counts())
      mode_valuecounts_list = mode_valuecounts_list.rename_axis(tempstring2).sort_values(by = ['count', tempstring2], ascending = [False, True])
      mode_valuecounts_list = list(mode_valuecounts_list.index)

      if len(mode_valuecounts_list) > 0:
        binary_mode = mode_valuecounts_list[-1]
      else:
        binary_mode = 0
      
#       if len(binary_mode) > 0:
#         binary_mode = binary_mode[0]
#       else:
#         binary_mode = tempdf['onehot'][0]

      if binary_mode != binary_mode:
        binary_mode = 0
      
#       if len(binary_mode) < 1:
#         binary_mode = 0
      
      #remove rows other than mode
      tempdf = tempdf[tempdf[tempstring1] == binary_mode]
      
      #mode is the current columns value associated with that mode
      mode = binary_mode[categorylist.index(column)]
      
      del tempdf
      del binary_mode
    
    #else if columns were not multi-column
    #note this scenario also includes 'concurrent_act' MLinfilltype
    else:
    
      #create df without rows that were subject to infill to dervie mode
      tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]

      #calculate mode of remaining rows
      tempstring = 'asdf'
      if tempstring in tempdf:
        while tempstring in tempdf:
          tempstring += 'z'
      #note lcinfill applied after encodings so don't need to worry about edge case for bytes type entries
      mode_valuecounts_list = pd.DataFrame(tempdf[column].value_counts())
      mode_valuecounts_list = mode_valuecounts_list.rename_axis(tempstring).sort_values(by = ['count', tempstring], ascending = [False, True])
      mode_valuecounts_list = list(mode_valuecounts_list.index)
      if len(mode_valuecounts_list) > 0:
        mode = mode_valuecounts_list[-1]
      else:
        mode = 0
      
      if mode != mode:
        mode = 0
      
#       if len(mode) > 0:
#         mode = mode[0]
#       else:
#         mode = 0

      del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.__insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df, mode

  #__FunctionBlock: PCA support functions

  def __evalPCA(self, df_train, PCAn_components, ML_cmnd):
    '''
    returns a PCA type (PCA_type)
    and a n_components (n_components)

    returns PCActgy as ML_cmnd['PCA_type']
    
    unless ML_cmnd['PCA_type'] == 'default'
    in which case determines a PCA_type as one of:
    'noPCA' -> self explanatory
    'KernelPCA' -> dataset suitable for automated KernelPCA application
                    (default PCA method when data is all non-negative)
    'SparsePCA' -> dataset suitable for automated SparsePCA application
                    (prefered default when data is not all non-negative)
    'PCA'       -> default PCA when user passes PCAn_components as float between 0-1

    also returns a n_components value which is based on the user passed PCAn_components
    which will be the same as PCAn_components when is passed as an integer or float

    or if user passes PCAn_components as None then
    a heuristic is evaluated to either derive a new n_components or return as None
    the heuristic is based on ratio of number of columns to number of rows
    thus performing PCA when there aren't enough samples for the number of features
    which defaults to the column/row ratio of 0.5 and can be adjusted with ML_cmnd['PCA_cmnd']['col_row_ratio']
    when applied the heuristic returns a new n_components as n_components = int(round(col_row_ratio * number_rows))
    '''

    #first we'll initialize a PCActgy to be applied when ML_cmnd['PCA_type'] == 'default'
    #although this initialized PCActgy will subsequently be overwritten if user specifies ML_cmnd['PCA_type']
    if any(df_train < 0.0):
      PCActgy = 'SparsePCA'

    #else if there were no negative values in the dataframe
    else:
      PCActgy = 'KernelPCA'

    #note SparsePCA or KernelPCA is reverted to 'PCA' below when PCAn_components is float between 0-1

    #_____

    #now we perform the heuristic inspection
    #I don't expect in current form this heuristic will be widely used
    #primiarily was implemented to crystalize some thoughts on the matter

    #ok this is to allow user to set the default columns/rows ratio for automated PCA
    if 'col_row_ratio' in ML_cmnd['PCA_cmnd']:
      col_row_ratio = ML_cmnd['PCA_cmnd']['col_row_ratio']
    else:
      col_row_ratio = 0.50

    number_rows = df_train.shape[0]
    number_columns = df_train.shape[1]

    if PCAn_components == None:

      if number_columns / number_rows < col_row_ratio:

        PCActgy = 'noPCA'
        n_components = PCAn_components

      elif number_columns / number_rows >= col_row_ratio:

        n_components = int(round(col_row_ratio * number_rows))

    #_____

    #returned n_components is otherwise same as PCAn_components
    if isinstance(PCAn_components, (int, float)):

      n_components = PCAn_components

      #and as noted above user passing PCAn_components as float between 0-1
      #makes the default PCA type as 'PCA' (linear PCA)
      if PCAn_components > 0.0 and PCAn_components < 1.0:
        
        PCActgy = 'PCA'

        #(note there is an edge case, user can't pass PCAn_components as float between 0-1
        #unless in conjunction with ML_cmnd['PCA_type'] = 'default' or ML_cmnd['PCA_type'] = 'PCA')

    #if user designated a PCA_type that takes precedence
    if ML_cmnd['PCA_type'] != 'default':

      PCActgy = ML_cmnd['PCA_type']
    
    return PCActgy, n_components

  def __populatePCAdefaults(self, randomseed):
    '''
    populates sa dictionary with default values for PCA methods PCA, 
    SparsePCA, and KernelPCA. (currently just applying the randomseed)
    '''

    PCAdefaults = {'PCA':{}, 'SparsePCA':{}, 'KernelPCA':{}}

    PCAdefaults['PCA'].update({'random_state':randomseed})

    PCAdefaults['SparsePCA'].update({'random_state':randomseed})

    PCAdefaults['KernelPCA'].update({'random_state':randomseed})

    return PCAdefaults

  def __initSparsePCA(self, ML_cmnd, PCAdefaults, PCAn_components, PCAtype='SparsePCA'):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a SparsePCA model
    '''

    PCA_cmnd = self.__autocopy(ML_cmnd['PCA_cmnd'])
    
    #first strike any automunge specific parameters
    for automunge_PCA_command in ['bool_ordl_PCAexcl', 'bool_PCA_excl', 'col_row_ratio']:
      if automunge_PCA_command in PCA_cmnd:
        del PCA_cmnd[automunge_PCA_command]
        
    #then update with any defaults (we populate a randomseed in __populatePCAdefaults)
    PCA_cmnd.update(PCAdefaults[PCAtype])
    PCA_cmnd.update({'n_components' : PCAn_components})

    if PCAtype == 'SparsePCA':
      from sklearn.decomposition import SparsePCA
      PCAmodel = SparsePCA(**PCA_cmnd)

    elif PCAtype == 'KernelPCA':
      from sklearn.decomposition import KernelPCA
      PCAmodel = KernelPCA(**PCA_cmnd)

    elif PCAtype == 'PCA':
      from sklearn.decomposition import PCA
      PCAmodel = PCA(**PCA_cmnd)
    
    return PCAmodel
  
  def __initKernelPCA(self, ML_cmnd, PCAdefaults, PCAn_components):
    #comparable to __initSparsePCA with different PCAtype for inspecting PCAdefaults
    return self.__initSparsePCA(ML_cmnd, PCAdefaults, PCAn_components, PCAtype='KernelPCA')
  
  def __initPCA(self, ML_cmnd, PCAdefaults, PCAn_components):
    #comparable to __initSparsePCA with different PCAtype for inspecting PCAdefaults
    return self.__initSparsePCA(ML_cmnd, PCAdefaults, PCAn_components, PCAtype='PCA')

  def __boolexcl(self, ML_cmnd, df, PCAexcl, postprocess_dict):
    """
    If user passed bool_PCA_excl as True in ML_cmnd['PCA_cmnd']
    {'PCA_cmnd':{'bool_PCA_excl': True}}
    Then add boolean columns to the PCAexcl list of columns
    to be carved out from PCA application
    If user passed bool_ordl_PCAexcl as True in ML_cmnd['PCA_cmnd']
    Then add ordinal columns (recognized becayuse they are catehgorical)
    to the PCAexcl list of columns
    to be carved out from PCA application

    Note that _check_ML_cmnd by defaults initalizes PCA_cmnd with {bool_ordl_PCAexcl : True}
    When not otherwise specified
    
    Note that PCAexcl may already be populated with user-passed
    columns to exclude from PCA. The returned bool_PCAexcl list
    seperately tracks just those columns that were added as part 
    of this function, in case may be of later use

    Note that integer mlinfilltype is not excluded from PCA
    """

    bool_PCAexcl = []
    if 'bool_PCA_excl' in ML_cmnd['PCA_cmnd']:
        
      #if user passed the bool_PCA_excl as True in ML_cmnd['PCA_cmnd'] 
      if ML_cmnd['PCA_cmnd']['bool_PCA_excl'] is True:
        for checkcolumn in df:
          #if column is boolean then add to lists
        #   if set(df[checkcolumn].unique()) == {0,1} \
        #   or set(df[checkcolumn].unique()) == {0} \
        #   or set(df[checkcolumn].unique()) == {1}:
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][checkcolumn]['category']]['MLinfilltype'] \
          in {'multirt', 'binary', '1010', 'boolexclude', 'concurrent_act', 'totalexclude'}:
            if checkcolumn not in PCAexcl:
              PCAexcl.append(checkcolumn)
            bool_PCAexcl.append(checkcolumn)
    
    #if bool_PCA_excl was specified it takes precedence over bool_ordl_PCAexcl
    elif 'bool_ordl_PCAexcl' in ML_cmnd['PCA_cmnd']:
      #if user passed the bool_ordl_PCAexcl as True in ML_cmnd['PCA_cmnd'] 
      if ML_cmnd['PCA_cmnd']['bool_ordl_PCAexcl'] is True:
        for checkcolumn in df:
          #if column is boolean then add to lists
        #   if set(df[checkcolumn].unique()) == {0,1} \
        #   or set(df[checkcolumn].unique()) == {0} \
        #   or set(df[checkcolumn].unique()) == {1} \
        #   or checkcolumn[-5:] == '_ordl':
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][checkcolumn]['category']]['MLinfilltype'] \
          in {'singlct', 'binary', 'multirt', '1010', 'boolexclude', 'ordlexclude', 'concurrent_ordl', 'concurrent_act', 'totalexclude'}:
            #or isinstance(df[checkcolumn].dtype, pd.api.types.CategoricalDtype):
            if checkcolumn not in PCAexcl:
              PCAexcl.append(checkcolumn)
            bool_PCAexcl.append(checkcolumn)
            
    return PCAexcl, bool_PCAexcl

  def __createPCAsets(self, df_train, df_test, PCAexcl, postprocess_dict, ML_cmnd):
    '''
    Function that takes as input the dataframes df_train and df_test 
    Removes those columns associated with the PCAexcl (which are the original 
    columns passed to automunge which are to be exlcuded from PCA), and returns 
    those sets as PCAset_trian, PCAset_test, and the list of columns extracted as
    PCAexcl_posttransform.
    '''

    #initiate list PCAexcl_postransform
    PCAexcl_posttransform = []

    PCAexcl_posttransform = self.__column_convert_support(PCAexcl, postprocess_dict, convert_to='returned')

    #we'll also exclude any features designated in ML_cmnd['full_exclude'], since they may have non-numeric data
    full_exclude = []
    if 'full_exclude' in ML_cmnd:
      full_exclude = ML_cmnd['full_exclude']
      if len(full_exclude) > 0:
        #convert to returned column convention
        full_exclude = self.__column_convert_support(full_exclude, postprocess_dict, convert_to='returned')

    #assemble the set of columns to be dropped
    PCAexcl_posttransform = list((set(PCAexcl_posttransform) | set(full_exclude)) & set(df_train))

    #assemble the sets by dropping the columns excluded
    PCAset_train = df_train.drop(PCAexcl_posttransform, axis=1)
    PCAset_test = df_test.drop(PCAexcl_posttransform, axis=1)

    return PCAset_train, PCAset_test, PCAexcl_posttransform

  def __PCAfunction(self, PCAset_train, PCAset_test, n_components, PCActgy, postprocess_dict, \
                  randomseed, ML_cmnd):
    '''
    Function that takes as input the train and test sets intended for PCA
    dimensionality reduction. Returns a trained PCA model saved in postprocess_dict
    and trasnformed sets.
    '''

    #initialize ML_cmnd
    #ML_cmnd = postprocess_dict['ML_cmnd']
    ML_cmnd = ML_cmnd
    
    #Save the PCActgy to the postprocess_dict
    postprocess_dict.update({'PCActgy' : PCActgy})
    
    #initialize PCA defaults dictionary
    PCAdefaults = \
    self.__populatePCAdefaults(randomseed)
    
    # #convert PCAsets to numpy arrays
    # PCAset_train = PCAset_train.to_numpy()
    # PCAset_test = PCAset_test.to_numpy()
    
    #initialize a PCA model
    if PCActgy == 'default' or PCActgy == 'SparsePCA':

      from sklearn.decomposition import SparsePCA
  
      PCAmodel = self.__initSparsePCA(ML_cmnd, PCAdefaults, n_components)

    if PCActgy == 'KernelPCA':

      from sklearn.decomposition import KernelPCA
  
      PCAmodel = self.__initKernelPCA(ML_cmnd, PCAdefaults, n_components)
    
    if PCActgy == 'PCA':

      from sklearn.decomposition import PCA
  
      PCAmodel = self.__initPCA(ML_cmnd, PCAdefaults, n_components)
    
    #derive the PCA model (note htis is unsupervised training, no labels)
    PCAmodel.fit(PCAset_train)

    #Save the trained PCA model to the postprocess_dict
    postprocess_dict.update({'PCAmodel' : PCAmodel})

    #apply the transform
    PCAset_train = PCAmodel.transform(PCAset_train)
    PCAset_test = PCAmodel.transform(PCAset_test)

    #get new number of columns (scikit PCA returns as numpy so using np.size)
    newcolumncount = np.size(PCAset_train,1)
    
    #___
    
    columnnames, PCA_columns_valresult = \
    self.__set_PCA_column(newcolumncount, postprocess_dict, root='PCA_')
        
    #___

    #convert output to pandas (note that the index is set externally with concat operation)
    PCAset_train = pd.DataFrame(PCAset_train, columns = columnnames)
    PCAset_test = pd.DataFrame(PCAset_test, columns = columnnames)

    return PCAset_train, PCAset_test, postprocess_dict, PCA_columns_valresult

  #__FunctionBlock: validation functions
  
  def __check_am_miscparameters(self, valpercent, floatprecision, shuffletrain, \
                             TrainLabelFreqLevel, dupl_rows, powertransform, binstransform, MLinfill, \
                             infilliterate, randomseed, eval_ratio, numbercategoryheuristic, pandasoutput, \
                             NArw_marker, featurethreshold, featureselection, inplace, \
                             Binary, PCAn_components, PCAexcl, printstatus, excl_suffix, \
                             trainID_column, testID_column, evalcat, privacy_encode, encrypt_key, \
                             noise_augment, ppd_append, orig_headers, cat_type, logger):
    """
    #Performs validation to confirm valid entries of passed automunge(.) parameters
    #Note that this function is intended specifically for non-dictionary parameters
    #eg assigncat, assigninfill, assignparam, transformdict, processdict validated elsewhere
    #also note that labels_column, trainID_column, testID_column are checked inside automunge function
    #also note that df_train, df_test, evalcat parameters validation methods still pending
    #returns a dictionary of results
    #False is good
    """
    
    #check valpercent
    valpercent_valresult = False
    if isinstance(valpercent, (int, float)) and not isinstance(valpercent, bool):
      if valpercent < 0 or valpercent >= 1:
        valpercent_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "Error: invalid entry passed for valpercent")
        self.__autoprint('warning', "Acceptable values are numbers in range 0 <= valpercent < 1.")
        self.__autoprint('warning', '')
    elif isinstance(valpercent, tuple):
      if len(valpercent) != 2 \
      or len(valpercent)==2 and not isinstance(valpercent[0], (int, float)) \
      or len(valpercent)==2 and not isinstance(valpercent[1], (int, float)) \
      or len(valpercent)==2 and isinstance(valpercent[0], (int, float)) and valpercent[0]<0 \
      or len(valpercent)==2 and isinstance(valpercent[0], (int, float)) and valpercent[0]>=1 \
      or len(valpercent)==2 and isinstance(valpercent[1], (int, float)) and valpercent[1]<=0 \
      or len(valpercent)==2 and isinstance(valpercent[1], (int, float)) and valpercent[1]>1 \
      or len(valpercent)==2 and isinstance(valpercent[0], (int, float)) and isinstance(valpercent[1], (int, float)) and valpercent[0] > valpercent[1]:
        valpercent_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "Error: invalid entry passed for valpercent")
        self.__autoprint('warning', "acceptable tuple format is two entries of floats in range 0-1")
        self.__autoprint('warning', "with first entry < second entry")
    else:
      valpercent_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for valpercent")
      self.__autoprint('warning', "Acceptable values are numbers in range 0 <= valpercent < 1 or tuple of two numbers in that range.")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'valpercent_valresult' : valpercent_valresult})
    
    #check floatprecision
    floatprecision_valresult = False
    if floatprecision not in {16, 32, 64}:
      floatprecision_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for floatprecision parameter.")
      self.__autoprint('warning', "Acceptable values are one of {16, 32, 64}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'floatprecision_valresult' : floatprecision_valresult})
    
    #check shuffletrain
    shuffletrain_valresult = False
    if shuffletrain not in {True, False, 'traintest'}:
      shuffletrain_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for shuffletrain parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'traintest'}")
      self.__autoprint('warning', '')
    elif shuffletrain not in {'traintest'} \
    and not isinstance(shuffletrain, bool):
      shuffletrain_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for shuffletrain parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'traintest'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'shuffletrain_valresult' : shuffletrain_valresult})
    
    #check TrainLabelFreqLevel
    TrainLabelFreqLevel_valresult = False
    if TrainLabelFreqLevel not in {True, False, 'test', 'traintest'}:
      TrainLabelFreqLevel_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test', 'traintest'}")
      self.__autoprint('warning', '')
    elif TrainLabelFreqLevel not in {'test', 'traintest'} and not isinstance(TrainLabelFreqLevel, bool):
      TrainLabelFreqLevel_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test', 'traintest'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'TrainLabelFreqLevel_valresult' : TrainLabelFreqLevel_valresult})

    #check dupl_rows
    dupl_rows_valresult = False
    if dupl_rows not in {True, False, 'test', 'traintest'}:
      dupl_rows_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for dupl_rows parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test', 'traintest'}")
      self.__autoprint('warning', '')
    elif dupl_rows not in {'test', 'traintest'} and not isinstance(dupl_rows, bool):
      dupl_rows_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for dupl_rows parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test', 'traintest'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'dupl_rows_valresult' : dupl_rows_valresult})
    
    #check powertransform
    powertransform_valresult = False
    if powertransform not in {True, False, 'excl', 'exc2', 'infill', 'infill2', 'DP1', 'DP2', 'DT1', 'DT2', 'DB1', 'DB2'}:
      powertransform_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for powertransform parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'excl', 'exc2', 'infill', 'infill2', 'DP1', 'DP2', 'DT1', 'DT2', 'DB1', 'DB2'}")
      self.__autoprint('warning', '')
    elif powertransform not in {'excl', 'exc2', 'infill', 'infill2', 'DP1', 'DP2', 'DT1', 'DT2', 'DB1', 'DB2'} \
    and not isinstance(powertransform, bool):
      powertransform_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for powertransform parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'excl', 'exc2', 'infill', 'infill2', 'DP1', 'DP2', 'DT1', 'DT2', 'DB1', 'DB2'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'powertransform_valresult' : powertransform_valresult})
    
    #check binstransform
    binstransform_valresult = False
    if binstransform not in {True, False} or not isinstance(binstransform, bool):
      binstransform_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for binstransform parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'binstransform_valresult' : binstransform_valresult})
    
    #check MLinfill
    MLinfill_valresult = False
    if MLinfill not in {True, False} or not isinstance(MLinfill, bool):
      MLinfill_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for MLinfill parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'MLinfill_valresult' : MLinfill_valresult})
    
    #check infilliterate
    infilliterate_valresult = False
    if not isinstance(infilliterate, (int)) \
    or isinstance(infilliterate, bool):
      infilliterate_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for infilliterate parameter.")
      self.__autoprint('warning', "Acceptable values are integers >= 0")
      self.__autoprint('warning', '')
    elif infilliterate < 0:
      infilliterate_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for infilliterate parameter.")
      self.__autoprint('warning', "Acceptable values are integers >= 0")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'infilliterate_valresult' : infilliterate_valresult})
    
    #check randomseed
    randomseed_valresult = False
    if not isinstance(randomseed, (int)) or randomseed is True:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**32-1 or False")
      self.__autoprint('warning', '')
    elif randomseed < 0 or randomseed > 2147483647:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**31-1 or False")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'randomseed_valresult' : randomseed_valresult})
    
    #check eval_ratio
    eval_ratio_valresult = False
    if not (isinstance(eval_ratio, (int)) \
    or isinstance(eval_ratio, (float))):
      eval_ratio_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for eval_ratio parameter.")
      self.__autoprint('warning', "Acceptable values are floats 0-1 or integers >1")
      self.__autoprint('warning', '')
    elif eval_ratio < 0:
      eval_ratio_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for eval_ratio parameter.")
      self.__autoprint('warning', "Acceptable values are floats 0-1 or integers >1")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'eval_ratio_valresult' : eval_ratio_valresult})
    
    #check numbercategoryheuristic
    numbercategoryheuristic_valresult = False
    if not isinstance(numbercategoryheuristic, (int,bool)):
      numbercategoryheuristic_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for numbercategoryheuristic parameter.")
      self.__autoprint('warning', "Acceptable values are integers >= 1 or False")
      self.__autoprint('warning', '')
    elif isinstance(numbercategoryheuristic, bool) and numbercategoryheuristic is not False:
      numbercategoryheuristic_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for numbercategoryheuristic parameter.")
      self.__autoprint('warning', "Acceptable values are integers >= 1 or False")
      self.__autoprint('warning', '')
    elif isinstance(numbercategoryheuristic, (int)) \
    and numbercategoryheuristic is not False \
    and numbercategoryheuristic < 1:
      numbercategoryheuristic_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for numbercategoryheuristic parameter.")
      self.__autoprint('warning', "Acceptable values are integers >= 1 or False")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'numbercategoryheuristic_valresult' : numbercategoryheuristic_valresult})
      
    #check pandasoutput
    pandasoutput_valresult = False
    if pandasoutput in {True, False} and not isinstance(pandasoutput, bool) \
    or pandasoutput not in {True, False, 'dataframe'}:
      pandasoutput_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for pandasoutput parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'dataframe'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'pandasoutput_valresult' : pandasoutput_valresult})
    
    #check NArw_marker
    NArw_marker_valresult = False
    if NArw_marker not in {True, False} or not isinstance(NArw_marker, bool):
      NArw_marker_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for NArw_marker parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'NArw_marker_valresult' : NArw_marker_valresult})

    #check featureselection
    featureselection_valresult = False
    if featureselection not in {True, False, 'pct', 'metric', 'report'} \
    or featureselection in {True, False} and not isinstance(featureselection, bool):
      featureselection_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for featureselection parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, True, 'pct', 'metric', 'report'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'featureselection_valresult' : featureselection_valresult})
    
    #check featurethreshold
    featurethreshold_valresult = False
    if not isinstance(featurethreshold, float) and featurethreshold != 0:
      featurethreshold_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for featurethreshold parameter.")
      self.__autoprint('warning', "Acceptable values are floats within range 0.0 <= featurethreshold <= 1.0")
      self.__autoprint('warning', '')
    elif (featurethreshold < 0.0 or featurethreshold > 1.0):
      featurethreshold_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for featurethreshold parameter.")
      self.__autoprint('warning', "Acceptable values are floats within range 0.0 <= featurethreshold <= 1.0")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'featurethreshold_valresult' : featurethreshold_valresult})

    #check inplace
    inplace_valresult = False
    if inplace not in {True, False} or not isinstance(inplace, bool):
      inplace_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for inplace parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, True}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'inplace_valresult' : inplace_valresult})
  
    #check Binary
    Binary_valresult = False
    if not isinstance(Binary, list) and Binary not in {True, False, 'retain', 'ordinal', 'ordinalretain', 'onehot', 'onehotretain'}:
      Binary_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for Binary parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'retain', 'ordinal', 'ordinalretain', 'onehot', 'onehotretain', [list]}")
      self.__autoprint('warning', '')
    elif not isinstance(Binary, list) \
    and not isinstance(Binary, bool) \
    and Binary not in {'retain', 'ordinal', 'ordinalretain', 'onehot', 'onehotretain'}:
      Binary_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for Binary parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'retain', 'ordinal', 'ordinalretain', 'onehot', 'onehotretain', [list]}")
      
    logger_dict['validations'].update({'Binary_valresult' : Binary_valresult})
    
    #check PCAn_components
    #accepts integers >1 or floats between 0-1, False, or None
    PCAn_components_valresult = False
    if isinstance(PCAn_components, (int, float)) \
    or PCAn_components is False \
    or PCAn_components == None:
      
      if PCAn_components is not False:

        if isinstance(PCAn_components, int):
          if PCAn_components < 1:
            PCAn_components_valresult = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "Error: invalid entry passed for PCAn_components")
            self.__autoprint('warning', "Acceptable values are integers > 1, floats between 0-1, False, or None.")
            self.__autoprint('warning', '')
          
        if isinstance(PCAn_components, float):
          if (PCAn_components > 1.0 or PCAn_components < 0.0):
            PCAn_components_valresult = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "Error: invalid entry passed for PCAn_components")
            self.__autoprint('warning', "Acceptable values are integers > 1, floats between 0-1, False, or None.")
            self.__autoprint('warning', '')

        if PCAn_components is True:
          PCAn_components_valresult = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "Error: invalid entry passed for PCAn_components")
          self.__autoprint('warning', "Acceptable values are integers > 1, floats between 0-1, False, or None.")
          self.__autoprint('warning', '')

    else:
      PCAn_components_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for PCAn_components")
      self.__autoprint('warning', "Acceptable values are integers > 1, floats between 0-1, False, or None.")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'PCAn_components_valresult' : PCAn_components_valresult})
    
    #check PCAexcl
    PCAexcl_valresult = False
    if not isinstance(PCAexcl, list):
      PCAexcl_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for PCAexcl parameter.")
      self.__autoprint('warning', "Acceptable values are a list of columns to exclude from PCA (defaults to empty list)")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'PCAexcl_valresult' : PCAexcl_valresult})
    
    #check printstatus
    printstatus_valresult = False
    if printstatus not in {True, False, 'summary', 'silent'} or \
    (printstatus in {True, False} and not isinstance(printstatus, bool)):
      printstatus_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for printstatus parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'summary', 'silent'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'printstatus_valresult' : printstatus_valresult})
    
    #check excl_suffix
    excl_suffix_valresult = False
    if excl_suffix not in {True, False} or not isinstance(excl_suffix, bool):
      excl_suffix_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for excl_suffix parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'excl_suffix_valresult' : excl_suffix_valresult})

    #check trainID_column
    trainID_column_valresult = False
    if trainID_column is not False \
    and not isinstance(trainID_column, int) \
    and not isinstance(trainID_column, str) \
    and not isinstance(trainID_column, list):
      trainID_column_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for trainID_column parameter.")
      self.__autoprint('warning', "trainID_column allowable values are False, int, str, or list.")

    logger_dict['validations'].update({'trainID_column_valresult' : trainID_column_valresult})

    #check testID_column
    testID_column_valresult = False
    if testID_column is not False \
    and not isinstance(testID_column, int) \
    and not isinstance(testID_column, str) \
    and not isinstance(testID_column, list):
      testID_column_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for testID_column parameter.")
      self.__autoprint('warning', "testID_column allowable values are False, int, str, or list.")

    logger_dict['validations'].update({'testID_column_valresult' : testID_column_valresult})

    #check evalcat
    evalcat_valresult = False
    if evalcat is not False \
    and callable(evalcat):
      evalcat_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for evalcat parameter.")
      self.__autoprint('warning', "evalcat allowable values are False or as a callable function per READ ME.")
      
    logger_dict['validations'].update({'evalcat_valresult' : evalcat_valresult})

    #check privacy_encode
    privacy_encode_valresult = False
    if privacy_encode not in {True, False, 'private'}:
      privacy_encode_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for privacy_encode parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'private'}")
      self.__autoprint('warning', '')
    elif privacy_encode not in {'private'} \
    and not isinstance(privacy_encode, bool):
      privacy_encode_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for privacy_encode parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'private'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'privacy_encode_valresult' : privacy_encode_valresult})

    #check encrypt_key
    encrypt_key_valresult = False
    if not isinstance(encrypt_key, (int, bytes, bool)) \
    or isinstance(encrypt_key, int) and encrypt_key not in {False, 16, 24, 32} \
    or isinstance(encrypt_key, bytes) and len(encrypt_key) not in {16, 24, 32}:
      encrypt_key_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for encrypt_key parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, 16, 24, 32, or a bytes object}")
      self.__autoprint('warning', "When passed as a bytes object the length can be one of {16, 24, 32}")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'encrypt_key_valresult' : encrypt_key_valresult})

    #check noise_augment
    noise_augment_valresult = False
    if not isinstance(noise_augment, (int, float)) \
    or isinstance(noise_augment, (int, float)) and noise_augment < 0:
      noise_augment_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for noise_augment parameter.")
      self.__autoprint('warning', "Acceptable values are non-negative integers (or float integers)")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'noise_augment_valresult' : noise_augment_valresult})

    #check ppd_append
    ppd_append_valresult = False
    if not isinstance(ppd_append, (bool, dict)) \
    or isinstance(ppd_append, (bool)) and ppd_append is not False:
      ppd_append_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for ppd_append parameter.")
      self.__autoprint('warning', "Acceptable values are False or dictionary")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'ppd_append_valresult' : ppd_append_valresult})

    #check orig_headers
    orig_headers_valresult = False
    if orig_headers not in {True, False} or not isinstance(orig_headers, bool):
      orig_headers_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for orig_headers parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'orig_headers_valresult' : orig_headers_valresult})

    #check cat_type
    cat_type_valresult = False
    if cat_type not in {True, False} or not isinstance(cat_type, bool):
      cat_type_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for cat_type parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'cat_type_valresult' : cat_type_valresult})

    #check logger_dict
    logger_valresult = False
    if not isinstance(logger, dict):
      logger_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for logger parameter.")
      self.__autoprint('warning', "Acceptable values are dict type")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'logger_valresult' : logger_valresult})

    logger = self.__logger_append(logger)

    return
    
  def __check_pm_miscparameters(self, pandasoutput, printstatus, TrainLabelFreqLevel, \
                                dupl_rows, featureeval, driftreport, inplace, \
                                returnedsets, shuffletrain, inversion, traindata, \
                                testID_column, randomseed, encrypt_key, noise_augment, logger):
    """
    #Performs validation to confirm valid entries of passed postmunge(.) parameters
    #note one parameter not directly passed is df_test, just pass a list of the columns
    #returns a dictionary of results
    #False is good
    """
    
    #check pandasoutput
    pandasoutput_valresult = False
    if pandasoutput in {True, False} and not isinstance(pandasoutput, bool) \
    or pandasoutput not in {True, False, 'dataframe'}:
      pandasoutput_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for pandasoutput parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'dataframe'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'pandasoutput_valresult' : pandasoutput_valresult})
    
    #check printstatus
    printstatus_valresult = False
    if printstatus not in {True, False, 'summary', 'silent'} or \
    printstatus in {True, False} and not isinstance(printstatus, bool):
      printstatus_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for printstatus parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'summary', 'silent'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'printstatus_valresult' : printstatus_valresult})
    
    #check inversion
    inversion_valresult = False
    if not isinstance(inversion, list) and not isinstance(inversion, set) and inversion not in {False, 'test', 'labels', 'denselabels'}:
      inversion_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for inversion parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, 'test', 'labels', 'denselabels', a list of columns, or a set of a single returned column}")
      self.__autoprint('warning', '')
    elif not isinstance(inversion, list) and not isinstance(inversion, set) and inversion not in {'test', 'labels', 'denselabels'} \
    and not isinstance(inversion, bool):
      inversion_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for inversion parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, 'test', 'labels', 'denselabels', a list of columns, or a set of a single returned column}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'inversion_valresult' : inversion_valresult})
    
    #check TrainLabelFreqLevel
    TrainLabelFreqLevel_valresult = False
    if TrainLabelFreqLevel not in {True, False} or not isinstance(TrainLabelFreqLevel, bool):
      TrainLabelFreqLevel_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'TrainLabelFreqLevel_valresult' : TrainLabelFreqLevel_valresult})

    #check dupl_rows
    dupl_rows_valresult = False
    if dupl_rows not in {True, False} or not isinstance(dupl_rows, bool):
      dupl_rows_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for dupl_rows parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'dupl_rows_valresult' : dupl_rows_valresult})
    
    #check featureeval
    featureeval_valresult = False
    if featureeval not in {True, False} or not isinstance(featureeval, bool):
      featureeval_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for featureeval parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'featureeval_valresult' : featureeval_valresult})
    
    #check driftreport
    driftreport_valresult = False
    if driftreport not in {True, False, 'efficient', 'report_effic', 'report_full'}:
      driftreport_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for driftreport parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'efficient', 'report_effic', 'report_full'}")
      self.__autoprint('warning', '')
    elif driftreport not in {'efficient', 'report_effic', 'report_full'} \
    and not isinstance(driftreport, bool):
      driftreport_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for driftreport parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'efficient', 'report_effic', 'report_full'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'driftreport_valresult' : driftreport_valresult})

    #check inplace
    inplace_valresult = False
    if inplace not in {True, False} or not isinstance(inplace, bool):
      inplace_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for inplace parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, True}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'inplace_valresult' : inplace_valresult})
    
    #check returnedsets
    returnedsets_valresult = False
    if returnedsets not in {True, False, 'test_ID', 'test_labels', 'test_ID_labels'}:
      returnedsets_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for returnedsets parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test_ID', 'test_labels', 'test_ID_labels'}")
      self.__autoprint('warning', '')
    elif returnedsets not in {'test_ID', 'test_labels', 'test_ID_labels'} \
    and not isinstance(returnedsets, bool):
      returnedsets_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for returnedsets parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'test_ID', 'test_labels', 'test_ID_labels'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'returnedsets_valresult' : returnedsets_valresult})
    
    #check shuffletrain
    shuffletrain_valresult = False
    if shuffletrain not in {True, False} or not isinstance(shuffletrain, bool):
      shuffletrain_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for shuffletrain parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'shuffletrain_valresult' : shuffletrain_valresult})

    #check traindata
    traindata_valresult = False
    if traindata not in {True, False, 'train_no_noise', 'test_no_noise'} \
    or traindata in {True, False} and not isinstance(traindata, bool):
      traindata_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for traindata parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'train_no_noise', 'test_no_noise'}")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'traindata_valresult' : traindata_valresult})

    #check testID_column
    testID_column_valresult = False
    if testID_column is not False \
    and not isinstance(testID_column, int) \
    and not isinstance(testID_column, str) \
    and not isinstance(testID_column, list):
      testID_column_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for testID_column parameter.")
      self.__autoprint('warning', "testID_column allowable values are boolean False, int, string, or list.")

    logger_dict['validations'].update({'testID_column_valresult' : testID_column_valresult})

    #check randomseed
    randomseed_valresult = False
    if not isinstance(randomseed, (int)) or randomseed is True:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**32-1 or False")
      self.__autoprint('warning', '')
    elif randomseed < 0 or randomseed > 2147483647:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**31-1 or False")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'randomseed_valresult' : randomseed_valresult})

    #check encrypt_key
    encrypt_key_valresult = False
    if not isinstance(encrypt_key, (bytes, bool)) \
    or isinstance(encrypt_key, bytes) and len(encrypt_key) not in {16, 24, 32}:
      encrypt_key_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for encrypt_key parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, or a bytes object}")
      self.__autoprint('warning', "When passed as a bytes object the length can be one of {16, 24, 32}")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'encrypt_key_valresult' : encrypt_key_valresult})

    #check noise_augment
    noise_augment_pm_valresult = False
    if not isinstance(noise_augment, (int, float)) \
    or isinstance(noise_augment, (int, float)) and noise_augment < 0:
      noise_augment_pm_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for noise_augment parameter.")
      self.__autoprint('warning', "Acceptable values are non-negative integers (or float integers)")
      self.__autoprint('warning', '')

    logger_dict['validations'].update({'noise_augment_pm_valresult' : noise_augment_pm_valresult})

    #check logger
    logger_pm_valresult = False
    if not isinstance(logger, dict):
      logger_pm_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for logger parameter.")
      self.__autoprint('warning', "Acceptable values are dict type")
      self.__autoprint('warning', '')
      
    logger_dict['validations'].update({'logger_pm_valresult' : logger_pm_valresult})

    logger = self.__logger_append(logger)
    
    return

  def __check_FSmodel(self, featureselection, FSmodel, printstatus):
    """
    If feature importance applied confirms that a model was successfully trained
    """

    check_FSmodel_result = False
    
    if featureselection is not False:
      if FSmodel is False:
        check_FSmodel_result = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: Feature importance model was not successfully trained")
        self.__autoprint('warning', '')
        
    return check_FSmodel_result

  def __check_df_type(self, df_train, df_test, printstatus):
    """
    Validates that df_train is one of np.array, pd.Series, pd.Dataframe
    Valides that df_test is one of those or is boolean False
    False is good
    """

    check_df_train_type_result = False
    check_df_test_type_result = False
    
    check_df_train_type_result = \
    not isinstance(df_train, (type(np.array([])), type(pd.Series([1])), type(pd.DataFrame())))
    
    check_df_test_type_result = \
    not isinstance(df_test, (type(np.array([])), type(pd.Series([1])), type(pd.DataFrame()), bool))
    
    if df_test is True:
      check_df_test_type_result = True

    if check_df_train_type_result is True and printstatus != 'silent':
      print("error: df_train received as type other than pd.DataFrame, pd.Series, or np.array.")

    if check_df_test_type_result is True and printstatus != 'silent':
      print("error: df_test received as type other than pd.DataFrame, pd.Series, np.array, or boolean False.")
    
    return check_df_train_type_result, check_df_test_type_result

  def __check_np_shape(self, df_train, df_test, printstatus):
    """
    Validates any passed numpy arrays are tabular (1D or 2D)
    """

    check_np_shape_train_result = False
    check_np_shape_test_result = False
    
    checknp = np.array([])
    
    if isinstance(checknp, type(df_train)):
      if len(df_train.shape) > 2:
        check_np_shape_train_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: numpy array passed to df_train is not tabular (>2D dimensions)")
        self.__autoprint('warning', '')
        
    if isinstance(checknp, type(df_test)):
      if len(df_test.shape) > 2:
        check_np_shape_test_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: numpy array passed to df_test is not tabular (>2D dimensions)")
        self.__autoprint('warning', '')
        
    return check_np_shape_train_result, check_np_shape_test_result

  def __check_ML_infill(self, printstatus, df_train, column, postprocess_dict, logger, infill_validations = {}):
    """
    #Perform validations that train set is suitable for MLinfill
    #For example ML infill requires >1 source columns in df_train

    #further validations of all valid numeric are performed after partitioning nonnumeric sets in createMLinfillsets
    """

    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    
    if 'MLinfill_validations' not in infill_validations:
      infill_validations.update({'MLinfill_validations':{}})
      logger_dict['validations'].update({'MLinfill_validations':{}})
    
      if len(columnslist) == len(list(df_train)):
        
        # if postprocess_dict['printstatus'] != 'silent':
        self.__autoprint('warning', "ML infill requires > 1 source features in df_train, proceeding without ML infill")
        self.__autoprint('warning', '')
        
        infill_validations.update({'MLinfill_validations': True})
        logger_dict['validations'].update({'MLinfill_validations': True})
          
      else:
        
        infill_validations.update({'MLinfill_validations': False})
        logger_dict['validations'].update({'MLinfill_validations': False})

      logger = self.__logger_append(logger)
      
    return infill_validations

  def __check_ML_infill_2(self, df_train_filltrain, df_train_filllabel, 
                         df_train_fillfeatures, df_test_fillfeatures, printstatus,
                         column, postprocess_dict, logger, ampm = 'am'):
    """
    #runs validations and records results for each of sets returned from _createMLinfillsets
    #using _validate_allvalidnumeric
    #results recorded in postprocess_dict[reportlocation]
    #this function applied in _MLinfillfunction
    #reportlocation and ampm used to distinguish between use for automunge vs postmunge MLinfill function
    """

    if ampm == 'am' and 'df_train_filltrain_numeric_data_result' not in logger_dict['validations']:
      logger_dict['validations'].update ({'df_train_filltrain_numeric_data_result' : {},
                                          'df_train_filltrain_all_valid_entries_result' : {},
                                          'df_train_filllabel_numeric_data_result' : {},
                                          'df_train_filllabel_all_valid_entries_result' : {},
                                          'df_train_fillfeatures_numeric_data_result' : {},
                                          'df_train_fillfeatures_all_valid_entries_result' : {},
                                          'df_test_fillfeatures_numeric_data_result' : {},
                                          'df_test_fillfeatures_all_valid_entries_result' : {},
                                          })
    
    if ampm == 'pm' and 'df_test_fillfeatures_numeric_data_result' not in logger_dict['validations']:
      logger_dict['validations'].update ({'df_test_fillfeatures_numeric_data_result' : {},
                                                'df_test_fillfeatures_all_valid_entries_result' : {},
                                               })
    
    if ampm == 'am':

      numeric_data_result, all_valid_entries_result = \
      self.__validate_allvalidnumeric(df_train_filltrain, printstatus)
      logger_dict['validations']['df_train_filltrain_numeric_data_result'].update({column : numeric_data_result})
      logger_dict['validations']['df_train_filltrain_all_valid_entries_result'].update({column : all_valid_entries_result})

      numeric_data_result, all_valid_entries_result = \
      self.__validate_allvalidnumeric(df_train_filllabel, printstatus)
      logger_dict['validations']['df_train_filllabel_numeric_data_result'].update({column : numeric_data_result})
      logger_dict['validations']['df_train_filllabel_all_valid_entries_result'].update({column : all_valid_entries_result})

      numeric_data_result, all_valid_entries_result = \
      self.__validate_allvalidnumeric(df_train_fillfeatures, printstatus)
      logger_dict['validations']['df_train_fillfeatures_numeric_data_result'].update({column : numeric_data_result})
      logger_dict['validations']['df_train_fillfeatures_all_valid_entries_result'].update({column : all_valid_entries_result})
    
    #df_test_fillfeatures run for both ampm scenarios
    numeric_data_result, all_valid_entries_result = \
    self.__validate_allvalidnumeric(df_test_fillfeatures, printstatus)
    logger_dict['validations']['df_test_fillfeatures_numeric_data_result'].update({column : numeric_data_result})
    logger_dict['validations']['df_test_fillfeatures_all_valid_entries_result'].update({column : all_valid_entries_result})
    
    logger = self.__logger_append(logger)

    return postprocess_dict

  def __validate_allvalidnumeric(self, df, printstatus):
    """
    #some methods in library, such as ML infilll, PCA, and feature importance, 
    #require all numeric data with all valid entries
    #most transforms always meet this requirement
    #but there are a few special cases where returned data may not be all numeric
    #such as excl, copy, shfl, strg
    #this function validates that a data set meets requirement of all numeric and all valid entries
    #False is good
    """

    numeric_data_result = False
    all_valid_entries_result = False
    
    #first ensure data is all numeric
    if df.shape[1] != df.select_dtypes(include=np.number).shape[1]:
      
      numeric_data_result = True
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: data was passed to ML infill, PCA, or feature importance with non-numeric data.")
      self.__autoprint('warning', "Some transforms in library may not convert data to numeric, such as the passthrough transform excl.")
      self.__autoprint('warning', "Alternatives to excl for pass-through with force to numeric and infill are available as exc2 - exc8.")
      self.__autoprint('warning', "Note that ML_cmnd designated exclusions may be used to circumvent this error, such as ML_cmnd['full_exclude']")
      self.__autoprint('warning', "This printout will still return in that case.")
      self.__autoprint('warning', '')
    
    #then check for all valid entries
    if df.isna().values.sum() > 0:
      
      all_valid_entries_result = True
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: data was passed to ML infill, PCA, or feature importance with missing entries (NaN values).")
      self.__autoprint('warning', "Some transforms in library may not conduct infill, such as the passthrough transform excl.")
      self.__autoprint('warning', "Alternatives to excl for pass-through with force to numeric and infill are available as exc2 - exc8.")
      self.__autoprint('warning', "Note that ML_cmnd designated exclusions may be used to circumvent this error, such as ML_cmnd['full_exclude']")
      self.__autoprint('warning', "This printout will still return in that case.")
      self.__autoprint('warning', '')
    
    return numeric_data_result, all_valid_entries_result

  def __check_assigncat(self, assigncat, printstatus):
    """
    #Here we'll do a quick check for any redundant column assignments in the
    #assigncat, if any found return an error message
    """

    aggregated_set = set()
    aggregated_list = []
    prior_identified_redundant = set()

    check_assigncat_result = False

    for key, value in assigncat.items():

      #value will either be a list or set after going through __assigncat_str_convert
      if isinstance(value, list):

        value_sets = [x for x in value if isinstance(x, set)]
        for value_set in value_sets:
          
          aggregated_set = aggregated_set | value_set
          aggregated_list = aggregated_list + list(value_set)

        value_list = [x for x in value if not isinstance(x, set)]

        aggregated_list = aggregated_list + value_list
        aggregated_set = aggregated_set | set(value_list)
      
      #set specification converted to [set()] in __assigncat_str_convert
#       elif isinstance(value, set):
#         aggregated_set = aggregated_set | value
#         aggregated_list = aggregated_list + list(value)

      if len(aggregated_list) > len(aggregated_set):

        aggregated_list_copy = aggregated_list.copy()
        for entry in aggregated_set:
          aggregated_list_copy.remove(entry)
          
        if len(set(aggregated_list_copy) - prior_identified_redundant) > 0:

          prior_identified_redundant = prior_identified_redundant | set(aggregated_list_copy)

          check_assigncat_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "Warning of expected error due to redundant column assignment in assigncat")
          self.__autoprint('warning', "for assignments to: ", key)
          self.__autoprint('warning', "for column headers: ", aggregated_list_copy)

    return check_assigncat_result
  
  def __check_assigncat2(self, assigncat, transform_dict, printstatus):
    """
    #Here we'll do a quick check to ensure all of the keys of passed assigncat
    #have corresponding entries in transform_dict, (which may include user
    #passed entries in transformdict parameter)
    
    #Note that in many cases a root category may be passed as a family tree primitive 
    #entry to it's own family tree set. The root category is used to access the family tree,
    #and the family tree primitive entries are used to access the transform functions and etc.
    """

    #False is good
    result = False
    
    for assigncat_key in assigncat:
      
      #eval is a special case, it triggers the application of evalcategory
      #which may be neccesary when automated inference turned off with powertransform
      #so it doesn't need a process_dit entry
      if assigncat_key not in transform_dict and assigncat_key not in {'eval', 'ptfm'}:
        
        result = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "Error, the following entry to user passed assigncat was not found")
        self.__autoprint('warning', "to have a corresponding entry in transform_dict.")
        self.__autoprint('warning', "")
        self.__autoprint('warning', "assigncat key missing transform_dict entry: ", assigncat_key)
        self.__autoprint('warning', "")

    return result
  
  def __check_assigncat3(self, assigncat, process_dict, transform_dict, printstatus):
    """
    #Here's we'll do a third check on assigncat
    #to ensure that for any listed root categories, 
    #any category entries to corresponding family tree primitives in transform_dict 
    #have a corresponding entry in the process_dict
    """

    #False is good
    result = False
    
    for assigncat_key in assigncat:
      
      if assigncat_key in transform_dict:
        
        familytree_entries = []
        
        for familytree_key in transform_dict[assigncat_key]:
          
          familytree_entries += transform_dict[assigncat_key][familytree_key]
          
        for familytree_entry in familytree_entries:
          
          if familytree_entry != None:

            if familytree_entry not in process_dict:

              # if printstatus != 'silent':
              self.__autoprint('warning', "Error, the following category was found as an entry")
              self.__autoprint('warning', "in a family tree without a corresponding entry ")
              self.__autoprint('warning', "in the process_dict.")
              self.__autoprint('warning', "")
              self.__autoprint('warning', "family tree entry missing process_dict entry: ", familytree_entry)
              self.__autoprint('warning', "this entry was passed in the family tree of root category: ", assigncat_key)

              result = True
        
    return result

  def __create_inverse_assigncat(self, assigncat):
    """
    #This will invert entries in assigncat 
    #to make more efficient accessing category associated with a column
    #where assigncat has entries at this point {'category' : ['column1', 'column2']}
    #and so inverse_assigncat translates to eg
    #{'column1':'category', 'column2':'category'}
    """

    inverse_assigncat = {}

    for entry1 in assigncat:
      for entry2 in assigncat[entry1]:
        inverse_assigncat.update({entry2 : entry1})

    #in cases of messy_data there may be redundant specifications to null and from evalcategory
    #so strip the redundancy, null governs
    if 'null' in assigncat:
      for entry1 in assigncat['null']:
        if entry1 in inverse_assigncat \
        and inverse_assigncat[entry1] != 'null':
          inverse_assigncat[entry1] = 'null'
        
    return inverse_assigncat

  def __check_assigninfill(self, assigninfill, printstatus):
    """
    #Here we'll do a quick check for any redundant column assignments in the
    #assigninfill, if any found return an error message
    """

    assigninfill_redundant_dict = {}
    result = False

    for assigninfill_key1 in sorted(assigninfill):
      #assigncat_list.append(set(assigncat[key]))
      current_set = set(assigninfill[assigninfill_key1])
      redundant_items = {}
      for assigninfill_key2 in assigninfill:
        if assigninfill_key2 != assigninfill_key1:
          second_set = set(assigninfill[assigninfill_key2])
          common = current_set & second_set
          if len(common) > 0:
            for common_item in common:
              if common_item not in assigninfill_redundant_dict:
                assigninfill_redundant_dict.update({common_item:[assigninfill_key1, assigninfill_key2]})
              else:
                if assigninfill_key1 not in assigninfill_redundant_dict[common_item]:
                  assigninfill_redundant_dict[common_item].append(assigninfill_key1)
                  #assigncat_redundant_dict[common_item] += key1
                if assigninfill_key2 not in assigninfill_redundant_dict[common_item]:
                  assigninfill_redundant_dict[common_item].append(assigninfill_key2)
                  #assigncat_redundant_dict[common_item] += key2

    #assigncat_redundant_dict

    if len(assigninfill_redundant_dict) > 0:
      result = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error, the following columns assigned to multiple root categories in assigninfill:")
      for assigninfill_key3 in sorted(assigninfill_redundant_dict):
        self.__autoprint('warning', "")
        self.__autoprint('warning', "Column: ", assigninfill_key3)
        self.__autoprint('warning', "Found in following assigninfill entries:")
        self.__autoprint('warning', assigninfill_redundant_dict[assigninfill_key3])
        self.__autoprint('warning', "")
    
    return result

  def __check_transformdict000(self, transformdict, printstatus):
    """
    #Validation of transformdict format
    #ensures that each root category key has values of a dicitonary
    #to ensure check_transformdict0 will run properly
    #also ensures primitives are valid / spelled properly
    """

    result1 = False
    result2 = False
    
    primitives_set = {'parents', 'siblings', 'auntsuncles', 'cousins', 
                      'children', 'niecesnephews', 'coworkers', 'friends'}
    
    for root in transformdict:
      
      if not isinstance(transformdict[root], type({})):
        
        result1 = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "Error: transformdict entry for root category ", root)
        self.__autoprint('warning', "was passed without value of a dictionary for primitives.")
        
      else:
        
        #this test is any of primitives aren't spelled properly or something
        if len(primitives_set & set(transformdict[root])) < len(set(transformdict[root])):
          
          result2 = True
          
          # if printstatus != 'silent':
          self.__autoprint('warning', "Error: transformdict entry for root category ", root)
          self.__autoprint('warning', "was passed with invalid primitives.")
          
    return result1, result2

  def __check_transformdict00(self, transformdict, printstatus):
    """
    #Validation of primitive entries format
    #checks that entries are a list of strings
    #or if entry is a string embeds in a list
    """

    result = False
    
    for root in transformdict:
      
      for primitive in transformdict[root]:
        
        if isinstance(transformdict[root][primitive], type('string')):
          
          transformdict[root][primitive] = [transformdict[root][primitive]]
          
        elif isinstance(transformdict[root][primitive], type([])):
          
          for entry in transformdict[root][primitive]:
            
            if not isinstance(entry, type('string')):
              
              result = True
              
              # if printstatus != 'silent':
              self.__autoprint('warning', "Error: user passed transformdict for root category ", root)
              self.__autoprint('warning', "Contained an entry to primitive ", primitive)
              self.__autoprint('warning', "that was not a valid data type.")
              self.__autoprint('warning', "Data type should be a string (representing a transformation category).")
              
    return result, transformdict

  def __check_transformdict0(self, transformdict, printstatus):
    """
    #For cases where user passes trasnformdict root category with partial family tree
    #populates the other primitives as empty sets
    #This will make user specfications much easier / less typing
    """

    result = False
    
    primitives_set = {'parents', 'siblings', 'auntsuncles', 'cousins', 
                      'children', 'niecesnephews', 'coworkers', 'friends'}
    
    for root in transformdict:
      #this checks that at least one primitive specified
      if len(primitives_set & set(transformdict[root])) > 0:
        
        #then any other primitives not yet populated are added without entries
        for primitive in primitives_set:
        
          if primitive not in transformdict[root]:
             
             transformdict[root].update({primitive : []})
      
      #else if no primitives are present
      else:
        result = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "Error, transformdict entry for root category ", root)
        self.__autoprint('warning', "was passed without any primitives populated.")
      
    return result, transformdict

  def __check_transformdict(self, transformdict, printstatus):
    """
    #Here we'll do a quick check for any entries in the user passed
    #transformdict which don't have at least one replacement column specified
    #and if not found apply an auntsuncles excl transform
    """

    result1 = False
    result2 = False
    
    for transformkey in sorted(transformdict):
      replacements = len(transformdict[transformkey]['parents']) \
                     + len(transformdict[transformkey]['auntsuncles'])

      if replacements == 0:
        
        transformdict[transformkey]['auntsuncles'].append('excl')

        #(This isn't an error just how we accomodate scenario to support populating data structures.)
        # # if printstatus != 'silent':
        # self.__autoprint('warning', "family tree defined without replacement primitive in upstream primitives")
        # self.__autoprint('warning', "for category ", transformkey)
        # self.__autoprint('warning', "adding an 'excl' trasnform to auntsuncles which is direct passthrough.")

        result1 = True
          
      #this ensures 'excl' is final transform in the auntsuncles list if included
      #such as to prioritize applying inplace on excl
      transformdict[transformkey]['auntsuncles'].append('excl')
      transformdict[transformkey]['auntsuncles'].remove('excl')

    return result1, transformdict
  
  def __check_transformdict2(self, transformdict, printstatus):
    """
    #Here we'll do an additional check on transformdict to ensure
    #no redundant specifications in adjacent primitives
    """

    result1 = False
    result2 = False
    
    upstream_entries = []
    downstream_entries = []
    
    for transformkey in sorted(transformdict):
      
      for primitive in transformdict[transformkey]:
        
        if primitive in {'parents', 'siblings', 'auntsuncles', 'cousins'}:
          
          for entry in transformdict[transformkey][primitive]:
            
            if entry in upstream_entries:
              
              result1 = True
              
              # if printstatus != 'silent':
              self.__autoprint('warning', "error warning: ")
              self.__autoprint('warning', "redundant entries found in the upstream primitives ")
              self.__autoprint('warning', "for user-passed transformdict key: ", transformkey)
              
            else:
          
              upstream_entries += [entry]
          
        if primitive in {'children', 'niecesnephews', 'coworkers', 'friends'}:
          
          for entry in transformdict[transformkey][primitive]:
            
            if entry in downstream_entries:
              
              result2 = True
              
              # if printstatus != 'silent':
              self.__autoprint('warning', "error warning: ")
              self.__autoprint('warning', "redundant entries found in the downstream primitives ")
              self.__autoprint('warning', "for user-passed transformdict key: ", transformkey)
              
            else:
          
              downstream_entries += [entry]

      upstream_entries = []
      downstream_entries = []

    return result1, result2

  def __check_transform_dict_roots(self, transform_dict, process_dict, printstatus):
    """
    #validates that transform_dict root categories after consolidation
    #have corresponding entries in process_dict after consolidation
    """

    check_transform_dict_roots_result = False
    
    for entry in transform_dict:
      
      if entry not in process_dict:
        
        check_transform_dict_roots_result = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: a root category was found in transformdict")
        self.__autoprint('warning', "without a corresponding entry in processdict")
        self.__autoprint('warning', "for transformdict root category: ", entry)
        self.__autoprint('warning', '')
        
    return check_transform_dict_roots_result
  
  def __check_haltingproblem(self, transformdict, transform_dict, printstatus, max_check_count = 1111):
    """
    #evaluates user passed transformdict entries to check for infinite loops
    #we'll arbitrarily check for a max depth of 1111 offspring to keep things manageable
    #we'll assume this takes place after user passed entries have been merged into transform_dict
    #such that transformdict is just user passed entries
    #and transform_dict is both user-passed and internal library

    #familytree_for_offspring_result triggered when category entered to primitive with offpsring
    #doesn't have a root category definition in trasnform_dict
    """

    haltingproblem_result = False
    familytree_for_offspring_result = False
    
    for root_category in transformdict:
      
      check_count = 0
      
      offspring_list = []
      
      parents_list = \
      transform_dict[root_category]['parents'] + transform_dict[root_category]['siblings']
      
      for parent in parents_list:

        if parent not in transform_dict:
          familytree_for_offspring_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "Error, no root category family tree defined for category ", parent)
          self.__autoprint('warning', "which is needed since it is entered as an entry to a primitive with offpsinrg")
          self.__autoprint('warning', "in family tree for root category ", root_category)
          self.__autoprint('warning', '')
        
        upstream_list = []
        upstream_list = [parent]
        
        offspring_list = \
        transform_dict[parent]['children'] + transform_dict[parent]['niecesnephews']
      
        if len(offspring_list) > 0:

          for offspring in offspring_list:

            check_count += 1

            if offspring in upstream_list:

              haltingproblem_result = True

              # if printstatus != 'silent':
              self.__autoprint('warning', "Error, infinite loop detected in transformdict for root category ", root_category)
              self.__autoprint('warning', '')

              break

            upstream_list += [offspring]

            if check_count < max_check_count:

              offspring_result, offspring_familytree_for_offspring_result, check_count = \
              self.__check_offspring(transform_dict, offspring, root_category, \
                                   upstream_list, check_count, max_check_count, printstatus)
#               offspring_result, check_count = \
#               check_offspring(transform_dict, offspring, root_category, \
#                               upstream_list, check_count, max_check_count)

              if offspring_result is True:
                haltingproblem_result = True
                break

              if offspring_familytree_for_offspring_result is True:
                familytree_for_offspring_result = True
                break

            else:

              haltingproblem_result = True

              # if printstatus != 'silent':
              self.__autoprint('warning', "Number of offspring generations for root category ", root_category)
              self.__autoprint('warning', "exceeded 1111, infinite loop check halted.")
              self.__autoprint('warning', '')
              
              break
    
    return haltingproblem_result, familytree_for_offspring_result

  def __check_offspring(self, transform_dict, root_category, orig_root_category, \
                      upstream_list, check_count, max_check_count, printstatus):
    """
    #support function for check_haltingproblem
    """

    offspring_result = False
    offspring_familytree_for_offspring_result = False

    if root_category not in transform_dict:
      offspring_familytree_for_offspring_result = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error, no root category family tree defined for category ", root_category)
      self.__autoprint('warning', "which is needed since it is entered as an entry to a primitive with offpsinrg")
      self.__autoprint('warning', "in family tree for root category ", orig_root_category)
      self.__autoprint('warning', '')

    offspring_list = \
    transform_dict[root_category]['children'] + transform_dict[root_category]['niecesnephews']

    if len(offspring_list) > 0:

      for offspring in offspring_list:
        
        check_count += 1

        if offspring in upstream_list:

          offspring_result = True

          # if printstatus != 'silent':
          self.__autoprint('warning', "Error, infinite loop detected in transformdict for root category ", orig_root_category)
          self.__autoprint('warning', '')

          break

        else:
            
          upstream_list2 = upstream_list.copy() + [offspring]

          if check_count < max_check_count:

            offspring_result2, offspring_familytree_for_offspring_result2, check_count = \
            self.__check_offspring(transform_dict, offspring, orig_root_category, \
                                 upstream_list, check_count, max_check_count, printstatus)
#             offspring_result2, check_count = \
#             check_offspring(transform_dict, offspring, orig_root_category, \
#                             upstream_list2, check_count, max_check_count)

            if offspring_result2 is True:
              offspring_result = True
              break

            if offspring_familytree_for_offspring_result2 is True:
              offspring_familytree_for_offspring_result = True
              break

          else:

            offspring_result = True

            # if printstatus != 'silent':
            self.__autoprint('warning', "Number of offspring generations for root category ", root_category)
            self.__autoprint('warning', "exceeded 1111, infinite loop check halted.")
            self.__autoprint('warning', '')
            
            break
    
    return offspring_result, offspring_familytree_for_offspring_result, check_count

  def __check_assignnan(self, assignnan, transform_dict, df_train_list, printstatus):
    """
    #validates automunge parameter assignnan
    #which accepts form:
    #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}
    
    #where 'cat1' / 'cat2' are examples of root categories
    #and 'col1' / 'col2' are examples of received source columns
    #and the lists contain entries for those source columns or root categories
    #which are to be converted to nan for purposes of infill
    
    #this function confirms only entries for {'categories', 'columns', 'global', 'injections'} in first tier
    #and that entries within categories are valid root categories from transform_dict
    #and that entries within columns are valid column headers from df_train
    #and that entries within injections are valid column headers from df_train
    """

    check_assignnan_toplevelentries_result = False
    check_assignnan_categories_result = False
    check_assignnan_columns_result = False
    
    for entry1 in assignnan:
      
      if entry1 not in {'categories', 'columns', 'global', 'injections'}:
        
        check_assignnan_toplevelentries_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: assignnan parameter valid entries for first tier are 'categories', 'columns', 'global', and 'injections'")
        self.__autoprint('warning', '')
        
    if 'categories' in assignnan:
      
      for entry2 in assignnan['categories']:        
        
        if entry2 not in transform_dict:
          
          check_assignnan_categories_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error: assignnan parameter valid entries under 'categories' must be root categories defined in transform_dict")
          self.__autoprint('warning', '')
          
    if 'columns' in assignnan:
      
      for entry2 in assignnan['columns']:        
        
        if entry2 not in df_train_list:
          
          check_assignnan_columns_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error: assignnan parameter valid entries under 'columns' must be source columns from passed df_train")
          self.__autoprint('warning', '')

    if 'injections' in assignnan:
      
      for entry3 in assignnan['injections']:        
        
        if entry3 not in df_train_list:
          
          check_assignnan_columns_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error: assignnan parameter valid entries under 'injections' must be source columns from passed df_train")
          self.__autoprint('warning', '')

    return check_assignnan_toplevelentries_result, check_assignnan_categories_result, check_assignnan_columns_result

  def __check_assignnan_injections(self, assignnan, columns_train, printstatus):
    """
    checks for valid assignnan actions under injections
    """

    assignnan_actions_valresult = False
    
    if isinstance(assignnan, dict):
    
      if 'injections' in assignnan:

        for columnkey in assignnan['injections']:
          if columnkey in columns_train:
            for actionkey in assignnan['injections'][columnkey]:
              if actionkey not in {'inject_ratio', 'range', 'minmax_range', 'entries', 'entry_ratio'}:

                assignnan_actions_valresult = True

                # if printstatus != 'silent':
                self.__autoprint('warning', "assignnan['injections'] has an invalid action entry")
                self.__autoprint('warning', "for column: ", columnkey)
                self.__autoprint('warning', "and action: ", actionkey)
                self.__autoprint('warning', "accepted form of injection specifications are documented in read me")
                
    return assignnan_actions_valresult

  def __check_ML_cmnd(self, ML_cmnd, printstatus):
    """
    #Here we'll do a quick check for any entries in the user passed
    #ML_cmnd and add any missing entries with default values
    #a future extension should validate any entries

    #also validates any entries to ML_cmnd['hyperparam_tuner'] are valid

    #also populates default for columns excluded from PCA unless already specified

    #note that current first tier commands accepted in ML_cmnd are:
    #'autoML_type', 'MLinfill_cmnd', 'hyperparam_tuner', 'randomCV_n_iter', 
    #'PCA_type', 'PCA_cmnd'
    """

    check_ML_cmnd_result = False
    
    def _populate_ML_cmnd_default(ML_cmnd, parameter, printstatus, check_ML_cmnd_result, default = False, valid_entries=False, valid_type = False):

      if parameter in ML_cmnd:
        if valid_entries is not False:
          if ML_cmnd[parameter] not in valid_entries:
            check_ML_cmnd_result = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "invalid entry passed to ML_cmnd key ", parameter)
            self.__autoprint('warning', "acceptable values are one of", valid_entries)
            self.__autoprint('warning', '')
              
        if valid_type is not False:
          if not isinstance(ML_cmnd[parameter], valid_type):
            check_ML_cmnd_result = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "invalid entry type passed to ML_cmnd key ", parameter)
            self.__autoprint('warning', "acceptable value type is", valid_type)
            self.__autoprint('warning', '')
        
      elif default != 'no_default_populated':
        ML_cmnd.update({parameter : default})
        
      return ML_cmnd, check_ML_cmnd_result
            
    #autoML_type is the library used for ML infill or feature importance, customML accesses user defined functions
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'autoML_type', 
                              printstatus, 
                              check_ML_cmnd_result,
                              default='randomforest', 
                              valid_entries={'randomforest', 'customML', 'flaml', 'catboost', 'xgboost'},
                              valid_type=str)
    
    #MLinfill_cmnd is used to pass parameters to MLinfill training
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'MLinfill_cmnd', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={}, 
                              valid_entries=False,
                              valid_type=dict)
    
    #ML_cmnd['MLinfill_cmnd']['RandomForestClassifier'] passes parameters to random forest clasification
    #expected as dicitonary {'parameter' : value}
    ML_cmnd['MLinfill_cmnd'], check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd['MLinfill_cmnd'], 
                              'RandomForestClassifier', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={}, 
                              valid_entries=False,
                              valid_type=dict)
    
    #ML_cmnd['MLinfill_cmnd']['RandomForestRegressor'] passes parameters to random forest regression
    #expected as dicitonary {'parameter' : value}
    ML_cmnd['MLinfill_cmnd'], check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd['MLinfill_cmnd'], 
                              'RandomForestRegressor', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={}, 
                              valid_entries=False,
                              valid_type=dict)
    
    #PCA_type selects which type of PCA to be applied when PCAn_components specified in automunge
    #default scenario defaults to kernel when all values non-negative, otherwise sparse
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'PCA_type', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default='default', 
                              valid_entries={'default', 'PCA', 'SparsePCA', 'KernelPCA'},
                              valid_type=str)
    
    #PCA_cmnd used to select options for PCA, including bool_ordl_PCAexcl, bool_PCA_excl, col_row_ratio
    #PCA_cmnd can also be used to pass parameters to PCA fit operation
    #e.g. ML_cmnd['PCA_cmnd']['kernel'] = {'parameter' : value}
    #PCA_cmnd distinguishes between automunge specific parameters and library options in init functions like __initSparsePCA
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'PCA_cmnd', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={'bool_ordl_PCAexcl' : True}, 
                              valid_entries=False,
                              valid_type=dict)
    
    #bool_ordl_PCAexcl excludes both boolean integer and ordinal integer encoded features from PCA 
    #(note some MLinfilltypes like 'exclude' are automatically excluded)
    if 'bool_PCA_excl' not in ML_cmnd['PCA_cmnd']:
      ML_cmnd['PCA_cmnd'], check_ML_cmnd_result = \
      _populate_ML_cmnd_default(ML_cmnd['PCA_cmnd'], 
                                'bool_ordl_PCAexcl', 
                                printstatus,  
                                check_ML_cmnd_result,
                                default=True, 
                                valid_entries={True, False},
                                valid_type=bool)
    else:
      #bool_PCA_excl if specified takes precedence over bool_ordl_PCAexcl
      #bool_PCA_excl excludes just boolean integer encoded features from PCA
      #(note some MLinfilltypes like 'exclude' are automatically excluded)
      ML_cmnd['PCA_cmnd'], check_ML_cmnd_result = \
      _populate_ML_cmnd_default(ML_cmnd['PCA_cmnd'], 
                                'bool_PCA_excl', 
                                printstatus,  
                                check_ML_cmnd_result,
                                default=True, 
                                valid_entries={True, False},
                                valid_type=bool)
    
    #As a special convention, if PCAn_components passed as _None_ PCA is performed when # features exceeds 0.5 # rows (as a heuristic). 
    #(The 0.5 value can also be updated in ML_cmnd by passing to ML_cmnd['PCA_cmnd']['col_row_ratio'].)
    ML_cmnd['PCA_cmnd'], check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd['PCA_cmnd'], 
                              'col_row_ratio', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.5, 
                              valid_entries=False,
                              valid_type=float)

    #PCA_retain when activated the PCA dimensionality reduciton supplements instead of replaces columns serving as basis
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'PCA_retain', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=False, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    #leakage_tolerance can be used to set tolerance or deactivate the automated leakage detection for ML infill exclusion
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'leakage_tolerance', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.85, 
                              valid_entries=False,
                              valid_type=(float, bool))
    
    #leakage_sets can be used to bidirectionally exclude particular features from each other's imputation model bases
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'leakage_sets', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=[], 
                              valid_entries=False,
                              valid_type=list)

    #full_exclude can be used to exclude a feature from all ML infill and PCA basis
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'full_exclude', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=[], 
                              valid_entries=False,
                              valid_type=list)
    
    #leakage_dict can be used to unidirectionally exclude features from another's basis
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'leakage_dict', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={}, 
                              valid_entries=False,
                              valid_type=dict)
    
    #hyperparam_tuner selects between types of hyperparameter tuning for randomforest or xgboost
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'hyperparam_tuner', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default='gridCV', 
                              valid_entries={'gridCV', 'randomCV', 'optuna_XG1'},
                              valid_type=str)
    
    #when randomCV hyperparam_tuner applied, randomCV_n_iter designates the number of iterations
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'randomCV_n_iter', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=100, 
                              valid_entries=False,
                              valid_type=int)

    #when optuna_XG1 hyperparam_tuner applied, optuna_n_iter designates the number of iterations per feature
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_n_iter', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=100, 
                              valid_entries=False,
                              valid_type=int)
    
    #when optuna_XG1 hyperparam_tuner applied, optuna_fasttune_ratio activates only inspecing one cross fold per trial
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_fasttune', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=True, 
                              valid_entries=False,
                              valid_type=bool)

    #when optuna_XG1 hyperparam_tuner applied, optuna_timeout designates the max tuning duration per feature
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_timeout', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=600, 
                              valid_entries=False,
                              valid_type=int)

    #when optuna_XG1 hyperparam_tuner applied, optuna_kfolds selects number of folds for cross validation in tuning
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_kfolds', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=5, 
                              valid_entries=False,
                              valid_type=int)

    #when optuna_XG1 hyperparam_tuner applied, optuna_early_stop selects number of iterations without improvement to trigger early stopping
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_early_stop', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=50, 
                              valid_entries=False,
                              valid_type=int)

    #optuna_max_depth_tuning_stepsize, for long tuning durations, we expect may be beneficial to change 2 to 1 (the optuna tutorial demonstrated defaulting to 2)
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'optuna_max_depth_tuning_stepsize', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=2, 
                              valid_entries=False,
                              valid_type=(int))
    
    #xgboost_gpu_id designated which GPU will be used to train xgboost
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'xgboost_gpu_id', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=False, 
                              valid_entries=False,
                              valid_type=(int, bool))
    
    #by default the random seed passed to model training is stochastic between applications, 
    #stochastic_training_seed can be used to set a fixed value matched to the global randomseed used for pandas
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_training_seed', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=True, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    #stochastic_impute_numeric activates the injection of stochasticity into ML infill targeting numeric features
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_numeric', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=True, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    #stochastic_impute_numeric_mu is mean of noise for stochastic_impute_numeric
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_numeric_mu', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.0, 
                              valid_entries=False,
                              valid_type=float)
    
    #stochastic_impute_numeric_sigma is scale of noise for stochastic_impute_numeric
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_numeric_sigma', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.03, 
                              valid_entries=False,
                              valid_type=float)
    
    #stochastic_impute_numeric_flip_prob is injection ratio of noise for stochastic_impute_numeric
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_numeric_flip_prob', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.06, 
                              valid_entries=False,
                              valid_type=float)
    
    #stochastic_impute_numeric_noisedistribution is distribution type for stochastic_impute_numeric
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_numeric_noisedistribution', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default='normal', 
                              valid_entries={'normal', 'laplace'},
                              valid_type=str)
    
    #stochastic_impute_categoric activates the injection of stochasticity into ML infill targeting categoric features
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_categoric', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=True, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    #stochastic_impute_categoric_flip_prob is injection ratio of noise for stochastic_impute_categoric
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'stochastic_impute_categoric_flip_prob', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.03, 
                              valid_entries=False,
                              valid_type=float)
    
    #halt_iterate activates early stopping assessment for ML infill infilliterate
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'halt_iterate', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=False, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    #categoric_tol sets categoric tolerance for early stopping criteria for halt_iterate (for ML infill infilliterate)
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'categoric_tol', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.05, 
                              valid_entries=False,
                              valid_type=float)
    
    #numeric_tol sets numeric tolerance for early stopping criteria for halt_iterate (for ML infill infilliterate)
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'numeric_tol', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default=0.03, 
                              valid_entries=False,
                              valid_type=float)
    
    #customML_inference_support used to support populating ML_cmnd, not returned in the final postprocess_dict to reduce memory overhead since is redundant with ML_cmnd
    ML_cmnd, check_ML_cmnd_result = \
    _populate_ML_cmnd_default(ML_cmnd, 
                              'customML_inference_support', 
                              printstatus,  
                              check_ML_cmnd_result,
                              default={}, 
                              valid_entries=False,
                              valid_type=dict)
    
    return check_ML_cmnd_result, ML_cmnd

  def __check_sampling_dict(self, sampling_dict, printstatus, random_generator_accepts_seeds):
    """
    #Here we'll do a quick check for any entries in the user passed
    #sampling_dict and add any missing entries with default values
    #a future extension should validate any entries

    #note that current first tier commands accepted in sampling_dict are:
    #{sampling_type, sampling_report_dict, stochastic_count_safety_factor, 
    #extra_seed_generator, sampling_generator}
    """

    if not isinstance(sampling_dict, dict):
      sampling_dict = {}
    
    check_sampling_dict_result = False
    
    def _populate_sampling_dict_default(sampling_dict, parameter, printstatus, check_sampling_dict_result, default = False, valid_entries=False, valid_type = False):
      if parameter in sampling_dict:
        if valid_entries is not False:
          if sampling_dict[parameter] not in valid_entries:
            check_sampling_dict_result = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "invalid entry passed to sampling_dict key ", parameter)
            self.__autoprint('warning', "acceptable values are one of", valid_entries)
            self.__autoprint('warning', '')
              
        if valid_type is not False:
          if not isinstance(sampling_dict[parameter], valid_type):
            check_sampling_dict_result = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "invalid entry type passed to sampling_dict key ", parameter)
            self.__autoprint('warning', "acceptable value type is", valid_type)
            self.__autoprint('warning', '')
        
      elif default != 'no_default_populated':
        sampling_dict.update({parameter : default})
        
      return sampling_dict, check_sampling_dict_result
            
    #sampling_type can be used to designate number of supplemental entropy seeds applie to each sampling
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'sampling_type', 
                              printstatus, 
                              check_sampling_dict_result,
                              default='default', 
                              valid_entries={'default', 'bulk_seeds', 'sampling_seed', 'transform_seed'},
                              valid_type=str)
    
    #seeding_type 'supplemental_seeds' means that entropy seeds are integrated into np.random.SeedSequence with entropy seeding from the operating system. 
    #Also accepts 'primary_seeds', in which user passed entropy seeds are the only source of seeding. 
    #Please note that 'primary_seeds' is used as the default for the bulk_seeds sampling_type 
    #and 'supplemental_seeds' is used as the default for other sampling_type options.
    #note the default scenario is updated following this call to one of those two configurations
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'seeding_type', 
                              printstatus, 
                              check_sampling_dict_result,
                              default='default', 
                              valid_entries={'supplemental_seeds', 'primary_seeds'},
                              valid_type=str)
    
    #for 'seeding_type' the default depends on whether we are using bulk_seeds samplingtype
    #so a temporary plug value now updated here
    if sampling_dict['seeding_type'] == 'default':
      if sampling_dict['sampling_type'] == 'bulk_seeds':
        sampling_dict['seeding_type'] = 'primary_seeds'
      else:
        sampling_dict['seeding_type'] = 'supplemental_seeds'
    
    #sampling_report_dict may be passed by user as returned in a seperate automunge(.) call as postprocess_dict['sampling_report_dict']
    #when not received an internal automunge(.) call takes place to populate one instead
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'sampling_report_dict', 
                              printstatus, 
                              check_sampling_dict_result,
                              default=False, 
                              valid_entries=False,
                              valid_type=(bool, dict))
    
    #stochastic_count_safety_factor refers to the safety factor applied to seeding budget for stochastic elements
    #like e.g. if we know the Bernoulli sampling ratio, we can derive the distribution sampling for bulk_seeds
    #applying this safety factor to account for variance in Bernoulli result
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'stochastic_count_safety_factor', 
                              printstatus, 
                              check_sampling_dict_result,
                              default=0.15, 
                              valid_entries=False,
                              valid_type=float)
    
    #extra_seed_generator selects which generator to use to populate extra seeds
    #which is performed when a sampling_type specification other than default is applied
    #and user did not pass sufficient external entropy_seeds to meet the budget
    #in some cases the extra_seed_generator may generate all of the entropy seeds at once for passing to a numpy generator in the transforms
    #which may benefit utilization rate of quantum hardware by eliminating latency of pandas operations from the QC utilization
    #the off case refers to not sampling additional seeds when user does not meet budget
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'extra_seed_generator', 
                              printstatus, 
                              check_sampling_dict_result,
                              default='custom', 
                              valid_entries={'custom', 'PCG64', 'MersenneTwister', 'off', 'sampling_generator'},
                              valid_type=str)
    
    #sampling_generator selects which generator will be applied within the transforms
    #which if the selection accepts seeding will include any seeds received as entropy_seeds
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'sampling_generator', 
                              printstatus, 
                              check_sampling_dict_result,
                              default='custom', 
                              valid_entries={'custom', 'PCG64', 'MersenneTwister'},
                              valid_type=str)
    
    #random_generator_accepts_seeds is derived internally when validating the user passed random_generator
    sampling_dict, check_sampling_dict_result = \
    _populate_sampling_dict_default(sampling_dict, 
                              'random_generator_accepts_seeds', 
                              printstatus, 
                              check_sampling_dict_result,
                              default=random_generator_accepts_seeds, 
                              valid_entries={True, False},
                              valid_type=bool)
    
    return check_sampling_dict_result, sampling_dict

  def __check_entropy_seeds(self, entropy_seeds, printstatus):
    """
    quick alignment to common form of np.array for entropy_seeds
    """

    entropy_seeds_result = False
    
    if entropy_seeds is False:
      entropy_seeds = np.array([])
      
    elif isinstance(entropy_seeds, int):
      entropy_seeds = np.array([entropy_seeds])
      
    elif isinstance(entropy_seeds, list):
      entropy_seeds = np.array(entropy_seeds)
      
    elif isinstance(entropy_seeds, type(np.array([]))):
      entropy_seeds = entropy_seeds.ravel()
    
    else:
      entropy_seeds_result = True
      
      # if printstatus is True:
      self.__autoprint('debug', "invalid entry received for entropy_seeds parameter")
      self.__autoprint('debug', "accepted types are bool False, int, list, np.array")
        
    entropy_seeds = entropy_seeds.astype(int)

    return entropy_seeds, entropy_seeds_result

  def __check_random_generator(self, random_generator, printstatus):
    """
    validates form of random_generator
    as well as determining if it accepts entropy seeds
    e.g. random_generator vs random_generator
    """

    #check random_generator
    random_generator_valresult = False
    random_generator_accepts_seeds = True
    
    if random_generator is not False:
      try:
        np.random.Generator(random_generator()).standard_normal()
        random_generator_accepts_seeds = True
      except TypeError:
        try:
          np.random.Generator(random_generator).standard_normal()
          random_generator_accepts_seeds = False
        except TypeError:
          random_generator_valresult = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "Error: invalid entry passed for random_generator parameter.")
          self.__autoprint('warning', "Acceptable values are False or np.random formatted generator")
          self.__autoprint('warning', '')
            
    return random_generator_valresult, random_generator_accepts_seeds
  
  def __check_assignparam(self, assignparam, process_dict, printstatus):
    """
    #Here we'll do a quick check to validate the passed assign param.
    
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    
    #we'll validate that any category entries are valid
    #we won't validate column entries since they may be dervied columns which we don't know yet
    """

    result = False
    
    for key in assignparam:
      
      if key not in {'global_assignparam', 'default_assignparam', '(category)'} \
      and key not in process_dict:
        
        result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error, assignparam category key ", key)
        self.__autoprint('warning', "was not found in process_dict")
        
      elif key == 'default_assignparam':
        
        for key2 in assignparam['default_assignparam']:
          
          if key2 != '(category)' and key2 not in process_dict:
            
            result = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "error, assignparam['default_assignparam'] category key ", key2)
            self.__autoprint('warning', "was not found in process_dict")
    
    return result
  
  def __check_columnheaders(self, columnheaders_list, printstatus):
    """
    #Performs a validation that all of the column headers are unique
    """

    result = False
    
    if len(columnheaders_list) > len(set(columnheaders_list)):
      
      result = True
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "Warning of potential error from duplicate column headers.")
      self.__autoprint('warning', "")
      
    return result

  def __check_processdict(self, processdict, printstatus):
    """
    #runs validations on user passed processdict
    #assumes any conversion from functionpointer already taken place
    
    #checks that NArowtype and MLinfilltype have valid entries
    #checks that labelctgy is present
    #checks that dualprocess postprocess and singleprocess have entries
    """

    check_processdict_result = False
    check_processdict_result2 = False

    #the reason these assignparam strings are reserved for use as category keys
    #is because if used tham as categories wouldn't be able to pass assignparam params to them
    #since assignparam parsing would interpret the categories as assignparam keywords
    for reserved_str in ['global_assignparam', 'default_assignparam', 'messy_data_headers']:
      if reserved_str in processdict:
        check_processdict_result2 = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: processdict has entry for ", reserved_str)
        self.__autoprint('warning', "which is a reserved category string for use in assignparam")
    
    for entry in processdict:
      
      if 'NArowtype' not in processdict[entry]:
        check_processdict_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: processdict missing 'NArowtype' entry for category: ", entry)
        self.__autoprint('warning', '')
      else:
        if processdict[entry]['NArowtype'] not in \
        {'numeric', 'integer', 'justNaN', 'binary', 'exclude', 'totalexclude', 'positivenumeric', 'nonnegativenumeric', \
        'nonzeronumeric', 'parsenumeric', 'datetime'}:
          check_processdict_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error: invalid 'NArowtype' processdict entry for category: ", entry)
          self.__autoprint('warning', '')
        
      if 'MLinfilltype' not in processdict[entry]:
        check_processdict_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: processdict missing 'MLinfilltype' entry for category: ", entry)
        self.__autoprint('warning', '')
      else:
        if processdict[entry]['MLinfilltype'] not in \
        {'numeric', 'singlct', 'integer', 'binary', 'multirt', 'concurrent_act', 'concurrent_ordl', 'concurrent_nmbr', '1010', \
        'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:
          check_processdict_result = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error: invalid 'MLinfilltype' processdict entry for category: ", entry)
          self.__autoprint('warning', '')
        
      #labelctgy is an optional entry to processdict, if not populated or accessed from functionpointer
      #it is given an arbitrary assignment in _check_processdict3 when entry is a root labels category
      # if 'labelctgy' not in processdict[entry]:
      #   check_processdict_result = True
      #   # if printstatus != 'silent':
      #   self.__autoprint('warning', "error: processdict missing 'labelctgy' entry for category: ", entry)
      #   self.__autoprint('warning', '')
      
      #we'll have convention that at least one entry for processing funtions required
      #even thought there is scenario where no corresponding function populated
      #such as when processdict is for a root category not used as a transformation category
      if ('singleprocess' not in processdict[entry]) and ('dualprocess' not in processdict[entry] or 'postprocess' not in processdict[entry]) \
      and 'custom_train' not in processdict[entry]:
        check_processdict_result = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: processdict entry missing processing function entrys for categery: ", entry)
        self.__autoprint('warning', "requires entries for (both 'dualprocess' and 'postprocess') or (entry for 'singleprocess') or 'custom_train'")
        self.__autoprint('warning', "(alternately a valid 'functionpointer' entry can be included)")
        self.__autoprint('warning', '')
      else:
        pass
        #for now won't validate the transformation function entries
      
    return check_processdict_result, check_processdict_result2

  def __check_processdict3(self, entry, processdict, postprocess_dict, transform_dict, mirror_dict, printstatus):
    """
    The processdict 'labelctgy' entry is only really used on small part of library
    Which is there to direct feature importance or levelizer to a specific returned set
    For cases where label set returned in multiple configurations

    Trying to get away from making labelctgy a required entry
    So convention applied here is that if labelctgy not specified
    An arbitrary target is accessed from the family trees
    prioritize to upstream primitive entries without offspring
    followed by same convention for downstream primitizes if upstring target isn't identified
    
    This function applied after accessing the labelscategory root category for labels in automunge
    Which is passed to this function as 'entry'
    """

    check_processdict3_valresult = False
    check_processdict3_validlabelctgy_valresult = False

    #intentionally limiting search to labelscategory in user passed processdict entries instead of full process_dict
    #note this function is applied after processdict and process_dict have already been consolidated
    #and functionpointers have been applied to populate missing entries
    #similarly transform_dict at this point has already been consolidated
    if entry in processdict:

      if 'labelctgy' not in postprocess_dict['process_dict'][entry]:

        check_processdict3_valresult = True

        #'labelctgy' only used when entry serves as a root category in transform_dict
        if entry in transform_dict:

          # if printstatus != 'silent':
          self.__autoprint('warning', "labelctgy processdict entry wasn't provided for ", entry)
          self.__autoprint('warning', "selecting arbitrary entry based on family tree")

          familytree = transform_dict[entry]

          if len(familytree['auntsuncles']) > 0:
            new_labelctgy = familytree['auntsuncles'][0]
            postprocess_dict['process_dict'][entry]['labelctgy'] = new_labelctgy
            mirror_dict['process_dict'][entry]['labelctgy'] = new_labelctgy

            # if printstatus is True:
            self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
            self.__autoprint('debug', '')

          elif len(familytree['cousins']) > 0:
            new_labelctgy = familytree['cousins'][0]
            postprocess_dict['process_dict'][entry]['labelctgy'] = new_labelctgy
            mirror_dict['process_dict'][entry]['labelctgy'] = new_labelctgy

            # if printstatus is True:
            self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
            self.__autoprint('debug', '')

          elif len(familytree['parents']) > 0:
            offspringparent = familytree['parents'][0]

            new_labelctgy = \
            self.__check_processdict3_support(transform_dict, offspringparent, printstatus)

            postprocess_dict['process_dict'][entry]['labelctgy'] = new_labelctgy
            mirror_dict['process_dict'][entry]['labelctgy'] = new_labelctgy

            # if printstatus is True:
            self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
            self.__autoprint('debug', '')

          elif len(familytree['siblings']) > 0:
            offspringparent = familytree['siblings'][0]

            new_labelctgy = \
            self.__check_processdict3_support(transform_dict, offspringparent, printstatus)

            postprocess_dict['process_dict'][entry]['labelctgy'] = new_labelctgy
            mirror_dict['process_dict'][entry]['labelctgy'] = new_labelctgy

            # if printstatus is True:
            self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
            self.__autoprint('debug', '')

      elif 'labelctgy' in postprocess_dict['process_dict'][entry]:

        if postprocess_dict['process_dict'][entry]['labelctgy'] not in postprocess_dict['process_dict']:

          check_processdict3_validlabelctgy_valresult = True

          # if printstatus != 'silent':
          printsupport = postprocess_dict['process_dict'][entry]['labelctgy']
          self.__autoprint('warning', "labelctgy processdict entry wasn't valid for entry ", entry)
          self.__autoprint('warning', "Was entered as ", printsupport)
          self.__autoprint('warning', "labelctgy needs to be a valid transformation category with entries in process_dict and transform_dict")

    return postprocess_dict, mirror_dict, check_processdict3_valresult, check_processdict3_validlabelctgy_valresult

  def __check_processdict3_support(self, transform_dict, offspringparent, printstatus):
    """
    Support function for _check_processdict3
    Used to inspect downstream primitive entries 
    When a labelctgy wasn't identified in upstream primitives
    """

    familytree = transform_dict[offspringparent]

    if len(familytree['coworkers']) > 0:
      new_labelctgy = familytree['coworkers'][0]

      # if printstatus is True:
      self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
      self.__autoprint('debug', '')

    elif len(familytree['friends']) > 0:
      new_labelctgy = familytree['friends'][0]

      # if printstatus is True:
      self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
      self.__autoprint('debug', '')

    elif len(familytree['children']) > 0:
      offspringparent = familytree['children'][0]

      new_labelctgy = \
      self.__check_processdict3_support(transform_dict, offspringparent, printstatus)

      # if printstatus is True:
      self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
      self.__autoprint('debug', '')

    elif len(familytree['niecesnephews']) > 0:
      offspringparent = familytree['niecesnephews'][0]

      new_labelctgy = \
      self.__check_processdict3_support(transform_dict, offspringparent, printstatus)

      # if printstatus is True:
      self.__autoprint('debug', "labelctgy selected as ", new_labelctgy)
      self.__autoprint('debug', '')

    return new_labelctgy

  def __check_processdict4(self, processdict, printstatus):
    """
    #Validates that any processing function entries in user passed processdict
    #are either a callable function or passed as None
    #note this takes place after functionpointer entries are accessed
    """

    check_processdict4_valresult = False
    check_processdict4_valresult2 = False

    checked_slots = ['dualprocess', 'singleprocess', 'postprocess', 'inverseprocess', \
                     'custom_train', 'custom_test', 'custom_inversion']

    for category in processdict:

      for checked_slot in checked_slots:

        if checked_slot in processdict[category]:

          if not callable(processdict[category][checked_slot]) \
          and processdict[category][checked_slot] != None:

            check_processdict4_valresult = True

            # if printstatus != 'silent':
            self.__autoprint('warning', "warning of potential error")
            self.__autoprint('warning', "for processdict entry associated with category ", category)
            self.__autoprint('warning', "a processing function was entered in slot for ", checked_slot)
            self.__autoprint('warning', "which was not callable or passed as None")

      #also validate that if a custom_train was not populated and a callable dualprocess was, a callable postprocess is available
      if not ('custom_train' in processdict[category] and callable(processdict[category]['custom_train'])):
        if 'dualprocess' in processdict[category] and callable(processdict[category]['dualprocess']):
          if 'postprocess' not in processdict[category] or 'postprocess' in processdict[category] and not callable(processdict[category]['postprocess']):

            check_processdict4_valresult2 = True

            print("warning of channel for postmunge error")
            print("for processdict entry associated with category ", category)
            print("a prioritized callable processing function was entered in slot for 'dualprocess'")
            #prioritized means that a custom_train funciton wasn't also entered which would otherwise take precedence
            print("without a corresponding callable processing function in slot for 'postprocess")
            print("this scenario will result in postmunge error")
            print("Note that if a common function is desired for both train and test data user can instead pass a function in the 'custom_train' or 'singleprocess' convention")

    return check_processdict4_valresult, check_processdict4_valresult2

  #__FunctionBlock: entropy_seeds and random_generator support

  def __initialize_sampling_report_dict(self, df_train, assigncat, 
                                        assignparam, transformdict, processdict, 
                                        sampling_dict, printstatus, numbercategoryheuristic,
                                        powertransform, evalcat, logger):
    """
    when a sampling_dict['sampling_type'] specification was received in automunge(.)
    there won't be a sampling_resource_dict in postprocess_dict yet
    so user can either run autounge(.) externally to populate one and pass to sampling_dict['sampling_report_dict']
    or when such entry wasn't received, if sampling_dict['sampling_type'] != default
    we run an internal simplified automunge(.) call to populate one
    and return in sampling_dict['sampling_report_dict']
    this may also be applied in postmunge(.) in conjunction with privacy_encoding when don't have access to sampling_report_dict externally
    in order to speed this operation up, if number of rows >5000, we'll do so with a 5000 row sample

    if this function is accessed inside the internal automunge(.) call we'll break the loop 
    by passing the sampling_dict['sampling_type'] as 'default'
    which will still populate the returned sampling_report_dict
    without needing to inspect an initial sampling_report_dict

    the reason we are using an internal automunge call to populate sampling_report_dict
    instead of the methods applied to generate counts in __prepare_seeds
    is that the __prepare_seeds method requires inspection of params
    and in order to access params need to navigate family trees
    """

    #in order to speed this operation up, if number of rows >5000, we'll do so with a 5000 row sample
    train_rowcount_orig = df_train.shape[0]
    if train_rowcount_orig > 5000:
      df_train_temp = df_train[:5000]
    else:
      df_train_temp = df_train.copy()
      
    #we'll use df_train_temp as both train and test data for this purpose
    #in case user didn't pass a df_test
    
    if isinstance(sampling_dict, dict) \
    and 'sampling_type' in sampling_dict \
    and sampling_dict['sampling_type'] not in {'default'}:
      
      if 'sampling_report_dict' not in sampling_dict \
      or 'sampling_report_dict' in sampling_dict \
      and not isinstance(sampling_dict['sampling_report_dict'], dict):
        
        # if printstatus is True:
        self.__autoprint('debug', "Received sampling_dict['sampling_type'] specification other than default")
        self.__autoprint('debug', "populating a sampling_resource_dict with an initial internal automunge(.) call")
        self.__autoprint('debug', '')

        #following automunge call will return validation results in original bucket
        #which we'll store in samplingprep_validations
        orig_validations_dict = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}

        _1, _2, _3, _4, _5, _6, _7, _8, _9, temp_ppd = \
        self.automunge(df_train_temp,
                      df_test = df_train_temp,
                      assigncat = assigncat,
                      assignparam = assignparam,
                      sampling_dict = {'sampling_type' : 'default'},
                      entropy_seeds = False,
                      random_generator = False,
                      MLinfill = False,
                      NArw_marker = False,
                      printstatus = False,
                      eval_ratio = 1.,
                      numbercategoryheuristic = numbercategoryheuristic,
                      transformdict = transformdict,
                      processdict = processdict,
                      evalcat=evalcat,
                      logger=logger,
                      )

        #logger_dict validation results stored in logger_dict['featureselection_validations']
        logger_dict['samplingprep_validations'] = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}
        logger_dict['validations'] = self.__autocopy(orig_validations_dict)
        
        sampling_dict['sampling_report_dict'] = temp_ppd['sampling_report_dict']
    
    return sampling_dict

  def __populate_sampling_report_dict(self, postprocess_dict, sampling_dict):
    """
    #sampling_report_dict
    #is intended as a resource 
    #for determining how many entropy seeds are needed 
    #to support various sampling_dict['sampling_type'] specification options
    #so will derive a specific number of seeds needed for each scenario
    
    #noting that if fewer seeds are received they will still be extracted to reach the desired threshold
    #(e.g. if you need 5 seeds for a scenario but only passed 3, 
    #we'll use those 3 seeds in a sampling of 5)

    #and if more seeds are received 
    #the extras will be unused
    
    #we'll have convention of adding one to the final number
    #which extra seed will be used as a seed to shuffle the other seeds
    
    #the derivation will be conducted by collecting the sampling_resource_dict reports 
    #recorded in noise transforms' normalization_dict
    #and aggregating counts from those results into a single master_sampling_resource_dict
    
    #note that in some cases the number of samples may be a stochastic property
    #such as when gaussian sample counts are based on result of a binomial sampling
    #so we'll follow convention that for stochastic sampling counts
    #we'll report a value with and without a 10% safety factor
    
    #note that noise transforms support passing noise parameters 
    #as distributions instead of static values
    #for now am assuming static noise parameters
    #as that option was primarily implemented to support future experiments
    
    #note that this applies a 15% contingency factor on bulk samples with a stochastic count
    #e.g. for when distribution sampling count is ap-plied as a funciton of binomial activations

    the +3 in the final counts are for an extra seed to shuffle the other seeds and sample a global randomseed

    note that the sampling_resource_dict saved in normalization_dict's
    will have a different format than the sampling_report_dict

    note that this report will be generated with each automunge call
    while the report included in sampling_dict['sampling_report_dict']
    will either be based on user specificaiton or a single internal automunge call
    due to some of the counts being derived as a function of stochastic sampling
    it is possible that the returned postprocess_dict['sampling_report_dict'] 
    may slightly differ to the sampling_dict['sampling_report_dict']
    this stochastic sampling basis is the reason for applying the sampling_dict['stochastic_count_safety_factor']
    """

    #note that sampling_dict will already have been initialized with any missing defaults in __check_sampling_dict
    stochastic_contingency = sampling_dict['stochastic_count_safety_factor']
    
    #master_sampling_resource_dict will be used to aggregate counts from sampling_resource_dict's populated in normalization_dict's
    master_sampling_resource_dict = {'binomial_train_call_count' : 0,
                                     'binomial_train_sample_count' : 0,
                                     'binomial_test_call_count' : 0,
                                     'binomial_test_sample_count' : 0,
                                     'distribution_train_call_count' : 0,
                                     'distribution_train_sample_count' : 0,
                                     'distribution_test_call_count' : 0,
                                     'distribution_test_sample_count' : 0,
                                     'choice_train_call_count' : 0,
                                     'choice_train_sample_count' : 0,
                                     'choice_test_call_count' : 0,
                                     'choice_test_sample_count' : 0,
                                     'parameterlist_train_call_count' : 0,
                                     'parameterlist_train_sample_count' : 0,
                                     'parameterlist_test_call_count' : 0,
                                     'parameterlist_test_sample_count' : 0,
                                     'statsdistribution_train_call_count' : 0,
                                     'statsdistribution_train_sample_count' : 0,
                                     'statsdistribution_test_call_count' : 0,
                                     'statsdistribution_test_sample_count' : 0,
                                    }
    
    noise_transform_count = 0
    
    #first we'll aggregate all of the sampling_resource_dict reports 
    #note this will include label sets when applicable
    for derived_column in postprocess_dict['column_dict']:
      #check if comes from a noise transform
      process_dict_entry = \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][derived_column]['category']]
      
      if 'noise_transform' in process_dict_entry \
      and process_dict_entry['noise_transform'] in {'numeric', 'categoric', 'binary'}:
        
        if derived_column in postprocess_dict['column_dict'][derived_column]['normalization_dict']:
          sampling_resource_dict = \
          postprocess_dict['column_dict'][derived_column]['normalization_dict'][derived_column]['sampling_resource_dict']
          
          noise_transform_count += 1

          for master_key in master_sampling_resource_dict:
            
            master_sampling_resource_dict[master_key] += sampling_resource_dict[master_key]
          
    #now derive total counts of generator calls and samples based on the different sampling operations apploied in transforms
    total_generator_calls_train = \
    master_sampling_resource_dict['binomial_train_call_count'] \
    + master_sampling_resource_dict['distribution_train_call_count'] \
    + master_sampling_resource_dict['choice_train_call_count'] \
    + master_sampling_resource_dict['parameterlist_train_call_count'] \
    + master_sampling_resource_dict['statsdistribution_train_call_count']
    
    total_generator_calls_test = \
    master_sampling_resource_dict['binomial_test_call_count'] \
    + master_sampling_resource_dict['distribution_test_call_count'] \
    + master_sampling_resource_dict['choice_test_call_count'] \
    + master_sampling_resource_dict['parameterlist_test_call_count'] \
    + master_sampling_resource_dict['statsdistribution_test_call_count']
    
    total_samples_train = \
    master_sampling_resource_dict['binomial_train_sample_count'] \
    + master_sampling_resource_dict['distribution_train_sample_count'] \
    + master_sampling_resource_dict['choice_train_sample_count'] \
    + master_sampling_resource_dict['parameterlist_train_sample_count'] \
    + master_sampling_resource_dict['statsdistribution_train_sample_count']
    
    total_samples_test = \
    master_sampling_resource_dict['binomial_test_sample_count'] \
    + master_sampling_resource_dict['distribution_test_sample_count'] \
    + master_sampling_resource_dict['choice_test_sample_count'] \
    + master_sampling_resource_dict['parameterlist_test_sample_count'] \
    + master_sampling_resource_dict['statsdistribution_test_sample_count']
    
    #currently only distribution sampling has a stochastic count
    #but choice sampling update is pending so will include choice in the safety factor
    stochastic_count_train = \
    master_sampling_resource_dict['distribution_train_sample_count'] \
    + master_sampling_resource_dict['choice_train_sample_count']
    
    stochastic_count_test = \
    master_sampling_resource_dict['distribution_test_sample_count'] \
    + master_sampling_resource_dict['choice_test_sample_count']
    
    #_____
    
    #here are the results we'll populate
    
    rowcount_basis_train = postprocess_dict['train_rowcount']
    rowcount_basis_test = postprocess_dict['test_rowcount']
    
    #bulk_seeds scenario is a seed for every sampling for every entry
    #we're adding the conginency (defauilts to 0.15) for stochastic sample counts here
    #the plus 3 is one for __prepare_seeds shuffle seed and 2 for setting the default randomseed
    #(the __prepare_seeds spawn seed is not budgeted since not used when entropy_seeds provided complete)
    #the total_generator_calls_train as used here is associated with a shuffle for each genrator's seeds performed in __random_parameters_params_append
    bulk_seeds_total_train = \
    total_samples_train + 3 + int(stochastic_contingency * stochastic_count_train) + total_generator_calls_train

    bulk_seeds_total_test = \
    total_samples_test + 3 + int(stochastic_contingency * stochastic_count_test) + total_generator_calls_test
    
    #the stochastic counts serving as basis for the contingency are returned for reference
    bulk_seeds_stochastic_count_train = stochastic_count_train
    bulk_seeds_stochastic_count_test = stochastic_count_test
    
    #sampling_seed scenario is a single seed for every generator call (*2 due to a shuffling in __random_parameters_params_append)
    #the plus 3 is one for __prepare_seeds shuffle seed and 2 for setting the default randomseed
    sampling_seed_total_train = (2 * total_generator_calls_train) + 3
    sampling_seed_total_test = (2 * total_generator_calls_test) + 3
    
    #transform_seed scenario is a single seed for every noise transform, (*2 due to a shuffling in __random_parameters_params_append)
    #the plus 3 is one for __prepare_seeds shuffle seed and 2 for setting the default randomseed
    #transform_seed_total by convention is same independent of train/test/both configuration
    transform_seed_total = (2 * noise_transform_count) + 3

    #please note that the "+ 3" seeds used for a shuffle and setting the default randomseed 
    #are redundantly budgetted in train and test buckets
    #this is due to potential alternate workflows of only applying train or test injections
    #such as in automunge with df_test = False or postmunge traindata scenarios

    #_____
    
    #now populate sampling_report_dict and save to postprocess_dict
    sampling_report_dict = \
    {
      'rowcount_basis_train' : rowcount_basis_train,
      'rowcount_basis_test' : rowcount_basis_test,
      'bulk_seeds_total_train' : bulk_seeds_total_train,
      'bulk_seeds_total_test' : bulk_seeds_total_test,
      'bulk_seeds_stochastic_count_train' : bulk_seeds_stochastic_count_train,
      'bulk_seeds_stochastic_count_test' : bulk_seeds_stochastic_count_test,
      'sampling_seed_total_train' : sampling_seed_total_train,
      'sampling_seed_total_test' : sampling_seed_total_test,
      'transform_seed_total' : transform_seed_total,
    }
    
    #if this automunge(.) call didn't include a df_test, the df_test rowcount basis will be 1
    #in such case, we'll populate the test values
    #with the initial test entries populated in __initialize_sampling_report_dict
    #which were derived using train data excerpts passed to df_test
    #this segment won't apply in the internal automunge call case since we are passing df_train to df_test
    if rowcount_basis_test == 1:
      for entry in ['rowcount_basis_test', 'bulk_seeds_total_test', 
                    'bulk_seeds_stochastic_count_test', 'sampling_seed_total_test']:
        if isinstance(sampling_dict['sampling_report_dict'], dict) and entry in sampling_dict['sampling_report_dict']:
          sampling_report_dict[entry] = sampling_dict['sampling_report_dict'][entry]
    
    #now save the final form to postprocess_dict, this will be the form returned from automunge(.)
    postprocess_dict.update({'sampling_report_dict' : sampling_report_dict})
    
    return postprocess_dict

  def __prepare_seeds(self, postprocess_dict, sampling_dict, random_generator, entropy_seeds, rowcount_train, rowcount_test, traintest, test_plug_marker):
    """
    if sampling_type != default
    we'll want to determine how many seeds are needed prior to entering processfamily
    and if additional seeds are needed to reach appropriate number
    they are extracted here
    this will also be where we shuffle the received seeds with one of the seeds

    entropy_seeds will be received as a flattened np.array of length >= 0 from preparations in __check_entropy_seeds
    
    the resulting seeds will then be stored in postprocess_dict['entropy_seeds']
    and with each access other than the default sampling_type scenario, any used seeds will be struck from the list
    
    this will be performed just after process_dict iniitalization in automunge
    and somewhere near start of postmunge just after saving seeds in ppd
    
    this function also serves to populate ppd with entropy_seeds, random_generator, sampling_dict
    
    this function is applied seperately targeting train or test data based on the traintest parameter
    
    test_plug_marker is True when df_test was not passed to automunge(.)
    
    for reference, here is structure of sampling_dict['sampling_report_dict']
    sampling_report_dict = \
    {
      'rowcount_basis_train' : rowcount_basis_train,
      'rowcount_basis_test' : rowcount_basis_test,
      'bulk_seeds_total_train' : bulk_seeds_total_train,
      'bulk_seeds_total_test' : bulk_seeds_total_test,
      'bulk_seeds_stochastic_count_train' : bulk_seeds_stochastic_count_train,
      'bulk_seeds_stochastic_count_test' : bulk_seeds_stochastic_count_test,
      'sampling_seed_total_train' : sampling_seed_total_train,
      'sampling_seed_total_test' : sampling_seed_total_test,
      'transform_seed_total' : transform_seed_total,
    }
    
    note that the bulk counts will include a 15% contingency factor derived from the stochastic counts
    which is configurable by sampling_dict['sampling_seed_custom_generator']
    
    the sampling_type scenarios are
    - default: every sample is from either default or when specified the custom generator, 
    any received entropy seeds are shuffled and passed to each call (same as prior config)
    - bulk_seeds: every sampling for every entry receives a unique supplemental seed
    - sampling_seed: every sampling operation receives one supplemental seed
    - transform_seed: every noise transform receives one supplemental seed

    note that the generator used to generate additional seeds will be based on sampling_dict['extra_seed_generator']
    and the generator used to apply those seeds for sampling in application based on sampling_dict['sampling_generator']
    """

    #note that sampling_dict will already have been initialized with any missing defaults in __check_sampling_dict
    sampling_type = sampling_dict['sampling_type']
    sampling_generator = sampling_dict['sampling_generator']    
    extra_seed_generator = sampling_dict['extra_seed_generator']
    random_generator_accepts_seeds = sampling_dict['random_generator_accepts_seeds']

    if sampling_type != 'default' \
    and sampling_generator == 'custom' \
    and random_generator_accepts_seeds is True:
      
      #sampling_report_dict will be in sampling_dict if this function called in automunge
      #or will be in postprocess_dict if this function called in postmunge
      #note that postmunge also accepts user specified sampling_dict['sampling_report_dict'] which takes precedence over postprocess_dict
      sampling_report_dict = False
      if sampling_dict['sampling_report_dict'] is not False:
        sampling_report_dict = self.__autocopy(sampling_dict['sampling_report_dict'])
      elif 'sampling_report_dict' in postprocess_dict:
        #this is the postmunge case
        sampling_report_dict = self.__autocopy(postprocess_dict['sampling_report_dict'])
        
      #note that different trainnoise / testnoise scenarios will be covered by the counts in sampling_report_dict

      if sampling_type in {'bulk_seeds'}:
        
        if traintest == 'train':
          seed_requirement = \
          sampling_report_dict['bulk_seeds_total_train']
          
          #adjust bulk scenario for rowcount basis
          seed_requirement = \
          seed_requirement * rowcount_train / sampling_report_dict['rowcount_basis_train']
        
        elif traintest == 'test':
          seed_requirement = \
          sampling_report_dict['bulk_seeds_total_test']

          #adjust bulk scenario for rowcount basis
          seed_requirement = \
          seed_requirement * rowcount_test / sampling_report_dict['rowcount_basis_test']
          
        elif traintest == 'traintest':
          
          #adjust bulk scenario for rowcount basis
          seed_requirement = \
          sampling_report_dict['bulk_seeds_total_train'] * rowcount_train / sampling_report_dict['rowcount_basis_train']
          
          if test_plug_marker is False:
            seed_requirement += sampling_report_dict['bulk_seeds_total_test'] * rowcount_test / sampling_report_dict['rowcount_basis_test']
          
      elif sampling_type in {'sampling_seed'}:

        if traintest == 'train':
          seed_requirement = \
          sampling_report_dict['sampling_seed_total_train']
          
          if rowcount_train == 0:
            seed_requirement = 0
        
        elif traintest == 'test':
          seed_requirement = \
          sampling_report_dict['sampling_seed_total_test']
          
          if rowcount_test == 0:
            seed_requirement = 0
          
        elif traintest == 'traintest':
          seed_requirement = \
          sampling_report_dict['sampling_seed_total_train']
          
          if test_plug_marker is False:
            seed_requirement += sampling_report_dict['sampling_seed_total_test']
            
          if rowcount_train + rowcount_test == 0:
            seed_requirement = 0
          
      elif sampling_type in {'transform_seed'}:
        
        if traintest in {'train', 'test'}:
          seed_requirement = \
          sampling_report_dict['transform_seed_total']
        elif traintest == 'traintest':
          seed_requirement = sampling_report_dict['transform_seed_total']
          if test_plug_marker is False:
            seed_requirement *= 2
        
        #this is case in no noise postmunge traindata scenario
        if rowcount_train + rowcount_test == 0:
          seed_requirement = 0
        
      #now that we have our seed requirement, let's compare to the available entropy seeds
      provided_seed_count = len(entropy_seeds)
        
      #if fewer seeds were provided, additional seeds are extracted except for the extra_seed_generator off case
      if provided_seed_count < seed_requirement and extra_seed_generator != 'off':
        
        spawn_seed = []
        if provided_seed_count > 0:
          spawn_seed = [int(entropy_seeds[0])]
          entropy_seeds = np.delete(entropy_seeds, 0)
          provided_seed_count -= 1

        #this determines whether we extract additional seeds with default or custom generator
        if extra_seed_generator == 'PCG64' \
        or extra_seed_generator == 'sampling_generator' \
        and sampling_generator == 'PCG64':
          randomgenerator = np.random.PCG64
          extra_seed_generator_accepts_seeds = True
        elif extra_seed_generator == 'MersenneTwister' \
        or extra_seed_generator == 'sampling_generator' \
        and sampling_generator == 'MersenneTwister':
          randomgenerator = np.random.MT19937
          extra_seed_generator_accepts_seeds = True
        elif extra_seed_generator == 'custom' \
        or extra_seed_generator == 'sampling_generator' \
        and sampling_generator == 'custom':
          #else only use default if random_generator not specified
          if random_generator is False:
            randomgenerator = np.random.PCG64
            extra_seed_generator_accepts_seeds = True
          else:
            randomgenerator = random_generator
            extra_seed_generator_accepts_seeds = random_generator_accepts_seeds

        if extra_seed_generator_accepts_seeds is True:
          nprandom = np.random.Generator(randomgenerator(np.random.SeedSequence(spawn_key=spawn_seed)))
        else:
          nprandom = np.random.Generator(randomgenerator)
        
        extra_seeds_needed = int(seed_requirement - provided_seed_count)
        
        #2**31-1 max selected to align with max capacity for np.int32
        max_capacity_integer = int(2**31 - 1)
        min_capacity_integer = 0

        extra_seeds = nprandom.integers(min_capacity_integer, high=max_capacity_integer, size=extra_seeds_needed).astype(int)

        extra_seeds = extra_seeds.astype(int)
        
        entropy_seeds = np.concatenate((entropy_seeds, extra_seeds), axis=0).astype(int)
        
      #now one more shuffle using the shuffle_seed
      if len(entropy_seeds) > 2:
        
        #let's access the first entry as our shuffle seed
        shuffle_seed = [int(entropy_seeds[0])]
        entropy_seeds = np.delete(entropy_seeds, 0)
        
        if sampling_generator =='PCG64':
          randomgenerator = np.random.PCG64
          sampling_generator_accepts_seeds = True
        elif sampling_generator =='MersenneTwister':
          randomgenerator = np.random.MT19937
          sampling_generator_accepts_seeds = True
        elif sampling_generator == 'custom':
          #else only use default if random_generator not specified
          if random_generator is False:
            randomgenerator = np.random.PCG64
            sampling_generator_accepts_seeds = True
          else:
            randomgenerator = random_generator
            sampling_generator_accepts_seeds = random_generator_accepts_seeds

        if sampling_generator_accepts_seeds is True:
          nprandom = np.random.Generator(randomgenerator(np.random.SeedSequence(spawn_key=shuffle_seed)))
        else:
          nprandom = np.random.Generator(randomgenerator)

        nprandom.shuffle(entropy_seeds)
        
    #now let's store the results in ppd
    postprocess_dict['entropy_seeds'] = entropy_seeds.astype(int)
    postprocess_dict['sampling_dict'] = sampling_dict
    postprocess_dict['random_generator'] = random_generator
    
    return postprocess_dict

  def __random_parameters_params_append(self, params, postprocess_dict, column, category, traintest, 
                                        rowcount_trian, rowcount_test, printstatus, df_train, df_test):
    """
    this function takes place in processcousin/postprocesscousin/processparent/postprocessparent
    immediately after __grab_params
    
    if automunge(.) recieved specification in any of parameters 
    entropy_seeds, random_generator, sampling_dict
    those specifications are accessed from postprocess_dict
    and added to params
    noting that since we are adding to params instead of assignparm
    we won't need a seperate function to later strike
    will just need to reset the specifications in postprocess_dict prior to return
    
    as recieved, the postprocess_dict contains entries at this point 
    for entropy_seeds, random_generator, sampling_dict
    
    note that as entropy seeds are expended, they will be struck from set
    
    the entropy seeds will have already been extracted to meat requirements in __prepare_seeds
    
    in automunge traintest will be received as 'traintest', in postmunge as 'test'
    
    will burn an entropy_seed in this operation for shuffling seeds prior to populating

    Please note that the string configurations of the keys in the sampling_resource_dict are used in methods
    and follow convention samplingid_traintest_suffix, 
    where traintest is one of {'train', 'test'}
    and suffix may be omitted or include one of {'_seeds', '_call_count', '_sample_count'}
    and examples of samplingid include {'binomial', 'distribution', etc.}
    samplingid_traintest is used to designate whether to apply custom or default generator
    samplingid_traintest_seeds is used to pass specific seeds to trasnforms
    samplingid_traintest_call_count is used to count number of generator calls associated with the samplingid
    samplingid_traintest_sample_count is used to count number of sampled entries associated with the samplingid

    the same sampling_resource_dict object is used to pass seeds to transforms
    as well as to log counts from the transforms and returned for use to identify how many seeds are needed
    basically the meta counts are initially populated by running automunge on a subset of data without seeding in __initialize_sampling_report_dict
    the results from sampling_resource_dict's are aggregated in __populate_sampling_report_dict and returned in postprocess_dict['sampling_report_dict']
    or alternately by user specification in sampling_dict['sampling_report_dict'] from a prior automunge call
    and then the counts specific to a transform are derived in this function based on assignparam specifications
    which are then used in this function to populate seeds to pass to the transform from the seed bank initialized in __prepare_seeds
    where __prepare_seeds is conducted based on meta counts to prepare the full seed bank when entropy_seeds didn't receive sufficient number of seeds for meta counts

    df_train and df_test are passed to this funciton for use to determine generator call counts
    in cases of protected_feature specificaiton
    which increases generator calls
    when accessed in postmunge df_train should be received as False
    """

    #access entropy seed parameters from postprocess_dict
    #note that these entries will be specific to an automunge or postmunge call
    entropy_seeds = postprocess_dict['entropy_seeds']
    random_generator = postprocess_dict['random_generator']
    sampling_dict = postprocess_dict['sampling_dict']
    #note that random_generator_accepts_seeds defaults to True when random_generator passed as False
    random_generator_accepts_seeds = sampling_dict['random_generator_accepts_seeds']
    
    #initalize entropy seed parameters to single form between alternate scenarios
    if len(entropy_seeds) == 0:
      # entropy_seeds = np.array([random.randint(0,2147483647)])
      entropy_seeds = np.array([])

    if random_generator == False:
      random_generator = np.random.PCG64
    
    #note that sampling_dict will already have been initialized with any missing defaults in __check_sampling_dict
      
    #the random_generator is saved in params, where the False case will have been converted to PCG64 
    #at conclusion we will also store a params entry for sampling_resource_dict
    #which will include transformation / taget column specific seedings based on trasnformation type and parameters
    #we may later update this entry in the sampling_type == 'default' scenario
    params.update({'random_generator' : random_generator})

    sampling_generator = sampling_dict['sampling_generator']
      
    #a shuffle operation will be performed preceding each access of seeds
    def _shuffle_seeds(entropy_seeds, random_generator, sampling_type, sampling_generator, random_generator_accepts_seeds):
      if len(entropy_seeds) > 2:
        shuffle_seed = [int(entropy_seeds[0])]
        #we dont' delete seeds in the default sampling_type scenario
        if sampling_type != 'default':
          entropy_seeds = np.delete(entropy_seeds, 0)

        if sampling_generator == 'PCG64':
          randomgenerator = np.random.PCG64
          shuffling_generator_accepts_seeds = True
        elif sampling_generator == 'MersenneTwister':
          randomgenerator = np.random.MT19937
          shuffling_generator_accepts_seeds = True
        elif sampling_generator == 'custom':
          #else only use default if random_generator not specified
          #if random_generator was received as False will have already been converted to PCG64
          randomgenerator = random_generator
          shuffling_generator_accepts_seeds = random_generator_accepts_seeds

        if shuffling_generator_accepts_seeds is True:
          nprandom = np.random.Generator(randomgenerator(np.random.SeedSequence(spawn_key=shuffle_seed)))
        else:
          nprandom = np.random.Generator(randomgenerator)

        nprandom.shuffle(entropy_seeds)
      return entropy_seeds

    #this is form of sampling_resource_dict passed to transforms
    #in the transforms the counts will be evaluated for use to populate the final sampling_resource_dict logged in postprocess_dict
    populated_sampling_resource_dict = {'binomial_train' : 'custom',
                                        'binomial_train_seeds' : [],
                                        'binomial_train_call_count' : 0,
                                        'binomial_train_sample_count' : 0,
                                        'binomial_test' : 'custom',
                                        'binomial_test_seeds' : [],
                                        'binomial_test_call_count' : 0,
                                        'binomial_test_sample_count' : 0,
                                        'distribution_train' : 'custom',
                                        'distribution_train_seeds' : [],
                                        'distribution_train_call_count' : 0,
                                        'distribution_train_sample_count' : 0,
                                        'distribution_test' : 'custom',
                                        'distribution_test_seeds' : [],
                                        'distribution_test_call_count' : 0,
                                        'distribution_test_sample_count' : 0,
                                        'choice_train' : 'custom',
                                        'choice_train_seeds' : [],
                                        'choice_train_call_count' : 0,
                                        'choice_train_sample_count' : 0,
                                        'choice_test' : 'custom',
                                        'choice_test_seeds' : [],
                                        'choice_test_call_count' : 0,
                                        'choice_test_sample_count' : 0,
                                        'parameterlist_train' : 'custom',
                                        'parameterlist_train_seeds' : [],
                                        'parameterlist_train_call_count' : 0,
                                        'parameterlist_train_sample_count' : 0,
                                        'parameterlist_test' : 'custom',
                                        'parameterlist_test_seeds' : [],
                                        'parameterlist_test_call_count' : 0,
                                        'parameterlist_test_sample_count' : 0,
                                        'statsdistribution_train' : 'custom',
                                        'statsdistribution_train_seeds' : [],
                                        'statsdistribution_train_call_count' : 0,
                                        'statsdistribution_train_sample_count' : 0,
                                        'statsdistribution_test' : 'custom',
                                        'statsdistribution_test_seeds' : [],
                                        'statsdistribution_test_call_count' : 0,
                                        'statsdistribution_test_sample_count' : 0,
                                        'random_generator_accepts_seeds' : random_generator_accepts_seeds,
                                       }

    #we'll use sampling_resource_dict to log the counts used to generate seeds
    #and populated_sampling_resource_dict will be passed to the transform with seeds and zero counts
    sampling_resource_dict = self.__autocopy(populated_sampling_resource_dict)
    
    #stochastic_count_safety_factor expected as float between 0-1
    stochastic_count_safety_factor = sampling_dict['stochastic_count_safety_factor']
      
    #sampling_type expected as one of 
    #{'default', 'bulk_seeds', 'sampling_seed', 'transform_seed'}
    sampling_type = sampling_dict['sampling_type']
    
    #note that in the custom generator scenario if custom generator not specified will also defer to PCG64
    if sampling_type in {'bulk_seeds', 'sampling_seed', 'transform_seed'}:
      for generator_specification in ['binomial_train', 'binomial_test', 'distribution_train',
                                      'distribution_test', 'choice_train', 'choice_test',
                                      'parameterlist_train', 'parameterlist_test',
                                      'statsdistribution_train', 'statsdistribution_test']:
        populated_sampling_resource_dict.update({generator_specification : 'default'})
    #else defer to the previously populated value 'custom'

    #seeding_type refers to:
    #'supplemental_seeds' means that entropy seeds are integrated into np.random.SeedSequence with entropy seeding from the OS. 
    #'primary_seeds', in which user passed entropy seeds are the only source of seeding.
    #note that when not specified bulk_seeds sampling_type defaults to primary_seeds,
    #and other sampling_types default to supplemental_seeds
    seeding_type = sampling_dict['seeding_type']
    populated_sampling_resource_dict.update({'seeding_type' : seeding_type})
    
    #the category's process_dict entry is inspected for a noise_transform classification
    #this results in only transforms with process_dict['noise_transform'] specification being served seeds
    #so if user wants to turn off seeding for a particular transform they can do so by update to process_dict entry
    #we'll also exclude for postmunge traindata no_noise specification
    if 'noise_transform' in postprocess_dict['process_dict'][category] \
    and postprocess_dict['process_dict'][category]['noise_transform'] is not False \
    and ( 'traindata' not in postprocess_dict or 'traindata' in postprocess_dict \
    and postprocess_dict['traindata'] not in {'train_no_noise', 'test_no_noise'}):
      
      #noise_transform expected as one of {'numeric', 'categoric', 'binary', False}
      noise_transform = postprocess_dict['process_dict'][category]['noise_transform']
      
      #_____
      
      #now let's calculate how many entropy seeds are expected for use in the transform
      
      #convert traintest to final form based on any entries for traindata and test_plug_marker
      #the goal here is to identify if we are populating seeds for train or test or both
      #which is needed since each may have disfferent distribution samples requiring different bulk seeds
      
      #traintest will be received as 'traintest' in automunge
      if traintest == 'traintest':
        #if user didn't pass a df_test set then we don't need entropy seeds for test data
        if postprocess_dict['test_plug_marker'] is True:
          traintest == 'train'
      #traintest will be received as 'test' in postmunge so we know postprocess_dict will have 'traindata' entry
      elif traintest == 'test':
        #traindata will be present at this point in postmunge, not yet populated for automunge
        #traindata == True means test data to be treated as train data
        if postprocess_dict['traindata'] is True:
          traintest == 'train'

      #___________
      #now we're determining number of seeds needed for each sampling operation
      #based on rowcount, distribution parameters, and sampling type
      #using process_dict['noise_transform'] specification to know which sampling operations will be performed
      #and using sampling_type to know what kind of seedings are applied (e.g. bulk vs sample)
      #once we've calculated the count of seeds, we'll populated the returned populated_sampling_resource_dict seeds entries
      #with seeds extracted from the entropy_seeds bank
      
      #now we'll calculate sample count for use with bulk_seeds sampling_type
      #and call count for use with sampling_seed sampling_type
      #note that we'll calculate for train and test sets seperately
      #and the derivations will be specific to the type fo samplings (e.g. binomial, distribution, etc)
      #and specific to the type of noise_transform (e.g. numeric, categoric, binomial)
      #and we'll inspect assignparam specificaitons to take into account
      #once derived we'll save the results in the sampling_resource_dict 
      #and use to populate seeds in the populated_sampling_resource_dict
      #noting that the final counts won't be passed to transforms, only the seeds will
      
      binomial_train_sample_count = 0
      distribution_train_sample_count = 0
      choice_train_sample_count = 0
      parameterlist_train_sample_count = 0
      statsdistribution_train_sample_count = 0
      binomial_test_sample_count = 0
      distribution_test_sample_count = 0
      choice_test_sample_count = 0
      parameterlist_test_sample_count = 0
      statsdistribution_test_sample_count = 0
      
      binomial_train_call_count = 0
      distribution_train_call_count = 0
      choice_train_call_count = 0
      parameterlist_train_call_count = 0
      statsdistribution_train_call_count = 0
      binomial_test_call_count = 0
      distribution_test_call_count = 0
      choice_test_call_count = 0
      parameterlist_test_call_count = 0
      statsdistribution_test_call_count = 0

      #____
      #to derive stochastic seed counts, we'll access flip_prob parameters
      #where stochastic counts refers to number of sampled entries for distribution or choice sampling
      #which are a function of activations received from the binomial

      #if initializing parameters not material to a noise_transform type they will be ignored

      if 'trainnoise' in params:
        trainnoise = params['trainnoise']
      else:
        #if update these need to update transforms too
        trainnoise = True

      if 'testnoise' in params:
        testnoise = params['testnoise']
      else:
        #if update these need to update transforms too
        testnoise = False

      if 'flip_prob' in params:
        flip_prob = params['flip_prob']
      else:
        #if update these need to update transforms too
        flip_prob = 0.03

      if 'test_flip_prob' in params:
        test_flip_prob = params['test_flip_prob']
      else:
        #if update these need to update transforms too
        if noise_transform == 'numeric':
          test_flip_prob = flip_prob
        elif noise_transform in {'categoric', 'binary'}:
          test_flip_prob = 0.01
          
      if 'sigma' in params:
        sigma = params['sigma']
      else:
        #only inspecting sigma for scenario where passed as a list or stats so the default is immaterial, 
        sigma = 0.03
          
      if 'test_sigma' in params:
        test_sigma = params['test_sigma']
      else:
        #only inspecting test_sigma for scenario where passed as a list or stats so the default is immaterial, 
        test_sigma = 0
        
      if 'mu' in params:
        mu = params['mu']
      else:
        #only inspecting mu for scenario where passed as a list or stats so the default is immaterial, 
        mu = 0
          
      if 'test_mu' in params:
        test_mu = params['test_mu']
      else:
        #only inspecting test_mu for scenario where passed as a list or stats so the default is immaterial, 
        test_mu = 0

      if 'protected_feature' in params:
        protected_feature = params['protected_feature']

        if protected_feature == column \
        or protected_feature not in df_test:
          protected_feature = False

      else:
        protected_feature = False

      if 'weighted' in params:
        weighted = params['weighted']
      else:
        weighted = True

      if 'test_weighted' in params:
        test_weighted = params['test_weighted']
      else:
        test_weighted = True

      #__
          
      #check for scenarios where parameters passed as a list
      
      flip_prob_list_sample_count = 0
      if isinstance(flip_prob, list):
        flip_prob_list = flip_prob.copy()
        flip_prob_list_sample_count += 1
        #assign flip_prob to max value from list which will serve as basis for bulk_seeds count
        flip_prob = pd.DataFrame({0:flip_prob_list})[0].max()
        #if max was 1., get next highest value
        if flip_prob == 1:
          flip_prob_list.remove(flip_prob)
          flip_prob = pd.DataFrame({0:flip_prob_list})[0].max()
      
      test_flip_prob_list_sample_count = 0
      if isinstance(test_flip_prob, list):
        test_flip_prob_list = test_flip_prob.copy()
        test_flip_prob_list_sample_count += 1
        #assign test_flip_prob to max value from list which will serve as basis for bulk_seeds count
        test_flip_prob = pd.DataFrame({0:test_flip_prob_list})[0].max()
        #if max was 1., get next highest value
        if test_flip_prob == 1:
          test_flip_prob_list.remove(test_flip_prob)
          test_flip_prob = pd.DataFrame({0:test_flip_prob_list})[0].max()
      
      sigma_list_sample_count = False
      if isinstance(sigma, list):
        sigma_list_sample_count += 1
      
      test_sigma_list_sample_count = False
      if isinstance(test_sigma, list):
        test_sigma_list_sample_count += 1
      
      #__

      #for scenarios where parameters passed as a scipy stats distribution
      #will just sample and apply a safety factor
      
      stats_train_calls = 0
      stats_test_calls = 0
      
      if isinstance(mu, type(stats.expon(1))):
        stats_train_calls += 1

      if isinstance(test_mu, type(stats.expon(1))):
        stats_test_calls += 1

      if isinstance(sigma, type(stats.expon(1))):
        stats_train_calls += 1

      if isinstance(test_sigma, type(stats.expon(1))):
        stats_test_calls += 1
      
      if isinstance(flip_prob, type(stats.expon(1))):
        stats_train_calls += 1
        flip_prob = flip_prob.rvs()
        flip_prob = flip_prob * (1 + stochastic_count_safety_factor)
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob, type(stats.expon(1))):
        stats_test_calls += 1
        test_flip_prob = test_flip_prob.rvs()
        test_flip_prob = test_flip_prob * (1 + stochastic_count_safety_factor)
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1

      #____

      #for 'numeric' noise_transform will be samplings for binomial and distribution sampling
      #the distribution sampling will be be stochastic count of entries based on binomial result
      #so will increase stochastic count by the safety factor stochastic_count_safety_factor defaulting to 0.15
      #and there may be an additional sampling when flip_prob or test_flip_prob / sigma or test_sigma passed as list
      if noise_transform == 'numeric':
        
        if traintest in {'train', 'traintest'} and trainnoise is True:
          
          #calculate sample counts for bulk_seeds sampling_type case
          #bulk samples will be based on rowcount * (1 + flip_prob * (1 + stochastic_count_safety_factor))
          binomial_train_sample_count = rowcount_trian
          if flip_prob == 1:
            binomial_train_sample_count = 0
          
          distribution_train_sample_count = \
          rowcount_trian * flip_prob * (1 + stochastic_count_safety_factor)
          
          parameterlist_train_sample_count = \
          flip_prob_list_sample_count + sigma_list_sample_count
          
          statsdistribution_train_sample_count = stats_train_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_train_call_count = 1
          if flip_prob == 1:
            binomial_train_call_count = 0
          
          distribution_train_call_count = 1

          if protected_feature is not False:
            if not isinstance(df_train, bool):
              if protected_feature in list(df_train):
                distribution_train_call_count = df_train[protected_feature].nunique()
            else:
              if protected_feature in list(df_test):
                distribution_train_call_count = df_test[protected_feature].nunique()
          
          parameterlist_train_call_count = \
          flip_prob_list_sample_count + sigma_list_sample_count
          
          statsdistribution_train_call_count = stats_train_calls
        
        if traintest in {'test', 'traintest'} and testnoise is True:
          
          #calculate sample counts for bulk_seeds sampling_type case
          binomial_test_sample_count = rowcount_test
          if test_flip_prob == 1:
            binomial_test_sample_count = 0
          
          distribution_test_sample_count = \
          rowcount_test * test_flip_prob * (1 + stochastic_count_safety_factor)
          
          parameterlist_test_sample_count = \
          test_flip_prob_list_sample_count + test_sigma_list_sample_count
          
          statsdistribution_test_sample_count = stats_test_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_test_call_count = 1
          if test_flip_prob == 1:
            binomial_test_call_count = 0
          
          distribution_test_call_count = 1

          if protected_feature is not False:
            if protected_feature in list(df_test):
              distribution_test_call_count = df_test[protected_feature].nunique()
          
          parameterlist_test_call_count = \
          test_flip_prob_list_sample_count + test_sigma_list_sample_count
          
          statsdistribution_test_call_count = stats_test_calls

      #for 'categoric' noise_transform will be samplings for binomial and choice sampling
      #the choice sampling will be stochastic count of entries based on binomial result with safety factor stochastic_count_safety_factor
      #and there may be an additional sampling when flip_prob or test_flip_prob passed as list
      if noise_transform == 'categoric':
        
        if traintest in {'train', 'traintest'} and trainnoise is True:

          #calculate sample counts for bulk_seeds sampling_type case
          #bulk samples will be based on rowcount * (1 + flip_prob * (1 + stochastic_count_safety_factor))
          binomial_train_sample_count = rowcount_trian
          if flip_prob == 1:
            binomial_train_sample_count = 0

          choice_train_sample_count = \
          rowcount_trian * flip_prob * (1 + stochastic_count_safety_factor)

          parameterlist_train_sample_count = \
          flip_prob_list_sample_count
          
          statsdistribution_train_sample_count = stats_train_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_train_call_count = 1
          if flip_prob == 1:
            binomial_train_call_count = 0
          
          choice_train_call_count = 1

          if protected_feature is not False and weighted is True:
            if not isinstance(df_train, bool):
              if protected_feature in list(df_train):
                choice_train_call_count = df_train[protected_feature].nunique()
            else:
              if protected_feature in list(df_test):
                choice_train_call_count = df_test[protected_feature].nunique()

          parameterlist_train_call_count = \
          flip_prob_list_sample_count
          
          statsdistribution_train_call_count = stats_train_calls
        
        if traintest in {'test', 'traintest'} and testnoise is True:

          #calculate sample counts for bulk_seeds sampling_type case
          binomial_test_sample_count = rowcount_test
          if test_flip_prob == 1:
            binomial_test_sample_count = 0

          choice_test_sample_count = \
          rowcount_test * test_flip_prob * (1 + stochastic_count_safety_factor)

          parameterlist_test_sample_count = \
          test_flip_prob_list_sample_count
          
          statsdistribution_test_sample_count = stats_test_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_test_call_count = 1
          if test_flip_prob == 1:
            binomial_test_call_count = 0

          choice_test_call_count = 1

          if protected_feature is not False and test_weighted is True:
            if protected_feature in list(df_test):
              choice_test_call_count = df_test[protected_feature].nunique()

          parameterlist_test_call_count = \
          test_flip_prob_list_sample_count
          
          statsdistribution_test_call_count = stats_test_calls
          
      #for 'binary' noise_transform will be samplings just for binomial sampling
      if noise_transform == 'binary':
        
        if traintest in {'train', 'traintest'} and trainnoise is True:

          #calculate sample counts for bulk_seeds sampling_type case
          binomial_train_sample_count = rowcount_trian
          
          parameterlist_train_sample_count = \
          flip_prob_list_sample_count
          
          statsdistribution_train_sample_count = stats_train_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_train_call_count = 1
          
          parameterlist_train_call_count = \
          flip_prob_list_sample_count
          
          statsdistribution_train_call_count = stats_train_calls

        if traintest in {'test', 'traintest'} and testnoise is True:

          #calculate sample counts for bulk_seeds sampling_type case
          binomial_test_sample_count = rowcount_test
          
          parameterlist_test_sample_count = \
          test_flip_prob_list_sample_count
          
          statsdistribution_test_sample_count = stats_test_calls
            
          #calculate call counts for sampling_seed sampling_type case
          binomial_test_call_count = 1
          
          parameterlist_test_call_count = \
          test_flip_prob_list_sample_count
          
          statsdistribution_test_call_count = stats_test_calls
        
      #_____
        
      sampling_resource_dict.update({'binomial_train_sample_count' : int(binomial_train_sample_count),
                                     'distribution_train_sample_count' : int(distribution_train_sample_count),
                                     'choice_train_sample_count' : int(choice_train_sample_count),
                                     'parameterlist_train_sample_count' : int(parameterlist_train_sample_count),
                                     'statsdistribution_train_sample_count' : int(statsdistribution_train_sample_count),
                                     'binomial_test_sample_count' : int(binomial_test_sample_count),
                                     'distribution_test_sample_count' : int(distribution_test_sample_count),
                                     'choice_test_sample_count' : int(choice_test_sample_count),
                                     'parameterlist_test_sample_count' : int(parameterlist_test_sample_count),
                                     'statsdistribution_test_sample_count' : int(statsdistribution_test_sample_count),
                                    })

      sampling_resource_dict.update({'binomial_train_call_count' : int(binomial_train_call_count),
                                     'distribution_train_call_count' : int(distribution_train_call_count),
                                     'choice_train_call_count' : int(choice_train_call_count),
                                     'parameterlist_train_call_count' : int(parameterlist_train_call_count),
                                     'statsdistribution_train_call_count' : int(statsdistribution_train_call_count),
                                     'binomial_test_call_count' : int(binomial_test_call_count),
                                     'distribution_test_call_count' : int(distribution_test_call_count),
                                     'choice_test_call_count' : int(choice_test_call_count),
                                     'parameterlist_test_call_count' : int(parameterlist_test_call_count),
                                     'statsdistribution_test_call_count' : int(statsdistribution_test_call_count),
                                    })
      
      #_____
      
      #now that we have the category/column specific counts, can populate seeds from our entropy bank
      
      #in the sampling_type == 'default' scenario, we pass same seeds to every operation
      #without striking those seeds from the bank
      #with a shuffle operation performed here first
      if sampling_type == 'default':

        entropy_seeds = _shuffle_seeds(entropy_seeds, random_generator, sampling_type, sampling_generator, random_generator_accepts_seeds)
        
        for sample_type_seeds in {'binomial_train_seeds', 'binomial_test_seeds', 
                                  'distribution_train_seeds', 'distribution_test_seeds',
                                  'choice_train_seeds', 'choice_test_seeds',
                                  'parameterlist_train_seeds', 'parameterlist_test_seeds',
                                  'statsdistribution_train_seeds', 'statsdistribution_test_seeds'}:
          
          populated_sampling_resource_dict.update({sample_type_seeds : entropy_seeds})
      
      #in bulk scenario we have one seed for each sampled entry
      elif sampling_type in {'bulk_seeds'}:
        
        for sample_type_count in {'binomial_train_sample_count', 'binomial_test_sample_count',
                                  'distribution_train_sample_count', 'distribution_test_sample_count',
                                  'choice_train_sample_count', 'choice_test_sample_count',
                                  'parameterlist_train_sample_count', 'parameterlist_test_sample_count',
                                  'statsdistribution_train_sample_count', 'statsdistribution_test_sample_count'}:
          
          #relies on string convention [-12:] == 'sample_count'
          sample_type_seeds = sample_type_count[:-12] + 'seeds'
          
          if sampling_resource_dict[sample_type_count] > 0:
            
            entropy_seeds = _shuffle_seeds(entropy_seeds, random_generator, sampling_type, sampling_generator, random_generator_accepts_seeds)
            
            populated_seeds = entropy_seeds
            if len(entropy_seeds) > sampling_resource_dict[sample_type_count]:
                            
              #grab seeds
              populated_seeds = entropy_seeds[:int(sampling_resource_dict[sample_type_count])]
              
              #now strike the accessed seeds
              entropy_seeds = entropy_seeds[int(sampling_resource_dict[sample_type_count]):]
              
            populated_sampling_resource_dict.update({sample_type_seeds : populated_seeds})
              
      #in sampling scenario we have one seed for each sampling operation
      elif sampling_type in {'sampling_seed'}:
              
        for sample_call_count in {'binomial_train_call_count', 'binomial_test_call_count',
                                  'distribution_train_call_count', 'distribution_test_call_count',
                                  'choice_train_call_count', 'choice_test_call_count',
                                  'parameterlist_train_call_count', 'parameterlist_test_call_count',
                                  'statsdistribution_train_call_count', 'statsdistribution_test_call_count'}:
          
          #relies on string convention [-10:] == 'call_count'
          sample_type_seeds = sample_call_count[:-10] + 'seeds'
          
          populated_seeds = entropy_seeds
          if sampling_resource_dict[sample_call_count] > 0:
          
            entropy_seeds = _shuffle_seeds(entropy_seeds, random_generator, sampling_type, sampling_generator, random_generator_accepts_seeds)
            
            if len(entropy_seeds) > sampling_resource_dict[sample_call_count]:
              
              #grab seeds
              populated_seeds = entropy_seeds[:int(sampling_resource_dict[sample_call_count])]
              
              #now strike the accessed seeds
              entropy_seeds = entropy_seeds[int(sampling_resource_dict[sample_call_count]):]
              
            populated_sampling_resource_dict.update({sample_type_seeds : populated_seeds})
      
      #in transform scenario we have one seed for each transformation function
      elif sampling_type in {'transform_seed'}:
        
        entropy_seeds = _shuffle_seeds(entropy_seeds, random_generator, sampling_type, sampling_generator, random_generator_accepts_seeds)

        populated_seeds = entropy_seeds
        if len(entropy_seeds) > 1:

          #grab seeds
          populated_seeds = entropy_seeds[:1]

          #now strike the accessed seeds
          entropy_seeds = entropy_seeds[1:]
        
        for sample_type_seeds in {'binomial_train_seeds', 'binomial_test_seeds',
                                  'distribution_train_seeds', 'distribution_test_seeds',
                                  'choice_train_seeds', 'choice_test_seeds',
                                  'parameterlist_train_seeds', 'parameterlist_test_seeds',
                                  'statsdistribution_train_seeds', 'statsdistribution_test_seeds'}:
            
          populated_sampling_resource_dict.update({sample_type_seeds : populated_seeds})
          
      #params receives populated sampling_resource_dict passing entropy seeds to the trasnform
      params.update({'sampling_resource_dict' : populated_sampling_resource_dict})
      
      #now update the postprocess_dict['entropy_seeds'] with the revised set
      #which may have struck some entries and will have been shuffled
      postprocess_dict['entropy_seeds'] = entropy_seeds
    
    return params, postprocess_dict

  def __populate_randomseed(self, randomseed, entropy_seeds, 
                            random_generator, sampling_dict):
    """
    when a specific randomseed wasn't specified
    a custom randomseed will be sampled
    if entropy_seeds are available
    one will be applied for the sampling
    if a custom random_generator was specified
    the use between PCG64 or custom 
    will follow the convention of sampling_dict['extra_seed_generator']
    
    entropy_seeds may either recieve parameter entropy_seeds or postprocess_dict['entropy_seeds']
    """

    populate_randomseed_expended_seed_count = 0

    #2**31 - 1
    #(note that np.random.SeedSequence accepts larger values, this range is used to align with np.int32
    max_capacity_seed = 2147483647
    
    randomrandomseed = False
    if randomseed is False:
      randomrandomseed = True
      
      #note sampling_dict initialized in __check_sampling_dict
      sampling_generator = sampling_dict['sampling_generator']  
      sampling_type = sampling_dict['sampling_type']
      random_generator_accepts_seeds = sampling_dict['random_generator_accepts_seeds']
      
      #this determines whether we extract additional seeds with default or custom generator
      if sampling_generator == 'PCG64':
        randomgenerator = np.random.PCG64
        sampling_generator_accepts_seeds = True
      elif sampling_generator == 'MersenneTwister':
        randomgenerator = np.random.MT19937
        sampling_generator_accepts_seeds = True
      elif sampling_generator == 'custom':
        #else only use default if random_generator not specified
        if random_generator is False:
          randomgenerator = np.random.PCG64
          sampling_generator_accepts_seeds = True
        else:
          randomgenerator = random_generator
          sampling_generator_accepts_seeds = random_generator_accepts_seeds
      
      if len(entropy_seeds) > 2:
    
        shuffle_seed = [int(entropy_seeds[0])]
        entropy_seeds = np.delete(entropy_seeds, 0)
        populate_randomseed_expended_seed_count += 1

        if sampling_generator_accepts_seeds is True:
          nprandom = np.random.Generator(randomgenerator(np.random.SeedSequence(spawn_key=shuffle_seed)))
        else:
          nprandom = np.random.Generator(randomgenerator)

        nprandom.shuffle(entropy_seeds)
        
      if len(entropy_seeds) >= 2:
        
        spawn_seed = [int(entropy_seeds[0])]
        entropy_seeds = np.delete(entropy_seeds, 0)
        populate_randomseed_expended_seed_count += 1
        
      elif len(entropy_seeds) == 1:
        
        spawn_seed = [entropy_seeds[0]]

      else:
        #(2**31 - 1)
        spawn_seed = [int(random.randint(0,max_capacity_seed))]

      if sampling_generator_accepts_seeds is True:
        nprandom = np.random.Generator(randomgenerator(np.random.SeedSequence(spawn_key=spawn_seed)))
      else:
        nprandom = np.random.Generator(randomgenerator)

      randomseed = int(nprandom.integers(0, high=max_capacity_seed, size=1))
          
    return randomseed, randomrandomseed, entropy_seeds, populate_randomseed_expended_seed_count

  def __sample_from_parameter_list(self, parameter_list, sampling_resource_dict, nprandom_dict, traintest):
    
    #traintest accepts {'train', 'test'}
    #this function specific to sampling_id = 'parameterlist'
    sampling_id = 'parameterlist_' + traintest
    
    if isinstance(parameter_list, list) and len(parameter_list) > 0:
      
      #we want a distinct seed for each list sampling operation
      #if multiple parameters passed as list sampling_resource_dict may have one seed for each based on teh sampling_type
      #the seeds will be received as a flattened numpy array
      #so we'll access the full array, pass a single entry to get_nprandom, and then re-enter after striking the used entry
      #which we'll only do when len > 1         
      received_seeds = sampling_resource_dict[sampling_id + '_seeds']
      if len(received_seeds) > 1:
        curent_seed = np.array([received_seeds[0]])
        sampling_resource_dict[sampling_id + '_seeds'] = curent_seed
        returned_seeds = received_seeds[1:]
      
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += 1
      
      max_capacity_integer = len(parameter_list) - 1
      
      parameter_index = \
      nprandom.integers(0, high=max_capacity_integer, size=1).astype(int)

      parameter_index = parameter_index[0]
      
      parameter = parameter_list[parameter_index]
      
      #now reset the seeds minus the expended entry
      if len(received_seeds) > 1:
        sampling_resource_dict[sampling_id + '_seeds'] = returned_seeds
      
    return parameter, sampling_resource_dict

  def __sample_from_scipy_stats(self, stats_distribution, sampling_resource_dict, nprandom_dict, traintest):
    
    #traintest accepts {'train', 'test'}
    #this function specific to sampling_id = 'statsdistribution'
    sampling_id = 'statsdistribution_' + traintest
    
    if isinstance(stats_distribution, type(stats.expon(1))):
      
      #we want a distinct seed for each stats sampling operation
      #if multiple parameters passed as list sampling_resource_dict may have one seed for each based on teh sampling_type
      #the seeds will be received as a flattened numpy array
      #so we'll access the full array, pass a single entry to get_nprandom, and then re-enter after striking the used entry
      #which we'll only do when len > 1         
      received_seeds = sampling_resource_dict[sampling_id + '_seeds']
      if len(received_seeds) > 1:
        curent_seed = np.array([received_seeds[0]])
        sampling_resource_dict[sampling_id + '_seeds'] = curent_seed
        returned_seeds = received_seeds[1:]
      
      nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
      sampling_resource_dict[sampling_id + '_call_count'] += 1
      sampling_resource_dict[sampling_id + '_sample_count'] += 1
      
      #this is a scipy stats option to set a np.random generator for basis of sampling
      stats_distribution.random_state=nprandom
      
      #now sample the value from the distribution
      sample = stats_distribution.rvs()
      
      #now reset the seeds minus the expended entry
      if len(received_seeds) > 1:
        sampling_resource_dict[sampling_id + '_seeds'] = returned_seeds
      
    return sample, sampling_resource_dict

  def __get_nprandom(self, sampling_id, sampling_resource_dict, nprandom_dict):
    
    #initializes nprandom for sampling based on sampling_id, sampling_resource_dict, and nprandom_dict
    #sampling_id is one of {'binomial_train', 'binomial_test', 'distribution_train', 'distribution_test', 'choice_train_seeds', 'choice_test_seeds'}
    #distinguishes between supplemental_seeds and primary_seeds seeding_type
    #note that if a generator is called to generate additional seeds it does not use this function
    #to avoid generating additional seeds for bulk_seeds sampling_type without supplemental seeding from OS
    
    entropy_seeds = sampling_resource_dict[sampling_id + '_seeds']
    if sampling_resource_dict[sampling_id] == 'custom' \
    and sampling_resource_dict['random_generator_accepts_seeds'] is False:
      nprandom = np.random.Generator(nprandom_dict[sampling_resource_dict[sampling_id]])
    else:
      
      #in supplemental_seeds seeding_type user passed entropy_seeds supplement seeding from OS
      if 'seeding_type' in sampling_resource_dict \
      and sampling_resource_dict['seeding_type'] == 'supplemental_seeds' \
      or 'seeding_type' not in sampling_resource_dict:
        nprandom = np.random.Generator(nprandom_dict[sampling_resource_dict[sampling_id]](np.random.SeedSequence(spawn_key=entropy_seeds)))
      
      #in primary_seeds seeding_type user passed entropy_seeds are only source of seeding
      #this is the default for bulk_seeds sampling_type
      elif 'seeding_type' in sampling_resource_dict \
      and sampling_resource_dict['seeding_type'] == 'primary_seeds':
        nprandom = np.random.Generator(nprandom_dict[sampling_resource_dict[sampling_id]](seed = entropy_seeds))

    return nprandom

  def __erase_seeds(self, sampling_resource_dict):
    
    #sampling_resource_dict has seeds erase before return to preserve privacy of entropy
    #not applied in postprocess functions since postmunge(.) doesn't return a postprocess_dict
    keys = list(sampling_resource_dict)
    for key in keys:
      if key[-5:] == 'seeds':
        del sampling_resource_dict[key]
    return sampling_resource_dict
  
  #__FunctionBlock: functionpointer support

  def __grab_functionpointer_entries_support(self, targetcategory, pointercategory, processdict_, process_dict, \
                                            i, check_functionpointer_result, printstatus):
    """
    #support function for grab_processdict_functions
    #takes as input the targetcategory that has a pointer entry
    #and the associated pointercategory
    #where if pointercategory itself has a pointer, calls this function recursively
    #to find the associated entries
    #and assign them to targetcategory entry in processdict unless previously specified
    #where first checks processdict and if not present checks process_dict
    #where processdict is user passed data strcuture
    #and process_dict is the internal library prior to consolidation
    #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries
    #ie only externally defined processdict can have functionpointers
    #tracks a counter i to ensure don't get caught in infinite loop, defaults to 1111 cycles limit

    #where targetcategory is the processdict category entry that has a functionpointer entry
    #pointercategory is the corresponding functionpointer entry
    #and for chains of functionpointer entries the targetcategory remains same and pointercategory is updated

    #functionpointer halts when it reaches a processdict or process_dict entry without a functionpointer
    
    #note that processdict_ may be recieved either as the external processdict or process_dict
    #so setting a distinct name as processdict_ to avoid overlap in mutable container naming space
    """

    #update_targets are the targets for functionpointer, 
    #includes all processdict options except for defaultparams and labelctgy
    #(defaultparams handled seperately, labelcty is specific to a family tree)
    
    update_targets = ['custom_train', 'custom_test', 'custom_inversion', 'dualprocess', \
                      'singleprocess', 'postprocess', 'inverseprocess', 'info_retention', \
                      'inplace_option', 'NArowtype', 'MLinfilltype', 'defaultinfill', 'dtype_convert']
    
    #counter i is here to ensure if we're recursively following chains of pointers we don't get caught in loop
    if i > 1111:
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: functionpointer cycled through 1111 entries without finding a stopping point")
      self.__autoprint('warning', "for processdict category entry: ", targetcategory)
      self.__autoprint('warning', "likely infinite loop")
      
      check_functionpointer_result = True
      
    else:
      
      #(providing #_1_ #_2_ etc as visualization aid for indentation scheme since using two space tabs)
      
      i+=1
      
      #_1_
      #checking that pointercategory != targetcategory ensures self-referential are accessed from process_dict instead of processdict
      if pointercategory in processdict_ and pointercategory != targetcategory:
        
        #if function poitner points to a category that itself has a functionpointer
        #then after accessing entries not previously specified 
        #we'll call _grab_processdict_functions_support recursively
        
        #_2_
        if 'functionpointer' in processdict_[pointercategory]:

          #_3_
          for update_target in update_targets:
            
            if update_target in processdict_[pointercategory] \
            and update_target not in processdict_[targetcategory]:
              processdict_[targetcategory][update_target] = processdict_[pointercategory][update_target]
              
          #_3_
          #defaultparams gets special treatment since accessing entries in a dictionary
          if 'defaultparams' in processdict_[pointercategory]:
            if 'defaultparams' in processdict_[targetcategory]:
              defaultparams = self.__autocopy(processdict_[pointercategory]['defaultparams'])
              defaultparams.update(processdict_[targetcategory]['defaultparams'])
              processdict_[targetcategory]['defaultparams'] = defaultparams
            else:
              processdict_[targetcategory]['defaultparams'] = processdict_[pointercategory]['defaultparams']
          
          #_3_
          #make sure the linked pointercategory doesn't have self-referential functionpointer
          #which happens when used to overwrite it's own process_dict entry
          if pointercategory != processdict_[pointercategory]['functionpointer']:

            #now new pointer category is the functionpointer entry of the prior functionpointer entry
            pointercategory = processdict_[pointercategory]['functionpointer']

            #follow through recursion
            processdict_, i, check_functionpointer_result = \
            self.__grab_functionpointer_entries_support(targetcategory, pointercategory, processdict_, process_dict, \
                                                       i, check_functionpointer_result, printstatus)
          
          #_3_
          #this elif is to accomodate self-referential functionpointers
          #when linked from a chain 
          #so that they aren't interpreted as infinite loops
          #so instead of calling _grab_functionpointer_entries_support on processdict
          #we'll just access from the corresponding process_dict entry
          elif pointercategory == processdict_[pointercategory]['functionpointer']:
            
            #_4_
            if pointercategory in process_dict:

              for update_target in update_targets:

                if update_target in process_dict[pointercategory] \
                and update_target not in processdict_[targetcategory]:
                  processdict_[targetcategory][update_target] = process_dict[pointercategory][update_target]

              #defaultparams gets special treatment since accessing entries in a dictionary
              if 'defaultparams' in process_dict[pointercategory]:
                if 'defaultparams' in processdict_[targetcategory]:
                  defaultparams = self.__autocopy(process_dict[pointercategory]['defaultparams'])
                  defaultparams.update(processdict_[targetcategory]['defaultparams'])
                  processdict_[targetcategory]['defaultparams'] = defaultparams
                else:
                  processdict_[targetcategory]['defaultparams'] = process_dict[pointercategory]['defaultparams']

            #_4_
            else:

              check_functionpointer_result = True

              # if printstatus != 'silent':
              self.__autoprint('warning', "error: user passed processdict entry for category ", pointercategory)
              self.__autoprint('warning', "contained a self-referential functionpointer that did not point to a category found in process_dict")
              self.__autoprint('warning', '')            
            
        #_2_
        #else grab and halt the chain since no functionpoitner populated for pointercategory
        elif 'functionpointer' not in processdict_[pointercategory]:

          #_3_
          for update_target in update_targets:

            if update_target in processdict_[pointercategory] \
            and update_target not in processdict_[targetcategory]:
              processdict_[targetcategory][update_target] = processdict_[pointercategory][update_target]

          #defaultparams gets special treatment since accessing entries in a dictionary
          if 'defaultparams' in processdict_[pointercategory]:
            if 'defaultparams' in processdict_[targetcategory]:
              defaultparams = self.__autocopy(processdict_[pointercategory]['defaultparams'])
              defaultparams.update(processdict_[targetcategory]['defaultparams'])
              processdict_[targetcategory]['defaultparams'] = defaultparams
            else:
              processdict_[targetcategory]['defaultparams'] = processdict_[pointercategory]['defaultparams']
                
      #_1_
      #if pointercategory wasn't in user passed processdict, we'll next check the internal library process_dict
      elif pointercategory in process_dict:

        #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries
        #so we don't need as many steps as above, can just assume processing functions are present
        
        #_2_
        for update_target in update_targets:

          if update_target in process_dict[pointercategory] \
          and update_target not in processdict_[targetcategory]:
            processdict_[targetcategory][update_target] = process_dict[pointercategory][update_target]

        #_2_
        #defaultparams gets special treatment since accessing entries in a dictionary
        if 'defaultparams' in process_dict[pointercategory]:
          if 'defaultparams' in processdict_[targetcategory]:
            defaultparams = self.__autocopy(process_dict[pointercategory]['defaultparams'])
            defaultparams.update(processdict_[targetcategory]['defaultparams'])
            processdict_[targetcategory]['defaultparams'] = defaultparams
          else:
            processdict_[targetcategory]['defaultparams'] = process_dict[pointercategory]['defaultparams']

      #_1_
      #if pointercategory wasn't found in either of processdict or process_dict
      else:
        
        check_functionpointer_result = True
        
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: user passed processdict entry for category ", targetcategory)
        self.__autoprint('warning', "contained a functionpointer that did not point to a category found in processdict or process_dict")
        self.__autoprint('warning', "note that self-referential functionpointers only supported when overwriting entry in process_dict")
        self.__autoprint('warning', '')

    return processdict_, i, check_functionpointer_result
  
  def __grab_functionpointer_entries(self, processdict_, process_dict, printstatus):
    """
    #checks for functionpointer entries in user passed processdict
    #when present populates that category with associated pointer entries not previously assigned
    #where processdict is user passed data structure
    #and process_dict is internal library prior to consolidation
    #and functionpointer refers to an option to populate processdict
    #with an entry that grabs entries from another process_dict entry
    #i.e. singleprocess, dualprocess, postprocess, NArowtype, MLinfilltype, etc
    #(basically everything except labelctgy)
    #functionpointer first checks the processdict, and if the pointer target is not present checks the process_dict
    #note that functionpointer also grabs defaultparams
    #although if any defaultparam entries in the set with functionpointer they will override corresponding entries
    #for example if category with pointer has defaultparam entires for suffix
    #which points to anotehr category with differnt defaultparam entries for suffix
    #the entry in the set with the pointer takes precedence as opposed to the target set
    #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries

    #note that also supports passing processdict as process_dict for use of functionpointer towards process_dict
    
    #note that processdict_ may be recieved either as the external processdict or process_dict
    #so setting a distinct name as processdict_ to avoid overlap in mutable container naming space
    """

    check_functionpointer_result = False
    
    for entry in processdict_:
      
      if 'functionpointer' in processdict_[entry]:
        
        i = 0
        targetcategory = entry
        pointercategory = processdict_[entry]['functionpointer']
        
        processdict_, i, check_functionpointer_result = \
        self.__grab_functionpointer_entries_support(targetcategory, pointercategory, processdict_, process_dict, \
                                               i, check_functionpointer_result, printstatus)

    return processdict_, check_functionpointer_result

  #__FunctionBlock: column assignments string conversion
  
  def __assigncat_str_convert(self, assigncat, printstatus):
    """
    #Converts all assigncat entries to string (just in case user passed integer)
    #
    #also converts any single string/integer entries passed without list brackets 
    #into a single entry list
    #and set specifications are retained
    """

    assigncat_str_convert_valresult = False
    
    for assigncatkey in assigncat:
      current_list = assigncat[assigncatkey]

      if isinstance(current_list, str):
        current_list = [current_list]

      elif isinstance(current_list, (int, float)):  
        current_list = [str(int(current_list))]
        
      elif isinstance(current_list, set):
        
        if len([x for x in current_list if not isinstance(x, (str, int, float))]) > 0:
          assigncat_str_convert_valresult = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "warning of potential error channel")
          self.__autoprint('warning', "assigncat accepts specification as a dictiaonry with str keys")
          self.__autoprint('warning', "and values of str/int/float/set types representing column headers")
          self.__autoprint('warning', "or a list/set with entries of those types")
          self.__autoprint('warning', '')
            
        else:
          
          current_list_set = set()
          for x in current_list:
            current_list_set = current_list_set | {str(x)}
            
          current_list = [current_list_set]

      elif isinstance(current_list, list):
        
        current_list_others = []
        current_list_sets = []
        
        if len([x for x in current_list if not isinstance(x, (str, int, float, set))]) > 0:

          assigncat_str_convert_valresult = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "warning of potential error channel")
          self.__autoprint('warning', "assigncat accepts specification as a dictiaonry with str keys")
          self.__autoprint('warning', "and values of str/int/float/set types representing column headers")
          self.__autoprint('warning', "or a list/set with entries of those types")
          self.__autoprint('warning', '')

        current_list_sets = [x for x in current_list if isinstance(x, set)]
        current_list_others = [str(x) for x in current_list if not isinstance(x, set)]

        for x in current_list_sets:
          current_set = set()
          for y in x:

            if isinstance(y, (str, int, float)):
              current_set = current_set | {str(y)}

            else:

              assigncat_str_convert_valresult = True
              # if printstatus != 'silent':
              self.__autoprint('warning', "warning of potential error channel")
              self.__autoprint('warning', "assigncat accepts specification as a dictiaonry with str keys")
              self.__autoprint('warning', "and values of str/int/float/set types representing column headers")
              self.__autoprint('warning', "or a list/set with entries of those types")
              self.__autoprint('warning', '')

          current_list_sets[current_list_sets.index(x)] = current_set
          
        current_list = current_list_others + current_list_sets

      assigncat[assigncatkey] = current_list

    return assigncat, assigncat_str_convert_valresult

  def __assigninfill_str_convert(self, assigninfill):
    """
    #Converts all assigninfill entries to string (just in case user passed integer)
    #
    #also converts any single string/integer entries passed without list brackets 
    #into a single entry list
    """

    #ignore edge case where user passes empty dictionary
    if assigninfill != {}:

      for assigninfillkey in sorted(assigninfill):
        current_list = assigninfill[assigninfillkey]
          
        #check if current_list is a list, if not populate it as single entry in a list
        if type(current_list) != type([]):
          assigninfill[assigninfillkey] = [current_list]
          current_list = assigninfill[assigninfillkey]
        
        #then convert any entries in the list to string type
        assigninfill[assigninfillkey] = [str(i) for i in current_list]

      del current_list

    return assigninfill
  
  def __parameter_str_convert(self, parameter):
    """
    #Converts parameter, such as one that might be either list or int or str, to a str or list of str
    #where True or False left unchanged
    #if parameter is a list and first entry encosed in set brackets it is retained as set
    #if parameter is a list and any entries are a list then entries of sublists are also converted
    #(the purpose of this operation is to align with convention that column headers are str type,
    #and so parameters specifying column headers, as may include integers, are converted to str to align)

    #note that float(int) are converted to str(float(int)) instead of str(int) to avoid edge case
    """

    if isinstance(parameter, int) and str(parameter) != 'False' and str(parameter) != 'True':
      parameter_copy = str(parameter)
    elif isinstance(parameter, float):
      parameter_copy = str(parameter)
    elif isinstance(parameter, bytes):
      parameter_copy = bytes.decode(parameter, 'utf-8')
    elif isinstance(parameter, list):
      first_target = 0
      #if first entry is set it may be a binary specificaiton so will leave intact
      if isinstance(parameter[0], set):
        first_target = 1
      
      parameter_copy = parameter.copy()
      for entry in parameter[first_target:]:
        
        if isinstance(entry, list):
          entry_str = self.__parameter_str_convert(entry)
        else:
          if isinstance(entry, bytes):
            entry_str = bytes.decode(entry, 'utf-8')
          else:
            entry_str = str(entry)
        
        parameter_copy[parameter.index(entry)] = entry_str

    else:
      parameter_copy = parameter

    return parameter_copy
  
  def __assignparam_str_convert(self, assignparam):
    """
    #Converts all column entries to assignparam to string in case user passed integer
    #such as for numpy arrays
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    """

    categorykeys = list(assignparam).copy()
    
    for categorykey in categorykeys:
      
      if categorykey not in {'global_assignparam', 'default_assignparam'}:
        
        columnkeys = list(assignparam[categorykey]).copy()
        
        for columnkey in columnkeys:
          
          assignparam[categorykey][str(columnkey)] = assignparam[categorykey].pop(columnkey)
          
    return assignparam

  def __assignnan_str_convert(self, assignnan):
    """
    #convention is that user can pass integers to column assignments
    #and they are converted to strings
    """

    if 'columns' in assignnan:
      
      columns_copy = self.__autocopy(assignnan['columns'])
      
      for entry in columns_copy:
        
        if isinstance(entry, int):
          
          assignnan['columns'].update({str(entry) : assignnan['columns'][entry]})
          
          del assignnan['columns'][entry]
          
    return assignnan

  def __assignnan_list_convert(self, assignnan):
    """
    #converts any passed infill values to lists in case passed as single value
    """

    if 'categories' in assignnan:
      
      #convert any entries to lists
      for entry1 in assignnan['categories']:
        
        if not isinstance(assignnan['categories'][entry1], list):
          
          assignnan['categories'][entry1] = [assignnan['categories'][entry1]]
          
    if 'columns' in assignnan:
      
      #convert any entries to lists
      for entry2 in assignnan['columns']:
        
        if not isinstance(assignnan['columns'][entry2], list):
          
          assignnan['columns'][entry2] = [assignnan['columns'][entry2]]
          
    if 'global' in assignnan:
      
      if not isinstance(assignnan['global'], list):
        
        assignnan['global'] = [assignnan['global']]
          
    return assignnan

  def __convert_floatnan_to_npnan(self, data_):
    """
    pickling and encryption may have possibility to convert np.nan objects to float("NaN")
    which has potential to interfere with their use as dictionary address keys
    as is used for protected_feature option in DP family of transforms
    this support function accecpts lists or dictionaries that may have been exposed to conversion
    and reverts to np.nan format
    currently only applied for first layer keys in a dicitonary
    """

    if isinstance(data_, list):
      data_ = [np.nan if x != x else x for x in data_]
      
    if isinstance(data_, dict):
      for key, value in self.__autocopy(data_).items():
        if key != key:
          del data_[key]
          data_[np.nan] = value

    return data_

  #__FunctionBlock: label smoothing support

  def __apply_LabelSmoothing(self, df, targetcolumn, epsilon, label_categorylist, label_category, categorycomplete_dict, LSfit, LSfitparams_dict):
    """
    #note that this is now used as a support function for _process_smth and _postprocess_smth

    #applies label smoothing based on user passed epsilon 
    
    #if LSfit is False
    #based on method described in "Rethinking the Inception Architecture for Computer Vision" by Szegedy et al
    #hat tip to Stack Overflow user lejlot for some implementaiton suggestions
    # https://stackoverflow.com/questions/39335535/label-smoothing-soft-targets-in-pandas
    
    #if LSfit is True
    #based on extension wherein the Smoothing factor K for each column is fit to the
    #distribution of the set is is a function of the activation column and target column
    
    #we'll follow convention that in both cases label smoothing applied to all columns in categorylist
    #and return a dictionary indicating which columns have received
    #(dictionary categorycomplete_dict initialized external to function)
    
    #we'll also return a dictionary containing any dervied parameters for LSfit such as for
    #subsequent consistent encoding
    
    #Note that categorylist is the list of columns originating from same transformation
    #and we currently exlcude '1010' binary encoded sets from the method
    #a future extension may address 1010 for MLinfill by smoothing after conversion to one-hot encoding 
    #such as for MLinfill
    """

#     unique_set = set(pd.unique(df[label_categorylist].to_numpy().ravel('K')))
    
#     if (unique_set == {0,1} \
#     or unique_set == {0} \
#     or unique_set == {1}) \
#     and label_category != '1010' \
#     and len(label_categorylist) > 1:
    
#     LSfitparams_dict = {}
  
    #initialize store of derived parameters
    for column1 in label_categorylist:
      
      LSfitparams_dict.update({column1 : {'LSfit' : LSfit, \
                                          'epsilon' : epsilon, \
                                          'label_categorylist' : label_categorylist, \
                                          'label_category' : label_category}})
    
    if LSfit is True:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        #if LSfit is True we'll apply LS to all columns in categorylist

        #activation_dcit will track the count of activations for each column in the categorylist
        activation_dict = {}

        for column1 in label_categorylist:

          activations = df[column1].sum()

          activation_dict.update({column1 : activations})
        
        #populate this in params dictionary
        for column1 in label_categorylist:
          
          LSfitparams_dict[column1].update({'activation_dict' : activation_dict})

        #LS_dict will be where we keep track of the activation distributions associated with each column
        #note we'll need activation ratios for each column as a function of activated column for a row
        LS_dict = {}
        for column1 in label_categorylist:

          LS_dict.update({column1 : {}})

          total_activations = 0

          for column2 in label_categorylist:

            if column1 != column2:

              total_activations += activation_dict[column2]

          for column2 in label_categorylist:

            if column1 != column2:

              LS_dict[column1].update({column2 : activation_dict[column2] / total_activations})
           
        #populate this in params dictionary
        for column1 in label_categorylist:
          
          LSfitparams_dict[column1].update({'LS_dict' : LS_dict})

        for column1 in label_categorylist:

          for column2 in label_categorylist:

            if column2 != column1:
              
              Smoothing_K = LS_dict[column2][column1]

              df = \
              self.__autowhere(df, column1, df[column2] == 1, (1 - epsilon) * Smoothing_K, specified='replacement')

        for column1 in label_categorylist:

          df = \
          self.__autowhere(df, column1, df[column1]==1, df[column1] * (epsilon), specified='replacement')

          categorycomplete_dict[column1] = True
    
    #if LSfit is not True:
    #else we'll only apply LS to the passed column with assumption of level distribution for K
    else:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        for column1 in label_categorylist:
        
          Smoothing_K = len(label_categorylist)

          #==1 value corresponds to 'bnry' encoding, >1 corresonds to one-hot encodings eg 'text'
          if Smoothing_K > 1:
            Smoothing_K -= 1

          df[column1] = \
          df[column1] * (epsilon) + (1 - df[column1]) * (1 - epsilon) / Smoothing_K

          categorycomplete_dict[column1] = True

    return df, categorycomplete_dict, LSfitparams_dict
                
  def __postapply_LabelSmoothing(self, df, targetcolumn, categorycomplete_dict, LSfitparams_dict):
    """
    #note that this is now used as a support function for _process_smth and _postprocess_smth

    #applies label smoothing based on user passed LSfitparams_dict
    #consiostently to label smoothing from corresponding train data
    
    #if LSfit is False
    #based on method described in "Rethinking the Inception Architecture for Computer Vision" by Szegedy et al
    #hat tip to Stack Overflow user lejlot for some implementaiton suggestions
    # https://stackoverflow.com/questions/39335535/label-smoothing-soft-targets-in-pandas
    
    #if LSfit is True
    #based on extension wherein the Smoothing factor K for each column is fit to the
    #distribution of the set is is a function of the activation column and target column
    
    #we'll follow convention that in both cases label smoothing applied to all columns in categorylist
    #and return a diciotnary indicating which columns have received
    #(dictionary categorycomplete_dict initialized external to function)
    
    #we'll also return a dictionary containing any dervied parameters for LSfit such as for
    #subsequent consistent encoding
    
    #Note that categorylist is the list of columns originating from same transformation
    #and we currently exlcude '1010' binary encoded sets from the method
    #a future extension will address 1010 for MLinfill by smoothing after conversion to one-hot encoding 
    #such as for MLinfill
    """

    #grab passed parameters from LSfitparams_dict
    LSfit = LSfitparams_dict[targetcolumn]['LSfit']
    epsilon = LSfitparams_dict[targetcolumn]['epsilon']
    label_categorylist = LSfitparams_dict[targetcolumn]['label_categorylist']
    label_category = LSfitparams_dict[targetcolumn]['label_category']
    
    
    if LSfit is True:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
#         activation_dict = LSfitparams_dict[targetcolumn]['activation_dict']
        LS_dict = LSfitparams_dict[targetcolumn]['LS_dict']
        
        #if LSfit is True we'll apply LS to all columns in categorylist

#         #activation_dcit will track the count of activations for each column in the categorylist
#         activation_dict = {}

#         for column1 in label_categorylist:

#           activations = df[column1].sum()

#           activation_dict.update({column1 : activations})
        
#         #populate this in params dictionary
#         for column1 in label_categorylist:
          
#           LSfitparams_dict.update({column1 : {'activation_dict' : activation_dict}})

#         #LS_dict will be where we keep track of the activation distributions associated with each column
#         #note we'll need activation ratios for each column as a function of activated column for a row
#         LS_dict = {}
#         for column1 in label_categorylist:

#           LS_dict.update({column1 : {}})

#           total_activations = 0

#           for column2 in label_categorylist:

#             if column1 != column2:

#               total_activations += activation_dict[column2]

#           for column2 in label_categorylist:

#             if column1 != column2:

#               LS_dict[column1].update({column2 : activation_dict[column2] / total_activations})
                 
#         #populate this in params dictionary
#         for column1 in label_categorylist:
          
#           LSfitparams_dict[column1].update({'LS_dict' : LS_dict})

        for column1 in label_categorylist:

          for column2 in label_categorylist:

            if column2 != column1:
              
              Smoothing_K = LS_dict[column2][column1]

              df = \
              self.__autowhere(df, column1, df[column2] == 1, (1 - epsilon) * Smoothing_K, specified='replacement')

        for column1 in label_categorylist:

          df = \
          self.__autowhere(df, column1, df[column1]==1, df[column1] * (epsilon), specified='replacement')

          categorycomplete_dict[column1] = True
    
    #if LSfit is not True:
    #else we'll only apply LS to the passed column with assumption of level distribution for K
    else:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        for column1 in label_categorylist:
        
          Smoothing_K = len(label_categorylist)

          #==1 value corresponds to 'bnry' encoding, >1 corresonds to one-hot encodings eg 'text'
          if Smoothing_K > 1:
            Smoothing_K -= 1

          df[column1] = \
          df[column1] * (epsilon) + (1 - df[column1]) * (1 - epsilon) / Smoothing_K

          categorycomplete_dict[column1] = True
  
    return df, categorycomplete_dict

  #__FunctionBlock: Binary categoric consolidations support

  def __BinaryConsolidate(self, df_train, df_test, Binary, postprocess_dict, root, logger):
    """
    Binary categoric consolidation master function applied in automunge
    also used seperately for label consolidations
    """

    #Binary dimensionality reduction goes here
    #test data activation sets not found in train data will return with all zeros
    #Binary will target columns containing categoric integer encodings per MLinfilltype
    #requires all valid entries 
    #(training data performed after infill, so no naninfill, labels performed after family trees)
    
    printstatus = postprocess_dict['printstatus']
    
    if isinstance(Binary, list):
      Binary_orig = self.__autocopy(Binary)
    else:
      Binary_orig = Binary
    
    if isinstance(Binary, list) or Binary in {True, 'retain', 'ordinal', 'ordinalretain', 'onehot', 'onehotretain'}:
      
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin Binary categoric consolidations")
      self.__autoprint('debug', "")
      self.__autoprint('debug', "Before Binary train set column count = ")
      self.__autoprint('debug', df_train.shape[1])
      self.__autoprint('debug', "")

      #_(1)_
      #this is to convert to common form as Binary passed as list of lists
      if not isinstance(Binary, list):
        Binary = [{Binary}] + list(df_train)
      if not isinstance(Binary[0], list):
        Binary = [Binary]
        
      Binary_sublist_count = len(Binary)
        
      Binary_dict = {}
      final_returned_Binary_columns = []
      final_returned_Binary_sets = {'type' : {},
                                    '1010' : [],
                                    'onht' : [],
                                    'ord3' : []}
        
      #_(1)_
      for Binary_sublist_number, Binary_sublist in enumerate(Binary):

        #convert Binary specifications to str (used in cases of integer headers)
        Binary_sublist = self.__parameter_str_convert(Binary_sublist)
        
        Binary_sublist_count_tuple = (Binary_sublist_number, Binary_sublist_count)
        
        #_(2)_
        #check for any first entry signaling deviation from default Binary 1010 encoding with replacement,
        #we'll allow first entry as a set to signal it is a specification i.e. as {<specification>}
        #we'll pass the specification to _Binary_convert as temp_Binary
        #and it will be returned in Binary_dict['Binary_specification']
        if isinstance(Binary_sublist[0], set):
          if str(Binary_sublist[0]) == "{True}":
            temp_Binary = True
            Btype = '1010'
          elif str(Binary_sublist[0]) == "{'retain'}":
            temp_Binary = 'retain'
            Btype = '1010'
          elif str(Binary_sublist[0]) == "{'ordinal'}":
            temp_Binary = 'ordinal'
            Btype = 'ord3'
          elif str(Binary_sublist[0]) == "{'ordinalretain'}":  
            temp_Binary = 'ordinalretain'
            Btype = 'ord3'
          elif str(Binary_sublist[0]) == "{'onehot'}":
            temp_Binary = 'onehot'
            Btype = 'onht'
          elif str(Binary_sublist[0]) == "{'onehotretain'}":
            temp_Binary = 'onehotretain'
            Btype = 'onht'
          del Binary_sublist[0]
        else:
          temp_Binary = True
          Btype = '1010'
        
        #record valresult if Binary specification included columns not found in data
        if 'Binary_columnspresent_valresult' not in logger_dict['validations']:
          logger_dict['validations'].update({'Binary_columnspresent_valresult' : False})
          logger = self.__logger_append(logger)

        if len(set(Binary_sublist) - (set(df_train) | set(postprocess_dict['origcolumn']))) > 0:
          logger_dict['validations']['Binary_columnspresent_valresult' ] = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "Binary specification included at least one column header not found in the data.")
          self.__autoprint('warning', '')

        #for Binary need returned columns
        Binary_sublist = \
        self.__column_convert_support(Binary_sublist, postprocess_dict, convert_to='returned')

        #validate all valid column assignments
        valid_Binary_specified_columns_val = False
        if len(Binary_sublist)== 0:
          valid_Binary_specified_columns_val = True
          self.__autoprint('warning', "Binary specification included no column headers found in the data.")
          self.__autoprint('warning', '')
          logger_dict['validations'].update({'valid_Binary_specified_columns_val' : valid_Binary_specified_columns_val})
          logger = self.__logger_append(logger)

        #_(2)_
        #categoric_column_tuple = (bool_column_list, ordinal_column_list, categoric_column_list)
        bool_column_list = []
        ordinal_column_list = []
        categoric_column_list = []

        for column in Binary_sublist:

          if column in postprocess_dict['column_dict']:

            column_category = postprocess_dict['column_dict'][column]['category']

            if postprocess_dict['process_dict'][column_category]['MLinfilltype'] in \
            ['multirt', 'binary', '1010', 'boolexclude', 'concurrent_act']:

              bool_column_list.append(column)
              categoric_column_list.append(column)
              
            elif postprocess_dict['process_dict'][column_category]['MLinfilltype'] in \
            ['singlct', 'ordlexclude', 'concurrent_ordl']:
              
              ordinal_column_list.append(column)
              categoric_column_list.append(column)
        
        categoric_column_tuple = (bool_column_list, ordinal_column_list, categoric_column_list)
        
        # if printstatus is True:
        self.__autoprint('debug', "Consolidating categoric columns:")
        self.__autoprint('debug', categoric_column_list)
        self.__autoprint('debug', '')
        self.__autoprint('debug', "categoric column count = ")
        self.__autoprint('debug', len(categoric_column_list))
        self.__autoprint('debug', "")

        if len(categoric_column_list) > 0:
          df_train, df_test, Binary_sublist_dict, set_Binary_column_valresult = \
          self.__Binary_convert(df_train, df_test, categoric_column_tuple, temp_Binary, postprocess_dict, Binary_sublist_count_tuple, root)

          returned_Binary_columns = Binary_sublist_dict['returned_Binary_columns']

          #Btype is the correspondign transformation category
          Binary_sublist_dict.update({'Btype' : Btype})

        else:
          set_Binary_column_valresult = False
          Binary_sublist_dict = {'categoric_column_tuple' : ([], [], []),
                                 'ordinal_width_dict' : {},
                                 'column_dict' : {},
                                 'Binary_root' : '',
                                 'returned_Binary_columns' : [],
                                 'Binary_specification' : False}
          if 'Binary_suffixoverlap_results' not in logger_dict['validations']:
            logger_dict['validations'].update({'Binary_suffixoverlap_results' : {}})
            logger = self.__logger_append(logger)
          returned_Binary_columns = []
          
        final_returned_Binary_columns += returned_Binary_columns
        
        final_returned_Binary_sets['type'].update(dict(zip(returned_Binary_columns, [Btype] * len(returned_Binary_columns))))
        if Btype in {'1010', 'onht'}:
          final_returned_Binary_sets[Btype].append(returned_Binary_columns)
        elif Btype in {'ord3'}:
          final_returned_Binary_sets[Btype] = final_returned_Binary_sets[Btype] + returned_Binary_columns
        
        #aggregate suffix overlap validations
        Binary_suffixoverlap_results = {}
        for entry in Binary_sublist_dict['column_dict']:
          Binary_suffixoverlap_results.update(Binary_sublist_dict['column_dict'][entry]['suffixoverlap_results'])

        #log the validation results for Binary_suffixoverlap_results
        if 'Binary_suffixoverlap_results' in logger_dict['validations']:
          logger_dict['validations']['Binary_suffixoverlap_results'].update(Binary_suffixoverlap_results)
        else:
          logger_dict['validations'].update({'Binary_suffixoverlap_results' : Binary_suffixoverlap_results})
          logger = self.__logger_append(logger)

        if 'set_Binary_column_valresult' not in logger_dict['validations'] \
        or 'set_Binary_column_valresult' in logger_dict['validations'] \
        and logger_dict['validations']['set_Binary_column_valresult'] is not True:
          logger_dict['validations'].update({'set_Binary_column_valresult' : set_Binary_column_valresult})
          logger = self.__logger_append(logger)

        #_(2)_
        if Binary_sublist_dict['returned_Binary_columns'] != []:
          Binary_dict.update({Binary_sublist_number : Binary_sublist_dict})
        
        # if printstatus is True:
        self.__autoprint('debug', "Returned Binary columns:")
        self.__autoprint('debug', returned_Binary_columns)
        self.__autoprint('debug', '')
        self.__autoprint('debug', "Returned Binary column count = ")
        self.__autoprint('debug', len(returned_Binary_columns))
        self.__autoprint('debug', "")
          
      #_(1)_
      # if printstatus is True:
      self.__autoprint('debug', "After Binary train set column count = ")
      self.__autoprint('debug', df_train.shape[1])
      self.__autoprint('debug', "")

    else:
      
      Binary_dict = {0 : {'categoric_column_tuple' : ([], [], []),
                          'ordinal_width_dict' : {},
                          'column_dict' : {},
                          'Binary_root' : '',
                          'returned_Binary_columns' : [],
                          'Binary_specification' : False}}

      if 'Binary_suffixoverlap_results' not in logger_dict['validations']:
        logger_dict['validations'].update({'Binary_suffixoverlap_results' : {}})
        logger = self.__logger_append(logger)
      returned_Binary_columns = []
      final_returned_Binary_columns = []
      final_returned_Binary_sets = {'type' : {},
                                    '1010' : [],
                                    'onht' : [],
                                    'ord3' : []}
      
    return df_train, df_test, Binary_dict, postprocess_dict, final_returned_Binary_columns, final_returned_Binary_sets, Binary_orig, Binary

  def __postBinaryConsolidate(self, df_test, meta_Binary_dict, Binary, printstatus):
    """
    Binary categoric consolidation master function applied in postmunge
    also used seperately for label consolidations
    """

    #note if Binary was originally passed to automunge as anything other than False
    #postprocess_dict will record Binary as a list of lists
    #with the Binary type either omitted or recorded as a first entry embedded in set brackets
    #the origional passed Binary list is recorded in postprocess_dict as Binary_orig
    if Binary is not False:
      
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin Binary dimensionality reduction")
      self.__autoprint('debug', "")
      self.__autoprint('debug', "Before transform test set column count = ")
      self.__autoprint('debug', df_test.shape[1])
      self.__autoprint('debug', "")

      #_(1)_
      for key in meta_Binary_dict:

        Binary_dict = meta_Binary_dict[key]
        Binary = meta_Binary_dict[key]['Binary_specification']
      
        # if printstatus is True:
        self.__autoprint('debug', "Consolidating categoric columns:")
        self.__autoprint('debug', Binary_dict['categoric_column_tuple'][2])
        self.__autoprint('debug', '')
        self.__autoprint('debug', "Boolean column count = ")
        self.__autoprint('debug', len(Binary_dict['categoric_column_tuple'][2]))
        self.__autoprint('debug', "")

        df_test = self.__postBinary_convert(df_test, Binary_dict, Binary)

        # if printstatus is True:
        self.__autoprint('debug', "Returned Binary columns:")
        self.__autoprint('debug', Binary_dict['returned_Binary_columns'])
        self.__autoprint('debug', '')
        self.__autoprint('debug', "Returned Binary column count = ")
        self.__autoprint('debug', len(Binary_dict['returned_Binary_columns']))
        self.__autoprint('debug', "")
          
        #_(1)_
        # if printstatus is True:
        self.__autoprint('debug', "After transform test set column count = ")
        self.__autoprint('debug', df_test.shape[1])
        self.__autoprint('debug', "")
          
    return df_test

  def __Binary_convert(self, df_train, df_test, categoric_column_tuple, Binary, postprocess_dict, Binary_sublist_count_tuple, root):
    """
    #Binary_convert takes as input a processed dataframe and a list of boolean encoded columns
    #and applies a dimensionality reduction on the boolean set as a binary encodiong
    
    #returns the set wtih a reduced number of columns and a dictionary containing the
    #parameters of conversion
    
    #assumes columns in bool_column_list have all boolean integer entries

    #note that at this point if user had originally passed Binary as a list 
    #it will already have been converted to boolean or string
    #with the list contents incorporated into bool_column_list

    #null_activation parameter set to 'Binary' to accomodate cases where
    #activation sets in test data don't match activation sets found in train data
    #which will be returned as all zero in inversion
    
    #categoric_column_tuple = (bool_column_list, ordinal_column_list, categoric_column_list)
    #where categoric_column_list encompasses the other two in order of dr_train columns
    """

    #suffixrange is for evaluating overalaps with different numebr of returned columns
    #we'll run an initial check
    #and if suffixrange turns out to be insufficient after having access to number of unique encodings
    #we'll run this again below
    if Binary in {'ordinal', 'ordinalretain'}:
      suffixrange = 1
    else:
      suffixrange = 33
      
    #if only one consolidation root of new column header is 'Binary_'
#     Binary_sublist_count_tuple = (Binary_sublist_number, Binary_sublist_count)
    if Binary_sublist_count_tuple[1] > 1:
      root = root + str(Binary_sublist_count_tuple[0]) + '_'
    else:
      root = root + '_'
    
    #this converts column header root in case of overlap
    Binary_root, set_Binary_column_valresult = \
    self.__set_Binary_column(postprocess_dict, suffixrange, root = root)
    
#     Binary_root = 'Binary_'
    if Binary in {True, 'retain'}:
      Binary_column = Binary_root + '_1010'
    if Binary in {'ordinal', 'ordinalretain'}:
      Binary_column = Binary_root + '_ord3'
    if Binary in {'onehot', 'onehotretain'}:
      Binary_column = Binary_root + '_onht'
    
    #this ensures that the 'Binary' support column header added to dataframe isn't already present
    #(should not be the case since at this point columns have suffix appenders, just here to be consistent)
    Binary_present = \
    self.__df_check_suffixoverlap(df_train, 
                                 Binary_column, 
                                 suffixoverlap_results = {}, 
                                 printstatus = postprocess_dict['printstatus'])
    
    initial_columns = set(df_train)
    
    df_train[Binary_column] = ''
    df_test[Binary_column] = ''
    
    #ordinal_width_dict stores string width for ordinal entries, inversion needs a constant character width
    ordinal_width_dict = {}
  
    for column in categoric_column_tuple[2]:
      
      #boolean integer case
      if column in categoric_column_tuple[0]:

        df_train[Binary_column] = df_train[Binary_column] + df_train[column].astype(int).astype(str)
        df_test[Binary_column] = df_test[Binary_column] + df_test[column].astype(int).astype(str)
        
      #ordinal case
      else:
        
        #get data type from train set (e.g. might be different integer size)
        train_dtype = type(df_train[column].iat[0])
        
        #get max activation from train set
        train_max_activation = df_train[column].max()
        
        #use to derive a string width
        train_max_width = int(np.ceil(np.log10(train_max_activation + 1)))
        
        #add a contingency factor for case where encoding space wasn't fully represented
        #this scenario is not common in our library
        #as used here if ordinal encoding space found in train set was 10^3, this will support encodings up to 10^10
        train_max_width += 7
        
        #store width in ordinal_width_dict
        ordinal_width_dict.update({column : {'train_max_width' : train_max_width,
                                             'train_dtype' : train_dtype}})
        
        #append onto Binary_column with padding to consistent width
        #the slice avoids bug when width contingency factor was inadequate, tradeoff is that will result in false encoding
        df_train[Binary_column] = df_train[Binary_column] + df_train[column].astype(int).astype(str).str.pad(train_max_width, side='left', fillchar='0').str.slice(start=0, stop=train_max_width)
        df_test[Binary_column] = df_test[Binary_column] + df_test[column].astype(int).astype(str).str.pad(train_max_width, side='left', fillchar='0').str.slice(start=0, stop=train_max_width)

    #___
    #this is for a remote edge case to be comprehensive, 
    #in case suffixrange used above was insufficient, now that we have access to nunique we can recalculate
    if Binary in {'onehot', 'onehotretain'}:
      suffixrange2 = df_train[Binary_column].nunique()
    elif Binary in {True, 'retain'}:
      suffixrange2 = int(np.ceil(np.log2(df_train[Binary_column].nunique())))

    if Binary in {'onehot', 'onehotretain', True, 'retain'} and suffixrange2 > suffixrange:
      
      Binary_root2, set_Binary_column_valresult = \
      self.__set_Binary_column(postprocess_dict, suffixrange2, root = root)

      if Binary in {True, 'retain'}:
        Binary_column2 = Binary_root2 + '_1010'
      # if Binary in {'ordinal', 'ordinalretain'}:
      #   Binary_column2 = Binary_root2 + '_ord3'
      if Binary in {'onehot', 'onehotretain'}:
        Binary_column2 = Binary_root2 + '_onht'

      if Binary_column2 != Binary_column:

        df_train.rename(columns = {Binary_column : Binary_column2}, inplace = True)
        df_test.rename(columns = {Binary_column : Binary_column2}, inplace = True)

        Binary_root = Binary_root2
        Binary_column = Binary_column2

        Binary_present2 = \
        self.__df_check_suffixoverlap(df_train, 
                                    Binary_column, 
                                    suffixoverlap_results = {}, 
                                    printstatus = postprocess_dict['printstatus'])

        #this will result in Binary_present recording suffix overlap results for both Binary_column configurations
        Binary_present.update(Binary_present2)

    #___
    
    if Binary in {True, 'retain'}:
      
      df_train, Binary_dict = \
      self._custom_train_1010(df_train, Binary_column, {'null_activation' : 'Binary'})
    
      df_test = \
      self._custom_test_1010(df_test, Binary_column, Binary_dict)
      
    if Binary in {'ordinal', 'ordinalretain'}:
      
      df_train, Binary_dict = \
      self._custom_train_ordl(df_train, Binary_column, {'null_activation' : 'Binary'})
    
      df_test = \
      self._custom_test_ordl(df_test, Binary_column, Binary_dict)

    if Binary in {'onehot', 'onehotretain'}:
      
      df_train, Binary_dict = \
      self._custom_train_onht(df_train, Binary_column, {'null_activation' : 'Binary',
                                                        'suffix_convention' : 'onht'})
    
      df_test = \
      self._custom_test_onht(df_test, Binary_column, Binary_dict)
      
    returned_columns = set(df_train)
    
    newcolumns = returned_columns - initial_columns
    
    #this sorts to match order of returned columns
    newcolumns_sorted = []
    for returned_column in list(df_train):
      if returned_column in newcolumns:
        newcolumns_sorted.append(returned_column)
    
    suffixoverlap_results = self.__df_check_suffixoverlap(initial_columns, \
                                                         newcolumns_sorted, \
                                                         suffixoverlap_results = Binary_present, \
                                                         printstatus = postprocess_dict['printstatus'])
      
    Binary_dict.update({'categoric_column_tuple' : categoric_column_tuple})
    Binary_dict.update({'ordinal_width_dict' : ordinal_width_dict})
    Binary_dict.update({'Binary_root' : Binary_root})
    
    #(this is a bit of a hack to carry suffix overlap result for 'Binary' to final report)
    Binary_dict.update({'column_dict':{}})
    for newcolumn in newcolumns_sorted:
      Binary_dict['column_dict'].update({newcolumn:{'suffixoverlap_results':suffixoverlap_results}})

    Binary_dict.update({'returned_Binary_columns' : newcolumns_sorted,
                        'Binary_specification' : Binary})
    
    #we won't delete the origin columns if Binary passed as 'retain'
    #(such that the binary encoding is a supplement instead of a replacement)
    if Binary not in {'retain', 'ordinalretain', 'onehotretain'}:

      for column in categoric_column_tuple[2]:

        del df_train[column]
        del df_test[column]

    return df_train, df_test, Binary_dict, set_Binary_column_valresult

  def __postBinary_convert(self, df_test, Binary_dict, Binary):
    """
    #Binary_convert takes as input a processed dataframe and a list of boolean encoded columns
    #and applies a dimensionality reduction on the boolean set as a binary encodiong
    
    #returns the set wtih a reduced number of columns and a dictionary containing the
    #parameters of conversion
    
    #note that infill has already been applied on these columns so no need for infill
    #on train set (but yes on test set)
    """

    Binary_root = Binary_dict['Binary_root']

    if Binary in {True, 'retain'}:
      Binary_column = Binary_root + '_1010'
    if Binary in {'ordinal', 'ordinalretain'}:
      Binary_column = Binary_root + '_ord3'
    if Binary in {'onehot', 'onehotretain'}:
      Binary_column = Binary_root + '_onht'
    
    categoric_column_tuple = Binary_dict['categoric_column_tuple']
    
    df_test[Binary_column] = ''
    
    for column in categoric_column_tuple[2]:
      
      #boolean integer case
      if column in categoric_column_tuple[0]:

        df_test[Binary_column] = df_test[Binary_column] + df_test[column].astype(int).astype(str)
        
      #ordinal case
      else:
        
        #get train_max_width from train set
        train_max_width = Binary_dict['ordinal_width_dict'][column]['train_max_width']
        
        #append onto Binary_column with padding to consistent width
        df_test[Binary_column] = df_test[Binary_column] + df_test[column].astype(int).astype(str).str.pad(train_max_width, side='left', fillchar='0').str.slice(start=0, stop=train_max_width)

    if Binary in {True, 'retain'}:

      df_test = \
      self._custom_test_1010(df_test, Binary_column, Binary_dict)
      
    if Binary in {'ordinal', 'ordinalretain'}:

      df_test = \
      self._custom_test_ordl(df_test, Binary_column, Binary_dict)

    if Binary in {'onehot', 'onehotretain'}:

      df_test = \
      self._custom_test_onht(df_test, Binary_column, Binary_dict)
    
    #we won't delete the origin columns if Binary passed as 'retain'
    #(such that the binary encoding is a supplement instead of a replacement)
    if Binary not in {'retain', 'ordinalretain', 'onehotretain'}:
      
      for column in categoric_column_tuple[2]:

        del df_test[column]
    
    return df_test

  def __masterBinaryinvert(self, df_test, inversion, inversion_orig, meta_Binary_dict, postprocess_dict, printstatus, logger):
    """
    accepts inversion as a list
    can be applied to df_test including test data or labels data, (inversion receives either as df_test)
    if meta_Binary_dict is empty no inversion performed
    returns an adjusted inversion list which removes Binary columns and replaces with their associated inputs
    if Binary was a retain consolidation simply removes the redundant consolidations
    """

    Binary_partialinversion_valresult = False

    for key in meta_Binary_dict:
      Binary_dict = meta_Binary_dict[key]
      Binary_specification = Binary_dict['Binary_specification']

      if Binary_specification in {'retain', 'ordinalretain', 'onehotretain'}:
        #we'll have convention that if Binary didn't replace columns 
        #partial inversion only available for Binary source columns
        #since any inversion from the Binary form would be redundant
        for entry in Binary_dict['column_dict']:
          if entry in inversion:
            inversion.remove(entry)

        #if inversion was originally passed as 'test' or 'labels'
        #then strike the Binary columns to be consistent with original input form
        #(if inversion was originally passed as list that means inversion only applied on subset of columns)
        if inversion_orig in {'test', 'labels'}:
          for entry in Binary_dict['column_dict']:
            if entry in df_test:
              del df_test[entry]

      if Binary_specification in {True, 'ordinal', 'onehot'}:
        Binary_inversion_marker = False
        if (set(Binary_dict['column_dict'])).issubset(set(inversion)):
          Binary_inversion_marker = True
          for entry in Binary_dict['column_dict']:
            inversion.remove(entry)
          inversion += Binary_dict['categoric_column_tuple'][2]

        #if inversion was passed as a list subset of returned columns
        #Binary_inversion_marker won't be activated if Binary columns aren"t in that list 

        elif len((set(Binary_dict['column_dict'])) & set(inversion)) == 0:
          pass

        #or is an error channel when only a partial set of Binary columns are included
        #where by partial set am referring to partial with respect to this specified Binary subaggregation

        else:

          Binary_partialinversion_valresult = True

          # if printstatus != 'silent':
          self.__autoprint('warning', "error: partial inversion lists only supported for columns returned from Binary")
          self.__autoprint('warning', "when entire set of Binary columns are included in the inversion list")

        if Binary_inversion_marker is True:

          df_test = self.__meta_inverseprocess_Binary(df_test, Binary_dict, postprocess_dict)

          inversion += Binary_dict['categoric_column_tuple'][2]

          # if printstatus is True:
          self.__autoprint('debug', "Recovered Binary columns:")
          self.__autoprint('debug', Binary_dict['categoric_column_tuple'][2])
          self.__autoprint('debug', '')
    logger_dict['validations'].update({'Binary_partialinversion_valresult' : Binary_partialinversion_valresult})
    logger = self.__logger_append(logger)

    return df_test, inversion

  def __meta_inverseprocess_Binary(self, df, Binary_dict, postprocess_dict):
    """
    #converts string boolean integers returned from inverseprocess_Binary to int
    #cleans up columns
    """

    Binary_returned_columns = Binary_dict['returned_Binary_columns']
    Binary = Binary_dict['Binary_specification']
    
    categoric_column_tuple = Binary_dict['categoric_column_tuple']
  
    df, inputcolumn = \
    self.__inverseprocess_Binary(df, Binary_returned_columns, Binary_dict, Binary)

    i=0
    for Binary_source_column in categoric_column_tuple[2]:
      
      #boolean integer case
      if Binary_source_column in categoric_column_tuple[0]:

        df[Binary_source_column] = df[inputcolumn].str.slice(start=i, stop=i+1)

        df[Binary_source_column] = df[Binary_source_column].astype(np.int8)
        i += 1
        
      #ordinal case
      else:
        
        #get train_dtype from train set
        train_dtype = Binary_dict['ordinal_width_dict'][Binary_source_column]['train_dtype']
        
        #get train_max_width from train set
        train_max_width = Binary_dict['ordinal_width_dict'][Binary_source_column]['train_max_width']
        
        #append onto Binary_column with padding to consistent width
        df[Binary_source_column] = df[inputcolumn].str.slice(start=i, stop=i+train_max_width)
        
        df[Binary_source_column] = df[Binary_source_column].astype(train_dtype)
        
        i += train_max_width

    #we'll have convention that the Binary columns not returned from inversion
    for Binary_returned_column in Binary_returned_columns:
      del df[Binary_returned_column]

    if inputcolumn not in Binary_returned_columns:
      del df[inputcolumn]
    
    return df
  
  def __inverseprocess_Binary(self, df, Binary_columns, Binary_dict, Binary):

    Binary_root = Binary_dict['Binary_root']
    
    if Binary in {True, 'retain'}:
      
      inputcolumn = Binary_root + '_1010'
    
      df= \
      self._custom_inversion_1010(df, Binary_columns, inputcolumn, Binary_dict)
      
    if Binary in {'ordinal', 'ordinalretain'}:
      
      inputcolumn = Binary_root + '_ord3'
      
      df = \
      self._custom_inversion_ordl(df, Binary_columns, inputcolumn, Binary_dict)

    if Binary in {'onehot', 'onehotretain'}:
      
      inputcolumn = Binary_root + '_onht'
      
      df = \
      self._custom_inversion_onht(df, Binary_columns, inputcolumn, Binary_dict)
      
    return df, inputcolumn

  #__FunctionBlock: data type management
  
  def __floatprecision_transform(self, df, columnkeylist, floatprecision):
    """
    #floatprecision is a parameter user passed to automunge
    #allowable values are 16/32/64
    #if 64 do nothing (we'll assume our transform functions default to 64)
    #if 16 or 32 then check each column in df for columnkeylist and if
    #float convert to this precision
    """

    if isinstance(columnkeylist, str):
      columnkeylist = [columnkeylist]
    
    #if floatprecision in {16, 32, 64}:
    if floatprecision in {16, 32}:
      
      for columnkey in columnkeylist:
        
        if pd.api.types.is_float_dtype(df[columnkey]):
          
          if floatprecision == 32:
            df[columnkey] = df[columnkey].astype(np.float32)
            
          elif floatprecision == 16:
            df[columnkey] = df[columnkey].astype(np.float16)
            
#           elif floatprecision == 64:
#             df[columnkey] = df[columnkey].astype(np.float64)

    return df
  
  def __convert_to_nan(self, df, column, category, postprocess_dict, convert_to_nan_list):
    """
    #converts all values from convert_to_nan_list in a dataframe to np.nan
    #the np.inf conversion is similar to pandas pd.options.mode.use_inf_as_na = True
    #except that it works since use_inf_as_na results in inf converted to NaN == NaN
    
    #note that if the column is received with bytes entries 
    #the conversion to nan returns as a bytes representation i.e. b'np.nan'
    #this has something to do with the pandas '|S9' dtype
    """

    #don't apply to totalexclude MLinfilltype
    if postprocess_dict['process_dict'][category]['NArowtype'] not in {'totalexclude'}:
      
      isna_already_performed = False

      for entry in convert_to_nan_list:
        
        if (entry == None or entry != entry) and isna_already_performed is False:
          #this also consolidates any diverse nan representations to common form (e.g. float("NaN") vs np.nan)
          df.loc[df[column].isna(), column] = np.nan
          isna_already_performed = True

        else:
          if entry in {np.inf, -np.inf}:
            #the to_numeric resolves a printout associated with elementwise comparison when comapring to bytes dtype
            df.loc[pd.to_numeric(df[column], errors='coerce') == entry, column] = np.nan
          else:
            df.loc[df[column] == entry, column] = np.nan

    return df

  def __assignnan_convert(self, df, column, category, assignnan, postprocess_dict):
    """
    #assignnan is automunge(.) parameter that allows user to designate values that will
    #be given infill treatment for a given root category or source column
    #such as to supplement processdict NArowtype entries with values that may be 
    #special for a data set
    #as an example, in some cases datasets may not be received with NaN for infill, 
    #and may instead be a designated value such as a number or string such as 'unknown'
    #assignnan_convert addresses this scenario by simply converting those designations to nan

    #where values are passed in automunge(.) parameter assignnan
    #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}
    
    #where in case of specification redundancy column designation takes precedence
    #and where category is reffering to the root category associated with a column
    
    #some further options for nan injections are documented in the assignnan_inject
    #including stochastic and range injections
    #which is called at the conclusion of this one
    """

    # if category in postprocess_dict['process_dict']:
      
    nanpoints = []

    cat_process = False

    if 'categories' in assignnan:

      if category in assignnan['categories']:

        cat_process = True

        if 'columns' in assignnan:

          if column in assignnan['columns']:

            cat_process = False

        if cat_process is True:

          nanpoints = assignnan['categories'][category]

    if cat_process is False:

      if 'columns' in assignnan:

        if column in assignnan['columns']:

          nanpoints = assignnan['columns'][column]
          
    #we'll have convention that for NArowtype=='totalexclude'
    #global assignnan assignments don't apply and must be assigned explicitly
    #either in assignnan categories or columns entries
    if postprocess_dict['process_dict'][category]['NArowtype'] not in {'totalexclude'}:

      if 'global' in assignnan:

        nanpoints += assignnan['global']
          
    #great we've got our designated infill values, now just convert to nan
    for entry in nanpoints:
      
      df.loc[df[column] == entry, column] = np.nan
      
    #finally some further options for nan injections are supported 
    #including stochastic and range injections
    #as documented further in assignnan_inject function
    if 'injections' in assignnan:

      #randomseed is accessible in the postprocess_dict
      #where if this function is being applied in postmunge we'll give precedence to postmunge randomseed
      if 'postmunge_randomseed' in postprocess_dict:
        random = postprocess_dict['postmunge_randomseed']
      else:
        random = postprocess_dict['randomseed']

      df = self.__assignnan_inject(df, column, assignnan, random, postprocess_dict['printstatus'])
    
    return df

  def __assignnan_inject(self, df, column, assignnan, randomseed, printstatus):
    """
    #allows custom range or stochastic nan injections to distinct source columns
    #assignnan now accepts entries as
    
    {'injections' : {'(column)' : {'inject_ratio' : (float), \
                                   'range' : {'ratio'  : (float), \
                                              'ranges' : [[min1, max1], [min2, max2]]}, \
                                   'minmax_range' : {'ratio'  : (float), \
                                                     'ranges' : [[min1, max1], [min2, max2]]}, \
                                   'entries' : ['(entry1)', '(entry2)'], \
                                   'entry_ratio' : {'(entry1)' : float, \
                                                    '(entry2)' : float}
                                  }
                    }
    }
    
    #where injections may be specified for each source column passed to automunge(.)
    #- inject_ratio is uniform randomly injected nan points to ratio of entries
    #- range is injection within a specified range based on ratio float defaulting to 1.0
    #- minmax_range is injection within scaled range (accepting floats 0-1 based on received 
    #column max and min (returned column is not scaled
    #- entries are full replacement of specific entries to a categoric set
    #- entry_ratio are partial injection to specific entries to a categoric set, 
    #per specified float ratio 
    
    #the purposes of these options are to support some experiments on missing data infill
    
    #currently assignnan_inject only supports dataframes with range integer index
    #this is deemed acceptable since this functionality not intended for mainstream use
    """

    if 'injections' in assignnan:
      
      #for injections functinoality we'll reset index to range integer
      #non-range indexes will have already been copied into ID set
      if type(df.index) != pd.RangeIndex:
        df = df.reset_index(drop=True)
      
      columnkey = column
      if columnkey in assignnan['injections']:
        if columnkey in df:
          for actionkey in assignnan['injections'][columnkey]:
              
            if actionkey == 'inject_ratio':
              #inject_ratio is uniform randomly injected nan points to ratio of entries
              ratio = assignnan['injections'][columnkey][actionkey]
              index = list(pd.DataFrame(df.index).sample(frac=ratio, replace=False).to_numpy().ravel())
              df.loc[index, columnkey] = np.nan
              
            elif actionkey == 'range':
              #range inserts nan in designated ranges of numeric set
              #accepts parameter for injection ratio, we'll actually use for our method 1-ratio
              if 'ratio' in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey]['ratio']
              else:
                ratio = 1 - 1
              if 'ranges' in assignnan['injections'][columnkey][actionkey]:
                #ranges is a list of list, each sub list with two entries for min and max inclusive
                for range_list in assignnan['injections'][columnkey][actionkey]['ranges']:
                  rangemin = range_list[0]
                  rangemax = range_list[-1]
                  #we'll create a support dataframe to populate masks
                  df_mask = pd.DataFrame()

                  df_mask = \
                  self.__autowhere(df_mask, 'lessthan', df[columnkey]<=rangemax, 1, specified='replacement')
                  
                  df_mask = \
                  self.__autowhere(df_mask, 'greaterthan', df[columnkey]>=rangemin, 1, specified='replacement')
                  
                  #this populates mask with 1's for candidates for injection, 0's elsewhere
                  df_mask['mask'] = df_mask['lessthan'] * df_mask['greaterthan']
                  #this populates mask with nan for injection candidates and 1 elsewhere
                  df_mask = \
                  self.__autowhere(df_mask, 'mask', df_mask['mask'] == 1, np.nan, 1)

                  if ratio > 0:
                    #this get's index list of mask nan entries for use with .loc
                    index = list(pd.DataFrame(df_mask[df_mask['mask'] != df_mask['mask']].index).sample(frac=ratio, replace=False).to_numpy().ravel())
                    #this reverts some of the nans if ratio was passed as <1
                    df_mask.loc[index, 'mask'] = 1
                  #now inject the nan to the target column in df
                  df[columnkey] = df[columnkey] * df_mask['mask']
                  del df_mask
                  
            elif actionkey == 'minmax_range':
              #minmax_range is similar to range but the max and min are received in range 0-1
              #and applied corresponding to a minmax scaling of received set
              #(set is returned without scaling)
              #range inserts nan in designated ranges of numeric set
              #accepts parameter for injection ratio, we'll actually use for our method 1-ratio
              if 'ratio' in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey]['ratio']
              else:
                ratio = 1 - 1
              if 'ranges' in assignnan['injections'][columnkey][actionkey]:
                #this might return nan if target column does not contain numeric entries
                setmin = df[columnkey].min()
                setmax = df[columnkey].max()
                #ranges is a list of list, each sub list with two entries for min and max inclusive
                for range_list in assignnan['injections'][columnkey][actionkey]['ranges']:
                  rangemin = range_list[0]
                  rangemax = range_list[-1]
                  #now scale these ranges based on inverse of min-max scaling formula
                  #minmax = (xi - min)/(max - min)
                  # => xi = minmax * (max - min) + min
                  rangemin = rangemin * (setmax - setmin) + setmin
                  rangemax = rangemax * (setmax - setmin) + setmin
                  #the rest is comparable to unscaled range option
                  #we'll create a support dataframe to populate masks
                  df_mask = pd.DataFrame()

                  df_mask = \
                  self.__autowhere(df_mask, 'lessthan', df[columnkey]<=rangemax, 1, specified='replacement')

                  df_mask = \
                  self.__autowhere(df_mask, 'greaterthan', df[columnkey]>=rangemin, 1, specified='replacement')

                  #this populates mask with 1's for candidates for injection, 0's elsewhere
                  df_mask['mask'] = df_mask['lessthan'] * df_mask['greaterthan']
                  #this populates mask with nan for injection candidates and 1 elsewhere
                  df_mask = \
                  self.__autowhere(df_mask, 'mask', df_mask['mask'] == 1, np.nan, 1)

                  if ratio > 0:
                    #this get's index list of mask nan entries for use with .loc
                    index = list(pd.DataFrame(df_mask[df_mask['mask'] != df_mask['mask']].index).sample(frac=ratio, replace=False).to_numpy().ravel())
                    #this reverts some of the nans if ratio was passed as <1
                    df_mask.loc[index, 'mask'] = 1
                  #now inject the nan to the target column in df
                  df[columnkey] = df[columnkey] * df_mask['mask']
                  del df_mask
                  
            elif actionkey == 'entries':
              #entries are full replacement of specific entries to a categoric set
              for targetentry in assignnan['injections'][columnkey][actionkey]:
                df = \
                self.__autowhere(df, columnkey, df[columnkey] == targetentry, np.nan, specified='replacement')
                
            elif actionkey == 'entry_ratio':
              #entry_ratio are partial injection to specific entries to a categoric set, 
              #per specified float ratio in range 0-1
              for targetentry in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey][targetentry]
                df_mask = pd.DataFrame()
                #here we'll use convention that 1 is a target for injection 0 remains static
                df_mask = \
                self.__autowhere(df_mask, 'mask', df[columnkey] == targetentry, 1, specified='replacement')

                if ratio > 0:
                  #this get's index list of mask 1 entries for use with .loc
                  index = list(pd.DataFrame(df_mask[df_mask['mask'] == 1].index).sample(frac=ratio, replace=False).to_numpy().ravel())
                  #this reverts some of the 1's if ratio was passed as <1
                  df_mask.loc[index, 'mask'] = 0
                df = \
                self.__autowhere(df, columnkey, df_mask['mask'] == 1, np.nan, specified='replacement')
                
    return df

  #__FunctionBlock: data set partitioning and shuffling

  def __df_split(self, df, ratio, shuffle_param, randomseed):
    """
    #performs a split of passed dataframe df
    #based on proportions of ratio where 0<ratio<1
    #bool shuffle_param False means rows taken from bottom of set sequentially
    #bool shuffle_param True means randomly sampled rows 
    #per seeding of randomseed
    #with the validation set df2 returned shuffled
    #and the train set df1 not shuffled, just removed rows present in df2
    #(if run on two df's with same number of rows, will return consistent partitioning)
    #returns two dataframes df and df2, where df is training data and df2 is validaiton set
    """

    if ratio > 0 and ratio < 1:
    
      if shuffle_param is False:

        start = int(df.shape[0] * (1-ratio))
        end = df.shape[0]
        
        df2 = df[start:end]
        df = df[0:start]

      elif shuffle_param is True:
        
        #these rows will be shuffled
        df2 = df.sample(frac=ratio, random_state=randomseed)
        
        #these rows won't be shuffled, this will just remove rows present in df2
        df = df.drop(df2.index)

    else:

      #df = df
      df2 = pd.DataFrame()

    return df, df2

  def __df_split_specified(self, df, ratio_tuple, shuffle_param, randomseed):
    """
    #performs a split of passed dataframe df
    #inspired by the _df_split for split based on ratio
    #this funciton differs in that it is based on specified partitions
    #as may be designated by user passing ratio as a tuple of (start, end)
    #where start is a float between 0-1
    #and end is a float between 0-1
    #and start < end
    #and each float is used to specify a boundary of the split by translating to integers
    #i.e.
    #start = int(df.shape[0] * (start))
    #end = int(df.shape[0] * (end))
    #such that the returned dataframe will have training data in rows 0:start and end:df.shape[0]
    #and the returned validation set will have rows of start:end
    #note that if shuffle_param is activated the rows of the validation set will be shuffled
    
    #df1 is training data
    #df2 is validation data
    
    #based on proportions of ratio where 0<ratio<1
    #bool shuffle_param False means rows taken from bottom of set sequentially
    #bool shuffle_param True means randomly sampled rows 
    #per seeding of randomseed
    #with the validation set df2 returned shuffled
    #and the train set df1 not shuffled, just removed rows present in df2
    #(if run on two df's with same number of rows, will return consistent partitioning)
    #returns two dataframes df1 and df2
    """

    if ratio_tuple[0] >= 0 and ratio_tuple[0] < 1 \
    and ratio_tuple[1] > ratio_tuple[0] and ratio_tuple[1] <= 1:
    
      start = ratio_tuple[0]
      start = int(df.shape[0] * (start))
      
      end = ratio_tuple[1]
      end = int(df.shape[0] * (end))
        
      df2 = df[start:end]
      df = df.drop(df2.index)
        
      #if shuffleparam is activated, we'll shuffle rows in df2 (df1 shuffled later in automunge)
      if shuffle_param is True:
        
        #these rows will be shuffled
        df2 = df2.sample(frac=1, random_state=randomseed)
        
    else:
      
      #df = df
      df2 = pd.DataFrame()

    return df, df2
  
  def __df_shuffle(self, df, randomseed, axis=0):
    """
    #Shuffles the rows of a dataframe
    #per seeding of randomseed
    #pass axis = 1 to shuffle order of columns
    """

    df = df.sample(frac=1, random_state=randomseed, axis=axis)
    
    return df  
  
  def __df_shuffle_series(self, df, column, randomseed):
    """
    #Shuffles single column in a dataframe
    """

    df_temp = df[column].copy()
    df_temp = self.__df_shuffle(df_temp, randomseed)
    df[column] = df_temp.to_numpy()
    
    del df_temp
    
    return df

  def __dupl_rows_consolidate(self, df, df_consol_id, df_consol_labels):
    """
    #consolidates duplicate rows in a dataframe
    #in other words if duplicate rows present only returns one of duplicates
    """

    mask = pd.DataFrame(df.duplicated())
    mask = pd.Series(mask[list(mask)[0]].astype(int) - 1).abs().astype(bool)

    if df_consol_id.shape[0] == df.shape[0]:
      df_consol_id = df_consol_id.iloc[mask.to_numpy()]
    
    if df_consol_labels.shape[0] == df.shape[0]:
      df_consol_labels = df_consol_labels.iloc[mask.to_numpy()]

    df = df.iloc[mask.to_numpy()]
    
    return df, df_consol_id, df_consol_labels

  #__FunctionBlock: populate returned reports

  def __populate_columntype_report(self, postprocess_dict, target_columns):
    """
    #populates a report for types of returned columns
    #such as to distingiush between continous, categoric, categoric sets, etc
    """

    columntype_report = {'continuous' : [], \
                         'integer'    : [], \
                         'boolean' : [], \
                         'ordinal' : [], \
                         'onehot' : [], \
                         'onehot_sets' : [], \
                         'binary' : [], \
                         'binary_sets' : [], \
                         'passthrough' : [],
                         'all_categoric':[],
                         'all_numeric':[]}
    
    populated_columns = []

    #_populate_columntype_report may be called prior to applying Binary
    if 'returned_Binary_sets' in postprocess_dict:
      #this supports Binary returned columns
      Binary_sets_log = {'trainlog' : self.__autocopy(postprocess_dict['returned_Binary_sets']),
                        'labellog' : self.__autocopy(postprocess_dict['final_returned_labelBinary_sets'])}
    else:
      Binary_sets_log = {'trainlog' : [],
                        'labellog' : []}
    
    #for column in postprocess_dict['finalcolumns_train']:
    for column in target_columns:
      
      if column not in populated_columns:
        
        if 'returned_PCA_columns' in postprocess_dict and \
        column in postprocess_dict['returned_PCA_columns']:
          
          #add to numeric
          columntype_report['continuous'].append(column)
          
          populated_columns.append(column)

        elif 'returned_Binary_columns' in postprocess_dict and \
        (column in postprocess_dict['returned_Binary_columns'] \
        or column in postprocess_dict['final_returned_labelBinary_columns']):

          if column in postprocess_dict['returned_Binary_columns']:
            Binary_sets_log_key = 'trainlog'
          else:
            Binary_sets_log_key = 'labellog'

          if column in Binary_sets_log[Binary_sets_log_key]['type']:
            Btype = Binary_sets_log[Binary_sets_log_key]['type'][column]

            #note the Binary treatment here operates on assumption that
            #if one column from a Binary set is included, they all are
            #which is consistent with implementation
            
            if Btype == '1010':
              
              for returned_1010_set in Binary_sets_log[Binary_sets_log_key]['1010']:
                
                if column in returned_1010_set:
                  
                  columntype_report['binary_sets'].append(returned_1010_set)

                  columntype_report['binary'] = \
                  columntype_report['binary'] + returned_1010_set

                  populated_columns = \
                  populated_columns + returned_1010_set
                  
                  #now reset Binary_sets_log marker so can circumvent redundant searches
                  for returned_1010_set_entry in returned_1010_set:
                    del Binary_sets_log[Binary_sets_log_key]['type'][returned_1010_set_entry]
                    # Binary_sets_log[Binary_sets_log_key]['type'][returned_1010_set_entry] = False
            
            if Btype == 'onht':
              
              for returned_onht_set in Binary_sets_log[Binary_sets_log_key]['onht']:
                
                if column in returned_onht_set:
                  
                  columntype_report['onehot_sets'].append(returned_onht_set)
                  
                  columntype_report['onehot'] = \
                  columntype_report['onehot'] + returned_onht_set
                  
                  populated_columns = \
                  populated_columns + returned_onht_set
                  
                  #now reset Binary_sets_log marker so can circumvent redundant searches
                  for returned_onht_set_entry in returned_onht_set:
                    del Binary_sets_log[Binary_sets_log_key]['type'][returned_onht_set_entry]
                    # Binary_sets_log[Binary_sets_log_key]['type'][returned_onht_set_entry] = False
              
            if Btype == 'ord3':
              
              columntype_report['ordinal'].append(column)
              populated_columns.append(column)
              
              Binary_sets_log[Binary_sets_log_key]['type'][column] = False
          
        elif 'excl_suffix' in postprocess_dict and \
        'excl_columns_without_suffix' in postprocess_dict and \
        postprocess_dict['excl_suffix'] is False and \
        column in postprocess_dict['excl_columns_without_suffix']:
            
          #add column to passthrough
          columntype_report['passthrough'].append(column)

          populated_columns.append(column)
            
        elif column in postprocess_dict['column_dict']:
          
          MLinfilltype = \
          postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype']
          
          if MLinfilltype in {'numeric', 'concurrent_nmbr', 'exclude'}:
            
            #add to numeric
            columntype_report['continuous'].append(column)
            
            populated_columns.append(column)

          elif MLinfilltype in {'integer'}:
            
            #add to numeric
            columntype_report['integer'].append(column)
            
            populated_columns.append(column)
            
          elif MLinfilltype in {'binary', 'concurrent_act', 'boolexclude'}:
            
            #add to boolean
            columntype_report['boolean'].append(column)
            
            populated_columns.append(column)
          
          elif MLinfilltype in {'singlct', 'ordlexclude', 'concurrent_ordl'}:
            
            #add to ordinal
            columntype_report['ordinal'].append(column)
            
            populated_columns.append(column)
            
          elif MLinfilltype in {'multirt'}:
            
            #initialize onehot_sets
            columntype_report['onehot_sets'].append([])
            
            for entry in postprocess_dict['column_dict'][column]['categorylist']:
              
              columntype_report['onehot_sets'][-1] = \
              columntype_report['onehot_sets'][-1] + [entry]
              
              #add to onehot
              columntype_report['onehot'].append(entry)
              
              populated_columns.append(entry)
              
          elif MLinfilltype in {'1010'}:
              
            #initialize binary_sets
            columntype_report['binary_sets'].append([])

            for entry in postprocess_dict['column_dict'][column]['categorylist']:

              columntype_report['binary_sets'][-1] = \
              columntype_report['binary_sets'][-1] + [entry]

              #add to binary
              columntype_report['binary'].append(entry)

              populated_columns.append(entry)
              
          elif MLinfilltype in {'totalexclude'}:
            
            #add to ordinal
            columntype_report['passthrough'].append(column)
            
            populated_columns.append(column)

    #now aggregate all numeric and seperately all categoric columns into single lists

    columntype_report['all_numeric'] = \
    columntype_report['continuous'] + columntype_report['integer']

    columntype_report['all_categoric'] = \
    columntype_report['boolean'] + columntype_report['ordinal'] + \
    columntype_report['onehot'] + columntype_report['binary']
            
    return columntype_report

  def __populate_column_map_report(self, postprocess_dict):
    """
    #populates a report that maps input columns to their corresponding set of output columns
    #as a dictionary with keys of input columns and values of a list of associated output columns
    #includes label columns
    #returned in postprocess_dict as postprocess_dict['column_map']
    #excludes any returned columns that were part of a dimensionality reduction consolidation
    #those are available in postprocess_dict as returned_Binary_columns and returned_PCA_columns
    """

    column_map = {}
    
    allfinalcolumns = postprocess_dict['finalcolumns_train'] + postprocess_dict['finalcolumns_labels']
    
    for finalcolumn in allfinalcolumns:
      
      finalcolumn2 = finalcolumn
      
      if finalcolumn2 not in postprocess_dict['returned_PCA_columns'] and \
      finalcolumn2 not in postprocess_dict['returned_Binary_columns'] and \
      finalcolumn2 not in postprocess_dict['final_returned_labelBinary_columns']:
      
        if postprocess_dict['excl_suffix'] is False:

          if finalcolumn in postprocess_dict['excl_columns_without_suffix']:
            
            #adds the '_excl' suffix to access columnkeylist
            finalcolumn2 = postprocess_dict['excl_suffix_conversion_dict'][finalcolumn2]

        columnkeylist = \
        postprocess_dict['origcolumn'][postprocess_dict['column_dict'][finalcolumn2]['origcolumn']]['columnkeylist']
  
        #special case for excl category to remove suffix to align with returned form
        if postprocess_dict['excl_suffix'] is False:
          columnkeylist_copy = columnkeylist.copy()
          for columnkeylistentry in columnkeylist_copy:
            if columnkeylistentry in postprocess_dict['excl_columns_with_suffix']:
              columnkeylist.remove(columnkeylistentry)
#               columnkeylist.append(columnkeylistentry[:-5])
              columnkeylist.insert(columnkeylist_copy.index(columnkeylistentry),columnkeylistentry[:-5])

        #if entry was already populated for multiple returned columns it overwrites it with same info
        column_map.update({postprocess_dict['column_dict'][finalcolumn2]['origcolumn'] : columnkeylist})

    #this handles columns that did not return any sets 
    # (such as for null category or source columns whose returned sets were consolidated as part of dimensionality reduction)
    for origcolumn in postprocess_dict['origcolumn']:
      if origcolumn not in column_map:
        column_map.update({origcolumn : []})
      
    return column_map

  #__FunctionBlock: various automunge utility functions

  def __list_replace(self, targetlist, conversion_dict):
    """
    targetlist is a list (for our use will be string entries, but this should work for other types too)
    conversion_dict is a dictionary mapping entries in targetlist to a desired substitution
    e.g. conversion_dict = {target_for_substitution : substitution}
    conversion_dict may have keys not found in targetlist
    performs substitution inplace with received list, so no return value
    assumes no redundant entries in targetlist which will be valid for our use

    may be used to translate lists of column headers between excl suffix conventions
    e.g. to remove excl suffix can apply
    self.__list_replace(targetlist, postprocess_dict['excl_suffix_inversion_dict'])
    or to add excl suffix can apply
    self.__list_replace(targetlist, postprocess_dict['excl_suffix_conversion_dict'])
    """

    for target_for_substitution in conversion_dict:
      if target_for_substitution in targetlist:
        targetlist[targetlist.index(target_for_substitution)] = conversion_dict[target_for_substitution]
    return

  def __list_sorting(self, ordered_list, mixed_set, placeholder=float("NaN")):
    """
    Function to sort the order of entries in a mixed set
    Based on the order of entries as are found in a corresponding ordered list
    Where the mixed set may have extra or fewer entries than are found in the ordered list
    And where extra entries from the mixed set
    Are sorted alphabetically and appended as final entries to the returned list
    
    accepts parameters ordered_list and mixed_set
    and returns sorted_list

    note that this relies on placeholder NaN entry, so if NaN is present in mixed_set it is not returned
    which is acceptable in context of our use for categoric encodings
    an alternate placeholder entry may be specified with reserved parameter if desired
    """

    #below assumes mixed_set is a set, this adds support for mixed_set received as a list
    if isinstance(mixed_set, list):
      mixed_set = set(mixed_set)
    
    #this populates a dicitonary mapping ordered_list entries to their index number
    order_key_dict = {}
    i=0
    for entry in ordered_list:
      order_key_dict.update({entry : i})
      i+=1
      
    #initalize the returned list as all nan's
    sorted_list = [placeholder] * len(ordered_list)
    
    #populate ordered_list entries that are present in mixed_set
    for entry in mixed_set:
      if entry in order_key_dict:
        sorted_list[order_key_dict[entry]] = entry
      
    #now remove any nan placeholders
    if placeholder != placeholder:
      sorted_list = [x for x in sorted_list if x==x]
    else:
      sorted_list = [x for x in sorted_list if x!=placeholder]
    
    #identify any extra entries that were present in mixed_set and not in ordered_list
    #and convert to an ordered list
    extras_list = sorted(mixed_set - set(ordered_list), key=str)
    
    #now append the extra entries
    sorted_list = sorted_list + extras_list
    
    return sorted_list

  def __autowhere(self, df, column, condition, replacement=True, alternative=False, specified='replacementalternative'):
    """
    Intended to serve as a compromise between np.where and pd.where
    Which solves issue of np overriding dtypes
    
    For example, if target column starts with a uniform dtype, including NaN, int, float
    When np.where inserts a string it converts all other dtypes to str
    Although does not appear to do so when target column starts with mixed dtypes
    
    This version is built on top of pd.loc which should avoid this numpy edge case
    We like the np.where conventions of in single operation specifying both replacement and alternative
    while pd.where only allows specifying one of the two at a time
    
    So we are aggregating as follows
    
    We previously used np.where in the form
    df[column] = np.where(condition, replacement, alternative)
    
    So new convention is we'll conduct as
    df = self.__autowhere(df, column, condition, replacement, alternative)
    
    Or may optionally be applied with an alternate specified parameter
    Which may be passed as one of {'replacementalternative', 'replacement', 'alternative'}
    designating that the operation will inspect only the replacement, alternative, or the default of both
    When applying only the replacement or alternative if column not previously initialized unspecified will return as integer 0
    
    Where df is the target dataframe (if an empty dataframe one will be initialized with rowcount per condition)
    column is the target column for insertion, 
    which in some cases may be a new column created by this function
    condition is a column of booleans of same number of rows as df
    when condition == True, df[column] = replacement
    when condition == False, df[column] = alternative
    
    replacement/alternative may be a single value (such as number or string)
    or may be a series/dataframe with same number of rows as df
    """

    #if df is empty dataframe, populate as column of zeros per shape of condition
    if type(df) == type(pd.DataFrame()) and df.empty:
      df = pd.DataFrame({column : [0] * condition.shape[0]}, index=pd.DataFrame(condition).index)

    #elif column not in df, populate as a new column of all zeros
    elif column not in df:
      #initialize with arbitrary plug value
      # df[column] = 0
      df.loc[:, column] = 0
      
    if df[column].dtype.name == 'category':
      if isinstance(replacement, (type(pd.DataFrame({0:[1]})), type(pd.Series([1])), type(np.array([])))):
        if len(set(pd.DataFrame(replacement).iloc[:,0].unique()) - set(df[column].cat.categories)) > 0:
          df[column] = df[column].astype('object')
      elif len({replacement} - set(df[column].cat.categories)) > 0:
        df[column] = df[column].astype('object')

    #replacement transforms just for condition == True
    #alternative transforms just for condition == False
    #replacementalternative transforms for both conditions True and False

    #transform for condition == True
    if specified in {'replacement', 'replacementalternative'}:
      df.loc[condition, column] = replacement
      
    #transform for condition == False
    if specified in {'alternative', 'replacementalternative'}:
      df.loc[condition == False, column] = alternative
      
    return df

  def __autocopy(self, original):
    """
    python's deepcopy is useful for copying dictionaries to avoid overwrite from memory sharing
    (python lists and dictionaries are mutable containers such that setting a=b means a is b)
    the only hitch is that there are some non-native entries incompatible with deepcopy
    this is not an issue in current implementation
    but since we are allowing custom defined functions
    it is possible that a user may wish to store a non-native entry in the normalization_dict
    this support function circumvents by copying in different fashion where appropriate
    resulting in what is equivalent to a deepcopy but with support for non-native entries

    this function has not been benchmarked for performance against large scale dictionaries
    (expected largest dictionaries it targets is the postprocess_dict, which has most of memory overhead for trained models)
    """

    if isinstance(original, dict):
      orig_dict_copy = {}
      for entry in original:
        if isinstance(original[entry], (dict, list)):
          entry_copy = self.__autocopy(original[entry])
          orig_dict_copy.update({entry : entry_copy})
        else:
          orig_dict_copy.update({entry : original[entry]})
      return orig_dict_copy
    
    elif isinstance(original, list):
      orig_list_copy = []
      for entry in original:
        if isinstance(entry, (dict, list)):
          entry_copy = self.__autocopy(entry)
          orig_list_copy.append(entry_copy)
        else:
          orig_list_copy.append(entry)
      return orig_list_copy
    
    else:
      return original

  def __autoprint(self, tier, printstring='', variable=''):
    """
    this function is a kind of compromise between
    python's logger and print statement
    the tier is associated with threshold of printstatus
    and is similar to python's logging tiers
    and it accepts entries of {'debug', 'info', 'warning', 'silent'}
    printstring is the same was would have been passed to print()
    
    this function relies on availability in memory of global logger_dict
    which records printstrings in tiers corresponding to debug/info/warning
    e.g.
    logger_dict = {'debug_report' : all_strings, #printstatus=True printouts
                   'info_report' : summarizing_strings, #printstatus='summary' printouts
                   'warning_report' : erorr_report_strings , #printstatus=False printouts
                   }
                   
    and the function also calls the python print() statement
    to print the printsting based on whcih tier is applied
    
    logger_dict['printstatus'] will have been initialized with the printstatus parameter

    logger_dict is periodically synchronized with the dictionary "logger"
    which may have been initailized as dicitonary externally
    """

    #print targets are all converted to str for use in reports
    if not isinstance(printstring, str):
      printstring = str(printstring)
    if not isinstance(variable, str):
      variable = str(variable)
    
    if tier == 'debug':
      logger_dict['debug_report'] = \
      logger_dict['debug_report'] + '\n' + printstring + variable
      
      if logger_dict['printstatus'] in {True}:
        print(printstring + variable)
      
    elif tier == 'info':
      
      logger_dict['debug_report'] = \
      logger_dict['debug_report'] + '\n' + printstring + variable
      
      logger_dict['info_report'] = \
      logger_dict['info_report'] + '\n' + printstring + variable
      
      if logger_dict['printstatus'] in {True, 'summary'}:
        print(printstring + variable)
      
    elif tier == 'warning':
      
      logger_dict['debug_report'] = \
      logger_dict['debug_report'] + '\n' + printstring + variable
      
      logger_dict['info_report'] = \
      logger_dict['info_report'] + '\n' + printstring + variable
      
      logger_dict['warning_report'] = \
      logger_dict['warning_report'] + '\n' + printstring + variable
      
      if logger_dict['printstatus'] in {True, 'summary', False}:
        print(printstring + str(variable))
        
    return

  def __logger_append(self, logger):
    """
    update logger with logging_dict
    which will carry through to logger when it was intialized externally
    """

    logger.update(logger_dict)

    return logger

  def __column_convert_support(self, mixedcolumns_list, postprocess_dict, convert_to='returned'):
    """
    Support function to convert a received list of column headers mixedcolumns_list
    Which may optionally include a mixed set of input column headers and returned column headers with suffix appenders
    to a returned list translatedcolumns_list
    with a single representation of either the corresponding input or returned headers
    (so either all input headers or all returned headers)
    note that redundant derived input columns are returned as single entry
    
    postprocess_dict assumes the column_dict entries assosciated with transformation functions have already been populated
    convert_to accepts one of {'returned', 'input'}, which is for specifying the returned form
    """

    #user can pass a single header as string if they are lazy
    if not isinstance(mixedcolumns_list, list):
      mixedcolumns_list = [mixedcolumns_list]
    
    translatedcolumns_list = []
    
    #accomodate excl suffix scenario
    if 'excl_suffix_inversion_dict' in postprocess_dict:
      #if this takes place after any suffix conversion based on excl_suffix parameter
      #we'll just convert to include suffix for simplicity
      self.__list_replace(mixedcolumns_list, postprocess_dict['excl_suffix_conversion_dict'])
    
    if convert_to == 'returned':
      
      for entry in mixedcolumns_list:
        
        if entry in postprocess_dict['origcolumn']:
          translatedcolumns_list += postprocess_dict['origcolumn'][entry]['columnkeylist']
          
        elif entry in postprocess_dict['column_dict']:
          translatedcolumns_list += [entry]
          
    elif convert_to == 'input':
      
      for entry in mixedcolumns_list:
        
        if entry in postprocess_dict['origcolumn']:
          translatedcolumns_list += [entry]
          
        elif entry in postprocess_dict['column_dict']:
          inputcolumn = postprocess_dict['column_dict'][entry]['origcolumn']
          if inputcolumn not in translatedcolumns_list:
            translatedcolumns_list += [inputcolumn]
            
    return translatedcolumns_list

  def __orig_headers_support(self, df, postprocess_dict):
    """
    when automunge parameter orig_headers passed as True
    the suffix appenders are stripped from returned column headers
    note that this may result in redundant column headers
    which pandas is ok with (but automunge doesn't accept as input)
    intended for use with integrating noise into existing data pipelines
    """

    orig_headers = postprocess_dict['orig_headers']
    privacy_encode = postprocess_dict['privacy_encode']
    excl_suffix = postprocess_dict['excl_suffix']
    
    #note that single column dataframes aren't converted to Series until near return
    if privacy_encode is False and isinstance(df, type(pd.DataFrame())) and len(list(df)) > 0:
      
      returned_headers = list(df)
      
      if excl_suffix is False:
        for excl_entry in postprocess_dict['excl_suffix_conversion_dict']:
          if excl_entry in returned_headers:
            returned_headers[returned_headers.index(excl_entry)] = \
            postprocess_dict['excl_suffix_conversion_dict'][excl_entry]
            
      orig_headers = [postprocess_dict['column_dict'][x]['origcolumn'] for x in returned_headers]
      
      rename_dict = dict(zip(returned_headers, orig_headers))
      
      df.rename(columns = rename_dict, inplace = True)
      
    return df

  #__FunctionBlock: encryption utility functions

  def __encrypt_postprocess_dict(self, postprocess_dict, encrypt_key, privacy_encode, printstatus):
    """
    replaces the postprocess_dict with a public version
    with entries contengent on the privacy_encode parameter
    
    when privacy_encode is True, includes
    ['encryption', 'encrypted', 'columntype_report', 'label_columntype_report', 'privacy_encode', 'automungeversion', 'labelsencoding_dict']
    when privacy_encode is False, includes those and also 'FS_sorted'
    when privacy encode is 'private', only includes
    ['encryption', 'encrypted']
    
    #encrypt_key is passed by user, can be either boolean False, an integer {16, 24, 32}, 
    #or can be a bytes object of one of those lengths
    #note block size of 16 aligns with 128 bit encryption, 32 aligns with 256 bit
    #default of boolean False deactivates
    #integer case is treated as the block size and used to derive a key returned in printouts
    #bytes case is treated as a user specified key with block size based on that length

    #aspects of the following implementation were inspired by a tutorial published by pythonprogramming.net
    #https://pythonprogramming.net/encryption-and-decryption-in-python-code-example-with-explanation/

    Please note that the AES encryption is applied with the [pycrypto](https://github.com/pycrypto/pycrypto) python library 
    which requires installation in order to run (we found there were installations available via conda install). 
    """
    
    #we'll populate public_dict to replace postprocess_dict
    public_dict = {'encryption' : True}
    
    public_entries = ['columntype_report', 'label_columntype_report', 'privacy_encode', 
                      'automungeversion', 'labelsencoding_dict', 'sampling_report_dict',
                      'final_model']
    #column_map reports origination of features
    #columntype_report report types and groupings of features
    #privacy_encode is value of privacy_encode as passed to automunge(.) for reference
    #automungeversion is version number of the automugne(.) call
    #labelsencoding_dict is a resource for label inversion without decrypting

    if privacy_encode is False:
      #FS_sorted is results of feature importance evaluation if elected
      public_entries = public_entries + ['column_map', 'FS_sorted']
    
    #public_dict doesn't populate these in privacy_encode == 'private' scenario
    if privacy_encode in {True, False}:
      
      for public_entry in public_entries:

        if public_entry in postprocess_dict:
          public_dict.update({public_entry : postprocess_dict[public_entry]})
        
    #now we'll convert our postprocess_dict to a bytes representation with pickle
    import pickle
    postprocess_dict = pickle.dumps(postprocess_dict)
    
    #now we'll encrypt that bytes representionation, which we'll then store the encryption in the public_dict
    
    #we'll use the AES package from pycrypto
    #note that we recomend installing pycrypto with conda vs pip

    # #user can conduct this import externally to speed up this function
    # module = 'Crypto.Cipher.AES'
    # if module not in sys.modules:
    from Crypto.Cipher import AES

    #for encryption we'll format data as base64
    import base64
    
    #encrypt_key is passed by user, can be either boolean False, an integer {16, 24, 32}, 
    #or can be a bytes object of one of those lengths
    #default of boolean False deactivates
    #integer case is treated as the block size and used to derive a key returned in printouts
    #bytes case is treated as a user specified key with block size based on that length
      
    #note block size of 16 aligns with 128 bit, 32 aligns with 256 bit
    if isinstance(encrypt_key, int) and encrypt_key in {16, 24, 32}:

      block_size = encrypt_key

      #os is a native python module used to access operating system functionality
      #as used here, the urandom module's source of randomness 
      #may differ between operating systems e.g. linux/mac/windows
      from os import urandom
      encrypt_key = urandom(block_size)

      print("_______________________________________________________")
      print("_______________________________________________________")
      print("*******************************************************")
      print("*******************************************************")
      print()
      print("Derived encrypt_key:")
      print()
      print(encrypt_key)
      print()
      print("Please copy and save this key for use with the postmunge(.) encrypt_key parameter")
      print("You will need this key to prepare additional data in postmunge(.)")
      print()
      print("*******************************************************")
      print("*******************************************************")
      print("_______________________________________________________")
      print("_______________________________________________________")

    elif isinstance(encrypt_key, bytes):

      block_size = len(encrypt_key)
      
      # if printstatus is True:
      self.__autoprint('debug', "_______________________________________________________")
      self.__autoprint('debug', "_______________________________________________________")
      self.__autoprint('debug', "*******************************************************")
      self.__autoprint('debug', "*******************************************************")
      self.__autoprint('debug', '')
      self.__autoprint('debug', "User passed a custom encrypt_key to automunge(.)")
      self.__autoprint('debug', "Please remember to save your key for use with the postmunge(.) encrypt_key parameter")
      self.__autoprint('debug', "You will need this key to prepare additional data in postmunge(.)")
      self.__autoprint('debug', '')
      self.__autoprint('debug', "*******************************************************")
      self.__autoprint('debug', "*******************************************************")
      self.__autoprint('debug', "_______________________________________________________")
      self.__autoprint('debug', "_______________________________________________________")
      
    #now create the cypher object
    cipher = AES.new(encrypt_key)

    #we'll use the b'{' character for padding to ensure value is always a multiple of block size
    padding = b'{'

    #here is the function to pad (pads characters on the right side of string)
    #hat tip pythonprogramming.net
    pad = lambda s: s + (block_size - len(s) % block_size) * padding

    #here is the function to encode
    #hat tip pythonprogramming.net
    EncodeAES = lambda c, s: base64.b64encode(c.encrypt(pad(s)))

    #now we can code the pickled dicitonary
    postprocess_dict = EncodeAES(cipher, postprocess_dict)
    
    #we'll then save that encryption in the public_dict
    public_dict.update({'encrypted' : postprocess_dict})
    
    return public_dict

  def __decrypt_postprocess_dict(self, postprocess_dict, encrypt_key, printstatus):
    """
    this funciton applied at start of postmunge
    postprocess_dict is as returned from an automunge(.) call with encryption
    and includes an entry 'encrypted' containing the encrypted full postprocess_dict
    
    encrypt_key is intended to be passed as a bytes object matching the automunge encryption key
    where in automunge encrypt_key will either have been generated and returned in printouts
    (when automunge encrypt_key was passed as one of True / 16 / 24 / 32 a key is generated,
    or when automunge encrypt_key was passed as a bytes object with length 16/24/32 that is treated as the key)
    
    this function decodes and returns the encrypted full postprocess_dict
    
    #aspects of the following implementation were inspired by a tutorial published by pythonprogramming.net
    #https://pythonprogramming.net/encryption-and-decryption-in-python-code-example-with-explanation/

    Please note that the AES encryption is applied with the [pycrypto](https://github.com/pycrypto/pycrypto) python library 
    which requires installation in order to run (we found there were installations available via conda install). 
    """

    decode_valresult = False
    
    if not (isinstance(encrypt_key, bytes) and len(encrypt_key) in {16, 24, 32}):
      
      decode_valresult = True
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: encryption was performed in automunge(.) without specifying the key in postmunge encrypt_key parameter")
      self.__autoprint('warning', "encrypt_key accepts as a bytes object with length of one of {16, 24, 32}")
      self.__autoprint('warning', "encrypt_key was either returned in printouts of corresponding automunge(.) call")
      self.__autoprint('warning', "or was designated by user through the automugne(.) encrypt_key parameter")
      self.__autoprint('warning', '')
      
    if 'encrypted' not in postprocess_dict:
      
      decode_valresult = True
      
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: encrypt_key accepts as a bytes object with length of one of {16, 24, 32}")
      
    # #user can conduct this import externally to speed up this function
    # module = 'Crypto.Cipher.AES'
    # if module not in sys.modules:
    from Crypto.Cipher import AES

    import base64
    import pickle

    padding = b'{'

    #here is the function to decode
    #hat tip pythonprogramming.net
    DecodeAES = lambda c, e: c.decrypt(base64.b64decode(e)).rstrip(padding)

    #encrypt_key is passed by user to postmunge
    #and needs to match the generated encrypt_key bytes either passed to automunge
    #or generated by automunge and returned in printouts when encrypt_key was passed to automunge as integer

    cipher = AES.new(encrypt_key)

    #this gives a pickled representation, then can convert that to a dictionary
    postprocess_dict = DecodeAES(cipher, postprocess_dict['encrypted'])
    
    #note that if this causes error it may be because user has not first initialized custom functions saved in the dictionary (e.g. for custom_train or customML)
    #or user may have entered an incorrect encrypt_key value
    postprocess_dict = pickle.loads(postprocess_dict)
    
    return postprocess_dict, decode_valresult

  #__FunctionBlock: automunge ppd_append

  def __ppd_append_support(self, ppd_append, postprocess_dict, encrypt_key, printstatus):
    """
    when ppd_append receives input of a prior populated postprocess_dict
    the current postprocess_dict is appended as an entry to the prior populated version
    and the postprocess_dict returned from automunge(.) is the prior with the new one as an entry
    
    the purpose of this operation is support use case where a model has already been trained
    and user wishes to add new features to the basis of that model
    as is possible in some decision tree paradigms
    
    thus the postprocess_dict assocaited with the prior trained model
    can be integrated with this new ppd with the new features
    in order to prepare both prior and new features in a single operation
    
    there is some reduced functionality associated with adding features to a prior populated ppd
    ML infill basis will be limited to boundaries between prior and new features
    if user wishes to extend prior ML infill basis to inlcude new features and visa versa
    will need to prepare the entries data set collectively with old and new features
    which is associated with an alternate workflow where user wqould need to retrain the associated model
    to account for the alternate ML infill basis of the original features
    
    if the prior postprocess_dict is recieved as encrypted
    then user needs to pass encrypt_key as the bitstring key
    which will result in the newly returned postprocess_dict
    having the same encryptioun key as the one prior
    
    the newly populated postprocess_dict is stored in ppd_append['new_feature_ppd']
    and then ppd_append is the automunge(.) returned postprocess_dict
    
    in postmunge, if 'new_feature_ppd' is populated
    the new features will be carved out from df_test
    and then processed seperately at the end of the workflow and concatinated
    note this approach supports multiple rounds of adding new features
    """

    #if prior postprocess_dict was recieved encrypted first we decrypt
    if 'encryption' in ppd_append and ppd_append['encryption'] is True:
      
      ppd_append = self.__autocopy(ppd_append)

      ppd_append, decode_valresult = \
      self.__decrypt_postprocess_dict(ppd_append, encrypt_key, printstatus)
      
    #now simply append newly populated ppd onto the original and return the original
    
    if 'new_feature_ppd' not in ppd_append:
      #backward compatibility preceding 7.53
      ppd_append.update({'new_feature_ppd' : {}})
      
    i = 0
    
    if ppd_append['new_feature_ppd'] != {}:
      
      i = int(pd.DataFrame({'getmax':list(ppd_append['new_feature_ppd'])}).astype(int).max()) + 1
      
    ppd_append['new_feature_ppd'].update({i : postprocess_dict})
    
    return ppd_append

  #__FunctionBlock: automunge(.) definition
  
  def automunge(self, df_train, df_test = False,
                labels_column = False, trainID_column = False, testID_column = False,
                valpercent=0.0, floatprecision = 32, cat_type = False, shuffletrain = True, noise_augment = 0,
                dupl_rows = False, TrainLabelFreqLevel = False, powertransform = False, binstransform = False,
                MLinfill = True, infilliterate=1, randomseed = False, eval_ratio = .5,
                numbercategoryheuristic = 255, pandasoutput = 'dataframe', NArw_marker = True,
                featureselection = False, featurethreshold = 0., inplace = False, orig_headers = False,
                Binary = False, PCAn_components = False, PCAexcl = [], excl_suffix = False,
                ML_cmnd = {'autoML_type':'randomforest',
                           'MLinfill_cmnd':{'RandomForestClassifier':{}, 'RandomForestRegressor':{}},
                           'PCA_type':'default',
                           'PCA_cmnd':{}},
                assigncat = {'1010':[], 'onht':[], 'ordl':[], 'bnry':[], 'hash':[], 'hsh2':[],
                             'DP10':[], 'DPoh':[], 'DPod':[], 'DPbn':[], 'DPhs':[], 'DPh2':[],
                             'nmbr':[], 'mnmx':[], 'retn':[], 'DPnb':[], 'DPmm':[], 'DPrt':[],
                             'bins':[], 'pwr2':[], 'bnep':[], 'bsor':[], 'por2':[], 'bneo':[],
                             'ntgr':[], 'srch':[], 'or19':[], 'tlbn':[], 'excl':[], 'exc2':[]},
                assignparam = {'default_assignparam' : {'(category)' : {'(parameter)' : 42}},
                                        '(category)' : {'(column)'   : {'(parameter)' : 42}}},
                assigninfill = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[],
                                'adjinfill':[], 'meaninfill':[], 'medianinfill':[], 'negzeroinfill':[],
                                'interpinfill':[], 'modeinfill':[], 'lcinfill':[], 'naninfill':[]},
                assignnan = {'categories':{}, 'columns':{}, 'global':[]},
                transformdict = {}, processdict = {}, evalcat = False, ppd_append = False,
                entropy_seeds = False, random_generator = False, sampling_dict = False,
                privacy_encode = False, encrypt_key = False, printstatus = 'summary', logger = {}):
    """
    #This function documented in READ ME, available online at:
    # https://github.com/Automunge/AutoMunge/blob/master/README.md

    #Note that automunge(.) has one global variable: logger_dict
    """

    """
    Workflow blocks can be navigated with control F search:
    #__WorkflowBlock: automunge Parameter prep
    #__WorkflowBlock: automunge Parameter validations
    #__WorkflowBlock: automunge Feature Importance
    #__WorkflowBlock: automunge Misc dataframe preps and validations
    #__WorkflowBlock: automunge noise_augment duplicate extraction
    #__WorkflowBlock: automunge ID set extraction
    #__WorkflowBlock: automunge validation and label set extraction
    #__WorkflowBlock: automunge column validations and variable initializations
    #__WorkflowBlock: automunge column processing
    #__WorkflowBlock: automunge label column processing
    #__WorkflowBlock: automunge protected feature processing
    #__WorkflowBlock: automunge infill derivation and insertion
    #__WorkflowBlock: automunge feature importance dimensionality reduction
    #__WorkflowBlock: automunge PCA dimensionality reduction
    #__WorkflowBlock: automunge Binary dimensionality reduction (categoric consolidations)
    #__WorkflowBlock: automunge duplicate row consolidation
    #__WorkflowBlock: automunge label rebalancing (e.g. oversampling)
    #__WorkflowBlock: automunge row shuffling
    #__WorkflowBlock: automunge data type management
    #__WorkflowBlock: automunge excl suffix management
    #__WorkflowBlock: automunge populate postprocess_dict
    #__WorkflowBlock: automunge privacy encoding
    #__WorkflowBlock: automunge noise_augment prep duplicate sets
    #__WorkflowBlock: automunge validation data prep
    #__WorkflowBlock: automunge final preps and return
    """

    #_________________________________________________________
    #__WorkflowBlock: automunge Parameter prep
    #includes copying lists and dictionaries to internal state
    #and string conversions for column assignments
    
    # application_timestamp = dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
    application_number = random.randint(100000000000,999999999999)
    trainID_column_orig = trainID_column
    testID_column_orig = testID_column

    #__autocopy passed dictionaries so as not to edit exterior objects
    #(these are not expected to be large objects so the memory impact is negligable)
    #including ML_cmnd, assigncat, assignparam, assigninfill, assignnan, transformdict, processdict 
    if isinstance(ML_cmnd, dict):
      ML_cmnd = self.__autocopy(ML_cmnd)
      #ML_cmnd_orig is to record state of ML_cmnd as received
      ML_cmnd_orig = self.__autocopy(ML_cmnd)
    if isinstance(assigncat, dict):
      assigncat = self.__autocopy(assigncat)
    if isinstance(assignparam, dict):
      assignparam = self.__autocopy(assignparam)
    if isinstance(assigninfill, dict):
      assigninfill = self.__autocopy(assigninfill)
    if isinstance(assignnan, dict):
      assignnan = self.__autocopy(assignnan)
    if isinstance(transformdict, dict):
      transformdict = self.__autocopy(transformdict)
    if isinstance(processdict, dict):
      processdict = self.__autocopy(processdict)
    if isinstance(sampling_dict, dict):
      sampling_dict = self.__autocopy(sampling_dict)

    #similarly copy any input lists to internal state
    #(these won't be large so not taking account of inplace parameter)
    if isinstance(labels_column, list):
      labels_column = self.__autocopy(labels_column)
    if isinstance(trainID_column, list):
      trainID_column = self.__autocopy(trainID_column)
    if isinstance(testID_column, list):
      testID_column = self.__autocopy(testID_column)
    if isinstance(Binary, list):
      Binary = self.__autocopy(Binary)
    if isinstance(PCAexcl, list):
      PCAexcl = self.__autocopy(PCAexcl)
    if isinstance(entropy_seeds, list):
      entropy_seeds = self.__autocopy(entropy_seeds)

    #initialize logger_dict
    global logger_dict
    logger_dict = {
      'debug_report' : '',
      'info_report' : '',
      'warning_report' : '',
      'printstatus' : printstatus, 
      'validations' : {'suffixoverlap_results':{}},
    }

    #quick conversion of any assigncat and assigninfill entries to str (such as for cases if user passed integers)
    assigncat, assigncat_str_convert_valresult = self.__assigncat_str_convert(assigncat, printstatus)
    assigninfill = self.__assigninfill_str_convert(assigninfill)
    assignparam = self.__assignparam_str_convert(assignparam)
    assignnan = self.__assignnan_str_convert(assignnan)

    #similarily, quick conversion of any passed column idenitfiers to str
    labels_column = self.__parameter_str_convert(labels_column)
    trainID_column = self.__parameter_str_convert(trainID_column)
    testID_column = self.__parameter_str_convert(testID_column)

    #convert assignnan if not received as lists in bottom tiers
    assignnan = self.__assignnan_list_convert(assignnan)

    #_________________________________________________________
    #__WorkflowBlock: automunge Parameter validations
    #includes aggregations of validation results
    #as well as validations and prepping for transformdict and processdict 
    #with consolidations into internal library transform_dict and process_dict
    #and a few other initializations

    #check the range of parameters 
    #(generally speaking other than passed dictionaries, dataframes, or column identifiers)
    self.__check_am_miscparameters(valpercent, floatprecision, shuffletrain, \
                                 TrainLabelFreqLevel, dupl_rows, powertransform, binstransform, MLinfill, \
                                 infilliterate, randomseed, eval_ratio, numbercategoryheuristic, pandasoutput, \
                                 NArw_marker, featurethreshold, featureselection, inplace, \
                                 Binary, PCAn_components, PCAexcl, printstatus, excl_suffix, \
                                 trainID_column, testID_column, evalcat, privacy_encode, encrypt_key, \
                                 noise_augment, ppd_append, orig_headers, cat_type, logger)

    #quick check to ensure each column only assigned once in assigncat and assigninfill
    check_assigncat_result = self.__check_assigncat(assigncat, printstatus)
    check_assigninfill_result = self.__check_assigninfill(assigninfill, printstatus)

    #initialize ML_cmnd if incompletely specified, also some validation tests
    check_ML_cmnd_result, ML_cmnd = \
    self.__check_ML_cmnd(ML_cmnd, printstatus)

    logger_dict['validations'].update({'check_assigncat_result' : check_assigncat_result, \
                                   'check_assigninfill_result' : check_assigninfill_result, \
                                   'check_ML_cmnd_result' : check_ML_cmnd_result,
                                   'assigncat_str_convert_valresult' : assigncat_str_convert_valresult})
    logger = self.__logger_append(logger)
    
    #initialize transform_dict which is the internal library of family trees
    transform_dict = self.__assembletransformdict(binstransform, NArw_marker)

    #transformdict is user passed data structure to add entries to transform_dict
    if bool(transformdict) is not False:

      #validates format of transformdict
      check_transformdict000_result1, check_transformdict000_result2 = \
      self.__check_transformdict000(transformdict, printstatus)

      logger_dict['validations'].update({'check_transformdict000_result1' : check_transformdict000_result1, \
                                     'check_transformdict000_result2' : check_transformdict000_result2})
      logger = self.__logger_append(logger)

      #This validates data types of primitive entries, converts string entry to embed in list brackets
      check_transformdict00_result, transformdict = \
      self.__check_transformdict00(transformdict, printstatus)

      logger_dict['validations'].update({'check_transformdict00_result' : check_transformdict00_result})
      logger = self.__logger_append(logger)

      #If only partial family tree populated this populates other primitives
      check_transformdict0_result, transformdict = \
      self.__check_transformdict0(transformdict, printstatus)

      logger_dict['validations'].update({'check_transformdict0_result' : check_transformdict0_result})
      logger = self.__logger_append(logger)
      
      #handling for family trees without replacement primitive entries (add an excl transform)
      check_transformdict_result1, transformdict = \
      self.__check_transformdict(transformdict, printstatus)

      logger_dict['validations'].update({'check_transformdict_result1' : check_transformdict_result1})
      logger = self.__logger_append(logger)
      
      #ensure no redundant specifications in adjacent primitives
      check_transformdict2_result1, check_transformdict2_result2 = \
      self.__check_transformdict2(transformdict, printstatus)
      
      logger_dict['validations'].update({'check_transformdict2_result1' : check_transformdict2_result1, \
                                     'check_transformdict2_result2' : check_transformdict2_result2})
      logger = self.__logger_append(logger)

      #now consolidate the transform_dict and transformdict into single dictionary
      transform_dict.update(transformdict)
      
    if isinstance(transformdict, dict) and transformdict != {}:
      #check for infinite loops in consolidated transform_dict
      check_haltingproblem_result, familytree_for_offspring_result = \
      self.__check_haltingproblem(transformdict, transform_dict, printstatus, max_check_count = 1111)
    else:
      check_haltingproblem_result = False
      familytree_for_offspring_result = False
    
    logger_dict['validations'].update({'check_haltingproblem_result' : check_haltingproblem_result,
                                   'familytree_for_offspring_result' : familytree_for_offspring_result})
    logger = self.__logger_append(logger)

    #initialize process_dict which is the internal library of transformation category properties
    process_dict = self.__assembleprocessdict()

    #this enables functionpointer support for internally defined process_dict
    #we do not currently use this functionality because __assembleprocessdict also serves as documentation 
    #it is being provided for future generations
    process_dict, check_process_dict_functionpointer_result = \
    self.__grab_functionpointer_entries(process_dict, process_dict, printstatus)

    logger_dict['validations'].update({'check_process_dict_functionpointer_result' : check_process_dict_functionpointer_result})
    logger = self.__logger_append(logger)

    #processdict is user passed data strucure to add entries to process_dict
    if bool(processdict) is not False:
      
      #this function checks if any category entries in user passed processdict 
      #have a functionpointer entry
      #and if so populate the entry with the associated pointer entries
      #when not previously specified
      processdict, check_functionpointer_result = \
      self.__grab_functionpointer_entries(processdict, process_dict, printstatus)
      logger_dict['validations'].update({'check_functionpointer_result' : check_functionpointer_result})
      logger = self.__logger_append(logger)
      
      #this funcion applies some misc validations on processdict
      check_processdict_result, check_processdict_result2 = \
      self.__check_processdict(processdict, printstatus)
      logger_dict['validations'].update({'check_processdict_result' : check_processdict_result,
                                     'check_processdict_result2': check_processdict_result2})
      logger = self.__logger_append(logger)

      #this function ensures any populated processing functions are either callable or None
      check_processdict4_valresult, check_processdict4_valresult2 = \
      self.__check_processdict4(processdict, printstatus)
      logger_dict['validations'].update({'check_processdict4_valresult' : check_processdict4_valresult,
                                     'check_processdict4_valresult2' : check_processdict4_valresult2})
      logger = self.__logger_append(logger)

      #now consolidate user passed entries from processdict and internal library in process_dict
      process_dict.update(processdict)
      
    else:
      logger_dict['validations'].update({'check_functionpointer_result' : False})
      logger_dict['validations'].update({'check_processdict_result' : False,
                                     'check_processdict_result2': False})
      logger = self.__logger_append(logger)

    #now that both transform_dict and process_dict are consolidated, validate transformdict roots have processdict entries
    check_transform_dict_roots_result = \
    self.__check_transform_dict_roots(transform_dict, process_dict, printstatus)
    logger_dict['validations'].update({'check_transform_dict_roots_result' : check_transform_dict_roots_result})
    logger = self.__logger_append(logger)
      
    #here we confirm that all of the keys of assigncat have corresponding entries in process_dict
    check_assigncat_result2 = self.__check_assigncat2(assigncat, transform_dict, printstatus)
    
    #now double check that any category entries in the assigncat have populated family trees
    #whose primitive entries are categories that all have properties defined with entries in the process_dict
    check_assigncat_result3 = self.__check_assigncat3(assigncat, process_dict, transform_dict, printstatus)
    
    logger_dict['validations'].update({'check_assigncat_result2' : check_assigncat_result2, \
                                   'check_assigncat_result3' : check_assigncat_result3})
    logger = self.__logger_append(logger)

    check_assignparam_result = self.__check_assignparam(assignparam, process_dict, printstatus)
    logger_dict['validations'].update({'check_assignparam_result' : check_assignparam_result})
    logger = self.__logger_append(logger)

    #initialize autoMLer which is data structure to support ML infill training and inference
    #custom training loops are supported as documented in read me section Custom ML Infill Functions
    autoMLer = self.__assemble_autoMLer()

    random_generator_valresult, random_generator_accepts_seeds = \
    self.__check_random_generator(random_generator, printstatus)
    logger_dict['validations'].update({'random_generator_valresult' : random_generator_valresult})
    logger = self.__logger_append(logger)

    #validate sampling_dict and populate with defaults
    check_sampling_dict_result, sampling_dict = \
    self.__check_sampling_dict(sampling_dict, printstatus, random_generator_accepts_seeds)
    logger_dict['validations'].update({'check_sampling_dict_result' : check_sampling_dict_result})
    logger = self.__logger_append(logger)

    #validate entropy_seeds and align to common type
    entropy_seeds, entropy_seeds_result = \
    self.__check_entropy_seeds(entropy_seeds, printstatus)
    logger_dict['validations'].update({'entropy_seeds_result' : entropy_seeds_result})
    logger = self.__logger_append(logger)

    #initialize randomseed for default configuration of random random seed
    #when a sampling performed randomrandomseed returned as True
    #this is used in row shuffling and other random seeds that don't need to match automunge random seed
    #note that this may expend two entropy_seeds
    randomseed, randomrandomseed, entropy_seeds, populate_randomseed_expended_seed_count = \
    self.__populate_randomseed(randomseed, entropy_seeds, 
                              random_generator, sampling_dict)

    #_________________________________________________________
    #__WorkflowBlock: automunge Feature Importance
    #a feature importance evaluation (based on featureselection parameter) performed here
    
    #feature selection analysis performed here if elected
    if featureselection in {True, 'pct', 'metric', 'report'}:

      if labels_column is False:
        # if printstatus != 'silent':
        self.__autoprint('warning', "featureselection not available without labels_column in training set")
        self.__autoprint('warning', '')

        labels_column_for_featureselect_valresult = True
        logger_dict['validations'].update({'labels_column_for_featureselect_valresult' : labels_column_for_featureselect_valresult})
        logger = self.__logger_append(logger)
        
        madethecut = []
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}
        featureimportance = {}
        logger_dict['validations'].update({'FS_numeric_data_result': False})
        logger_dict['validations'].update({'FS_all_valid_entries_result': False})
        logger = self.__logger_append(logger)

      elif len(list(df_train)) < 2:
        # if printstatus != 'silent':
        self.__autoprint('warning', "featureselection not available without at least two features in training set")
        self.__autoprint('warning', '')

        twofeatures_for_featureselect_valresult = True
        logger_dict['validations'].update({'twofeatures_for_featureselect_valresult' : twofeatures_for_featureselect_valresult})
        logger = self.__logger_append(logger)

        madethecut = []
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}
        featureimportance = {}
        logger_dict['validations'].update({'FS_numeric_data_result': False})
        logger_dict['validations'].update({'FS_all_valid_entries_result': False})
        logger = self.__logger_append(logger)

      else:

        madethecut, FSmodel, FScolumn_dict, FS_sorted = \
        self.__featureselect(df_train, labels_column, trainID_column, \
                          powertransform, binstransform, randomseed, \
                          numbercategoryheuristic, assigncat, transformdict, \
                          processdict, featurethreshold, featureselection, \
                          ML_cmnd, process_dict, valpercent, printstatus, NArw_marker, \
                          assignparam, entropy_seeds, random_generator, sampling_dict, logger)

      #the final returned featureimportance report consolidates the sorted results with raw data
      featureimportance = {'FS_sorted'     : FS_sorted, \
                           'FScolumn_dict' : FScolumn_dict}

      #if featureselection is report then no further processing just return the results
      if featureselection == 'report':

        #printout display progress
        # if printstatus in {True, 'summary'}:
        self.__autoprint('info', "_______________")
        self.__autoprint('info', "Feature Importance results returned")
        self.__autoprint('info', "")
        self.__autoprint('info', "_______________")
        self.__autoprint('info', "Automunge Complete")
        self.__autoprint('info', "")

        return [], [], [], \
        [], [], [], \
        [], [], [], \
        featureimportance

    else:

      madethecut = []
      FSmodel = False
      FScolumn_dict = {}
      FS_sorted = {}
      featureimportance = {}
      logger_dict['validations'].update({'FS_numeric_data_result': False})
      logger_dict['validations'].update({'FS_all_valid_entries_result': False})
      logger = self.__logger_append(logger)

    #validate that a model was trained
    check_FSmodel_result = self.__check_FSmodel(featureselection, FSmodel, printstatus)
    logger_dict['validations'].update({'check_FSmodel_result' : check_FSmodel_result})
    logger = self.__logger_append(logger)

    #_________________________________________________________
    #__WorkflowBlock: automunge Misc dataframe preps and validations
    #some of preps and validations are performed after feature importance to avoid redundancy
    #this includes dataframe preps such as conversion from numpy or series, header string conversions
    #including column header validations for featuers and labels
    #if df_test was received as boolean False we create a dummy df_test from first row of df_train

    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Begin Automunge")
    self.__autoprint('info', "")

    check_df_train_type_result, check_df_test_type_result = \
    self.__check_df_type(df_train, df_test, printstatus)
    logger_dict['validations'].update({'check_df_train_type_result' : check_df_train_type_result, \
                                   'check_df_test_type_result' : check_df_test_type_result})
    logger = self.__logger_append(logger)

    #copy input dataframes to internal state so as not to edit exterior objects
    #this helps with recursion in feature importance
    #note this is performed after any conversion from array to dataframe
    #or conversion from 
    if inplace is not True:
      df_train = df_train.copy()
      if df_test is not False:
        df_test = df_test.copy()

    #functionality to support passed numpy arrays
    #if passed object was a numpy array, convert to pandas dataframe
    #note that numpy array scenario requires matched ID columns between train and test
    checknp = np.array([])

    #first validate numpy data is tabular
    if isinstance(checknp, type(df_train)):
      check_np_shape_train_result, check_np_shape_test_result = \
      self.__check_np_shape(df_train, df_test, printstatus)
    else:
      check_np_shape_train_result, check_np_shape_test_result = False, False
    logger_dict['validations'].update({'check_np_shape_train_result' : check_np_shape_train_result, \
                                   'check_np_shape_test_result' : check_np_shape_test_result})
    logger = self.__logger_append(logger)

    #received numpy tabular data is converted to pandas, which will result in index integer column headers
    if isinstance(checknp, type(df_train)):
      df_train = pd.DataFrame(df_train)
    if isinstance(checknp, type(df_test)):
      df_test = pd.DataFrame(df_test)

    #received pandas Series (a pandas type for single column sets) are converted to dataframe
    checkseries_train_result = False
    checkseries_test_result = False
    checkseries = pd.Series([1])
    if isinstance(checkseries, type(df_train)):
      checkseries_train_result = True
      df_train = pd.DataFrame(df_train)
    if isinstance(checkseries, type(df_test)):
      checkseries_test_result = True
      df_test = pd.DataFrame(df_test)

    logger_dict['validations'].update({'checkseries_train_result' : checkseries_train_result, \
                                   'checkseries_test_result' : checkseries_test_result})
    logger = self.__logger_append(logger)

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    unique_string_headers_valresult = False
    trainlabels=[str(x) for x in list(df_train)]
    if len(set(trainlabels)) != len(set(df_train)):
      unique_string_headers_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: column header string conversion resulted in redundant headers (may be case where e.g. 1 and '1' are both headers)")
      self.__autoprint('warning', '')
    logger_dict['validations'].update({'unique_string_headers_valresult' : unique_string_headers_valresult})
    logger = self.__logger_append(logger)
    
    df_train.columns = trainlabels

    labels_column_listofcolumns = []

    #validate that labels_column, which at this point has been converted to string, is present in df_train
    labels_column_valresult = False
    if labels_column is not False and labels_column is not True and not isinstance(labels_column, list):
      labels_column_listofcolumns = [labels_column]
      if labels_column not in df_train.columns:
        labels_column_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: labels_column not found as a column header in df_train")
        self.__autoprint('warning', '')
    if isinstance(labels_column, list):
      for labels_column_entry in labels_column:
        if not isinstance(labels_column_entry, set): 
          labels_column_listofcolumns.append(labels_column_entry)
          if labels_column_entry not in df_train.columns:
            labels_column_valresult = True
            # if printstatus != 'silent':
            self.__autoprint('warning', "error: labels_column not found as a column header in df_train")
            self.__autoprint('warning', '')
    logger_dict['validations'].update({'labels_column_valresult' : labels_column_valresult})
    logger = self.__logger_append(logger)

    #convention is that if df_test provided as False then we'll create
    #a dummy set derived from df_train's first row
    #test_plug_marker used to identify that this step was taken
    test_plug_marker = False
    if not isinstance(df_test, pd.DataFrame):
      df_test = df_train[0:1].copy()
      testID_column = trainID_column
      test_plug_marker = True
      if labels_column is not False:
        if labels_column is True:
          del df_test[trainlabels[-1]]
        elif isinstance(labels_column, list):
          for labels_column_entry in labels_column:
            if not isinstance(labels_column_entry, set):
              del df_test[labels_column_entry]
        else:
          del df_test[labels_column]

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[str(x) for x in list(df_test)]
    df_test.columns = testlabels

    #confirm all unique column headers
    check_columnheaders_result = \
    self.__check_columnheaders(list(df_train), printstatus)

    logger_dict['validations'].update({'check_columnheaders_result' : check_columnheaders_result})
    logger = self.__logger_append(logger)

    origcolumns_all = list(df_train)

    #validate assignnan has valid root categories and source columns
    #note this takes place before any label column split from df_train
    check_assignnan_toplevelentries_result, check_assignnan_categories_result, check_assignnan_columns_result \
    = self.__check_assignnan(assignnan, transform_dict, list(df_train), printstatus)

    assignnan_actions_valresult = \
    self.__check_assignnan_injections(assignnan, origcolumns_all, printstatus)
  
    logger_dict['validations'].update({'check_assignnan_toplevelentries_result' : check_assignnan_toplevelentries_result, \
                                   'check_assignnan_categories_result'      : check_assignnan_categories_result, \
                                   'check_assignnan_columns_result'         : check_assignnan_columns_result, \
                                   'assignnan_actions_valresult'            : assignnan_actions_valresult})
    logger = self.__logger_append(logger)

    #_________________________________________________________
    #__WorkflowBlock: automunge noise_augment duplicate extraction
    #prior to extracting ID and label sets
    #if the noise_augment parameter was passed as >0
    #a copy of df_train is collected
    #which will later be prepared in postmunge for one or more concatinations
    
    if noise_augment > 0:
      df_augment_duplicate = df_train.copy()

    #_________________________________________________________
    #__WorkflowBlock: automunge ID set extraction
    #this includes extraction of columns associated with trainID_column and testID_column
    #extraction of any non ranged integer indexes from the dataframes for the ID sets
    #as well as aggregation of the Automunge_index for the ID sets

    #worth a disclaimer:
    #the trainID_column and testID_column column parameters are somewhat overloaded
    #can be passed as string, list, list of lists, boolean, integer 
    #(integers are converted to strings in __parameter_str_convert above)
    #and the next 120 lines or so are kind of inelegant
    #what is being accomplished here is ID columns can be passed as string column headers or list of column headers
    #and carved out from the train and test sets for inclusion in ID sets
    #in cases where passed as list of lists, the first list is deleted from features, the second list is retained in features
    #the ID columns are returned with the addition of new indexcolumn added to ID sets as 'Automunge_index'
    #and if there are existing non-range index column(s) those are carried over to ID sets as well
    #this code works, has been tested
    #there is a similar section in postmunge
    #this is probably the ugliest portion of codebase
    
    #the convention for ID columns is that they are struck from features 
    #and returned consistently partitioned and shuffled in ID sets
    #when the ID parameters are passed as list of list
    #the first list has ID columns that are struck from features
    #and the second list has ID columns that are retained in features
    #which may be useful to inspect before and after configurations for encodings since are consistently shuffled and etc
    trainID_struckfeatures = []
    trainID_retainedfeatures = []

    testID_struckfeatures = []
    testID_retainedfeatures = []
    
    if isinstance(trainID_column, list):
      #if first entry is list specifies features to be struck
      if len(trainID_column) > 0 and isinstance(trainID_column[0], list):
        trainID_struckfeatures = trainID_column[0]
      #if second entry is list specifies features to be retained
      if len(trainID_column) > 1 and isinstance(trainID_column[1], list):
        trainID_retainedfeatures = trainID_column[1]
      #now consolidate trainID_column to include both struck and retained
      if len(trainID_struckfeatures) > 0 or len(trainID_retainedfeatures) > 0:
        trainID_column = trainID_struckfeatures + trainID_retainedfeatures
      else:
        #if first or second entries weren't a list then all entries are struck
        trainID_struckfeatures = trainID_column
    elif trainID_column is not False:
      #in cases where trainID_column is a single entry it is added to trainID_struckfeatures
      trainID_struckfeatures = [trainID_column]
        
    if isinstance(testID_column, list):
      #if first entry is list specifies features to be struck
      if len(testID_column) > 0 and isinstance(testID_column[0], list):
        testID_struckfeatures = testID_column[0]
      #if second entry is list specifies features to be retained
      if len(testID_column) > 1 and isinstance(testID_column[1], list):
        testID_retainedfeatures = testID_column[1]
      #now consolidate trainID_column to include both struck and retained
      if len(testID_struckfeatures) > 0 or len(testID_retainedfeatures) > 0:
        testID_column = testID_struckfeatures.copy() + testID_retainedfeatures.copy()
      else:
        #if first or second entries weren't a list then all entries are struck
        testID_struckfeatures = testID_column.copy()
    elif testID_column is not False:
      #in cases where trainID_column is a single entry it is added to testID_struckfeatures
      testID_struckfeatures = [testID_column]
      
    #this either sets indexcolumn for returned ID sets as 'Automunge_index' 
    #or 'Automunge_index_' + str(application_number) if 'Automunge_index' is already in ID sets
    indexcolumn, indexcolumn_valresult = self.__set_indexcolumn(trainID_column, testID_column, application_number, indexcolumn_base = 'Automunge_index')
    origindexcolumn, origindexcolumn_valresult = self.__set_indexcolumn(trainID_column, testID_column, application_number, indexcolumn_base = 'Orig_index')
    #this is not logged with other suffix overlaps because not an error channel, just results in different index column header
    logger_dict['validations'].update({'indexcolumn_valresult' : indexcolumn_valresult,
                                   'origindexcolumn_valresult' : origindexcolumn_valresult})
    logger = self.__logger_append(logger)

    #this just casts trainID_column as list
    if trainID_column is False:
      trainID_column = []
    elif isinstance(trainID_column, str):
      trainID_column = [trainID_column]
    elif not isinstance(trainID_column, list):
      #validated in _check_am_miscparameters
      pass

    #this just casts testID_column as list
    if testID_column is False:
      testID_column = []
    elif isinstance(testID_column, str):
      testID_column = [testID_column]
    elif not isinstance(testID_column, list):
      #validated in _check_am_miscparameters
      pass

    #we'll have convention that if any of trainID_column in df_test
    #then apply those trainID_column to test set even if weren't specified
    #using a comparable retain / erase convention consistent with train specification when applicable
    trainID_columns_in_df_test = False
    if trainID_column != []:
      for trainIDcolumn in trainID_column:
        if trainIDcolumn in df_test.columns:
          trainID_columns_in_df_test = True
          if trainIDcolumn not in testID_column:
            testID_column.append(trainIDcolumn)
            if trainIDcolumn in trainID_struckfeatures:
              testID_struckfeatures.append(trainIDcolumn)
            elif trainIDcolumn in trainID_retainedfeatures:
              testID_retainedfeatures.append(trainIDcolumn)

    testID_struck_includes_trainID_retained = False
    if len(set(testID_struckfeatures) & set(trainID_retainedfeatures)) > 0:
      testID_struck_includes_trainID_retained = True
      # if printstatus != 'silent':
      self.__autoprint('warning', 'error: testID_column specification resulted in struck ID columns')
      self.__autoprint('warning', 'that were retained ID columns in the automunge train set')
      self.__autoprint('warning', 'for ID columns:')
      self.__autoprint('warning', set(testID_struckfeatures) & set(trainID_retainedfeatures))
      self.__autoprint('warning', 'to specify retained ID columns can use the [list1, list2] form documented for testID_column parameter in read me.')
      self.__autoprint('warning', '')

    #now run a quick validation that each entry in trainID_column list present in df_train
    trainID_column_subset_of_df_train_valresult = False
    if not set(trainID_column).issubset(set(df_train)):
      trainID_column_subset_of_df_train_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: entries to trainID_column were not found in df_train")
      self.__autoprint('warning', "note that trainID_column can either be passed as string for single entry")
      self.__autoprint('warning', "or trainID_column can be passed as a list for multiple entries")
      self.__autoprint('warning', "")
    
    logger_dict['validations'].update({'trainID_column_subset_of_df_train_valresult' : trainID_column_subset_of_df_train_valresult,
                                   'trainID_columns_in_df_test' : trainID_columns_in_df_test,
                                   'testID_struck_includes_trainID_retained' : testID_struck_includes_trainID_retained,
                                   })
    logger = self.__logger_append(logger)
    
    #this is trainID_column before adding index columns
    trainID_column_features = trainID_column.copy()
    
    #non-range indexes we'll move into the ID sets for consistent shuffling and validation splits
    nonrange_extract_marker = False
    if type(df_train.index) != pd.RangeIndex:

      nonrange_extract_marker = True

      orig_index_names = list(df_train.index.names)

      #if df_train.index.names == [None]:
      if None in df_train.index.names:

        revised_index_names = self.__autocopy(orig_index_names)

        revised_index_names[orig_index_names.index(None)] = origindexcolumn

        df_train = df_train.rename_axis(revised_index_names)
        
      trainID_column = trainID_column + list(df_train.index.names)
      trainID_struckfeatures = trainID_struckfeatures + list(df_train.index.names)

      df_train = df_train.reset_index(drop=False)

    #now run a quick validation that each entry in testID_column list present in df_test
    testID_column_subset_of_df_test_valresult = False
    if not set(testID_column).issubset(set(df_test)):
      testID_column_subset_of_df_test_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: entries to testID_column were not found in df_test")
      self.__autoprint('warning', "note that testID_column can either be passed as string for single entry")
      self.__autoprint('warning', "or testID_column can be passed as a list for multiple entries")
      self.__autoprint('warning', "Note that testID_column is primarily intended for use when ID columns in df_test")
      self.__autoprint('warning', "are different than those ID columns from trainID_column.")
      self.__autoprint('warning', "")
    
    logger_dict['validations'].update({'testID_column_subset_of_df_test_valresult' : testID_column_subset_of_df_test_valresult})
    logger = self.__logger_append(logger)

    #for unnamed non-range index we'll rename as 'Orig_index_###' and include that in ID sets
    if type(df_test.index) != pd.RangeIndex or nonrange_extract_marker is True:

      orig_index_names = list(df_test.index.names)
      
      #if df_train.index.names == [None]:
      if None in df_test.index.names:

        revised_index_names = self.__autocopy(orig_index_names)

        revised_index_names[orig_index_names.index(None)] = origindexcolumn

        df_test = df_test.rename_axis(revised_index_names)
        
      testID_column = testID_column + list(df_test.index.names)
      testID_struckfeatures = testID_struckfeatures + list(df_test.index.names)

      df_test = df_test.reset_index(drop=False)

    #here we derive a range integer index for inclusion in the ID sets
    #indexcolumn is a string defined above as 'Automunge_index_###'
    df_train_tempID = pd.DataFrame({indexcolumn:range(0,df_train.shape[0])})
    tempIDlist = []
    
    #extract the ID columns from train set
    df_trainID = pd.DataFrame(df_train[trainID_column])
    
    df_train_tempID.index = df_trainID.index
      
    df_trainID = pd.concat([df_trainID, df_train_tempID], axis=1)
  
    for IDcolumn in trainID_struckfeatures:
      del df_train[IDcolumn]
    
    #then append the indexcolumn to trainID_column list for use in later methods
    trainID_column = trainID_column + [indexcolumn]
      
    del df_train_tempID

    #here we derive a range integer index for inclusion in the test ID sets
    tempIDlist = []
    df_test_tempID = pd.DataFrame({indexcolumn:range(0,df_test.shape[0])})
    
    #extract the ID columns from test set
    #decided to do this stuff even if there's a dummy set for df_test 
    #to ensure downstream stuff works
    df_testID = pd.DataFrame(df_test[testID_column])
    
    df_test_tempID.index = df_testID.index
    
    df_testID = pd.concat([df_testID, df_test_tempID], axis=1)

    for IDcolumn in testID_struckfeatures:
      del df_test[IDcolumn]
    
    #then append the indexcolumn to testID_column list for use in later methods
    # testID_column.append(indexcolumn)
    testID_column = testID_column + [indexcolumn]

    del df_test_tempID

    #_________________________________________________________
    #__WorkflowBlock: automunge validation and label set extraction
    #this includes extraction of rows associated with the valpercent parameter
    #which will be prepared with an internal postmunge(.) call after populating postprocess_dict
    #and then label sets are extracted based on labels_column parameter
    #if labels aren't present in df_train we create an empty dataframe
    #if labels aren't present in df_test we create a dummy set based on first row of training labels

    #ok now carve out the validation rows. We'll process these later
    #(we're processing train data from validation data seperately to
    #ensure no leakage)
    #note that a shuffle operation to df_train is performed as part of __df_split based on shuffletrain and randomseed
    #otherwise if shuffletrain not activated (and privacy_encode is not 'private') validation split based on bottom sequential rows
    #an additional shuffle operation may be conducted later in the workflow for df_train and/or df_test based on shuffletrain
    totalvalidationratio = valpercent

    if isinstance(totalvalidationratio, float) and totalvalidationratio > 0.0:
      
      if shuffletrain in {True, 'traintest'} or privacy_encode == 'private':
        shuffle_param=True
      else:
        shuffle_param=False

      #we'll wait to split out the validation labels
      df_train, df_validation1 = \
      self.__df_split(df_train, totalvalidationratio, shuffle_param, randomseed)

      if trainID_column is not False:
        df_trainID, df_validationID1 = \
        self.__df_split(df_trainID, totalvalidationratio, shuffle_param, randomseed)

      else:
        df_trainID = pd.DataFrame()
        df_validationID1 = pd.DataFrame()
      
    elif isinstance(totalvalidationratio, tuple) and len(totalvalidationratio) == 2:
      
      totalvalidationratio = valpercent[1] - valpercent[0]
      
      if shuffletrain in {True, 'traintest'} or privacy_encode == 'private':
        shuffle_param=True
      else:
        shuffle_param=False
        
      #we'll wait to split out the validation labels 
      #(it is ok that this randomseed might be reset, the labels don't need a comaprable ranodmseed)
      df_train, df_validation1 = \
      self.__df_split_specified(df_train, valpercent, shuffle_param, randomseed)

      if trainID_column is not False:
        df_trainID, df_validationID1 = \
        self.__df_split_specified(df_trainID, valpercent, shuffle_param, randomseed)

      else:
        df_trainID = pd.DataFrame()
        df_validationID1 = pd.DataFrame()

    #else if total validation was <= 0.0
    else:
      df_validation1 = pd.DataFrame()
      df_validationID1 = pd.DataFrame()

    #extract labels from train set
    #an extension to this function could be to delete the training set rows\
    #where the labels are missing or improperly formatted prior to performing\
    #this step
    #initialize a helper 
    labelspresenttrain = False
    labelspresenttest = False
    single_train_column_labels_case = False

    #if user passes labels_column as True, labels_column converted to final column (including in single column scenario)
    if labels_column is True:
      labels_column = trainlabels[-1]
      labels_column_listofcolumns = [labels_column]

    if labels_column is False:
      labelsencoding_dict = {}

    if labels_column is not False:
    
      df_labels = pd.DataFrame(df_train[labels_column_listofcolumns])

      for labels_column_listofcolumns_entry in labels_column_listofcolumns:
        del df_train[labels_column_listofcolumns_entry]
      labelspresenttrain = True

      #if we only had one (label) column to begin with we'll create a dummy train set
      if df_train.shape[1] == 0:
        df_train = df_labels[0:1].copy()
        single_train_column_labels_case = True

      #if the labels column is present in test set too
      if len(set(df_test) - set(labels_column_listofcolumns)) < len(set(df_test)):
        df_testlabels = pd.DataFrame(df_test[labels_column_listofcolumns])
        for labels_column_listofcolumns_entry in labels_column_listofcolumns:
          del df_test[labels_column_listofcolumns_entry]
        labelspresenttest = True

    if labelspresenttrain is False:
      df_labels = pd.DataFrame()
    if labelspresenttest is False:

      #we'll introduce convention that if no df_testlabels we'll create
      #a dummy set derived from df_label's first rows
      df_testlabels = df_labels[0:1].copy()
        
    #if we only had one (label) column to begin with we'll create a dummy test set
    if df_test.shape[1] == 0:
      df_test = df_testlabels[0:1].copy()

    #_________________________________________________________
    #__WorkflowBlock: automunge column validations and variable initializations
    #this includes various validations associated with df_train and df_test column headers
    #such as confirming consistent quantity, composition and order
    #we also initialize a few variables here
    #including initialization of the postprocess_dict
    #which is the dictionary passed through processing and returned from automunge(.)

    #confirm consistency of train an test sets

    #check number of columns is consistent
    validate_traintest_columnnumbercompare = False
    if df_train.shape[1] != df_test.shape[1]:
      validate_traintest_columnnumbercompare = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error, different number of columns in train and test sets")
      self.__autoprint('warning', "(This assessment excludes labels and ID columns.)")
      self.__autoprint('warning', "Note that if label column present in df_train and not df_test")
      self.__autoprint('warning', "it should be designated with labels_column parameter.")
      logger_dict['validations'].update({'validate_traintest_columnnumbercompare' : validate_traintest_columnnumbercompare})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_traintest_columnnumbercompare' : validate_traintest_columnnumbercompare})
    logger = self.__logger_append(logger)

    #check column headers are consistent (this works independent of order)
    columns_train = set(list(df_train))
    columns_test = set(list(df_test))
    validate_traintest_columnlabelscompare = False
    if columns_train != columns_test:
      validate_traintest_columnlabelscompare = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error, different column labels in the train and test set")
      self.__autoprint('warning', "(This assessment excludes labels and ID columns.)")
      logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
    logger = self.__logger_append(logger)

    column_labels_count = len(list(df_train))
    unique_column_labels_count = len(set(list(df_train)))
    validate_redundantcolumnlabels = False
    if unique_column_labels_count < column_labels_count:
      validate_redundantcolumnlabels = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error, redundant column labels found, each column requires unique label")
      logger_dict['validations'].update({'validate_redundantcolumnlabels' : validate_redundantcolumnlabels})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_redundantcolumnlabels' : validate_redundantcolumnlabels})
    logger = self.__logger_append(logger)

    columns_train = list(df_train)
    columns_test = list(df_test)
    validate_traintest_columnorder = False
    if columns_train != columns_test:
      validate_traintest_columnorder = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error, different order of column labels in the train and test set")
      self.__autoprint('warning', "(This assessment excludes labels and ID columns.)")
      logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
    logger = self.__logger_append(logger)

    #extract column lists again but this time as a list
    columns_train = list(df_train)
    columns_test = list(df_test)

    #create an empty dataframe to serve as a store for each column's NArows
    #the column id's for this df will follow convention from NArows of 
    #column+'_NArows' for each column in columns_train
    #these are used in the ML infill methods
    masterNArows_train = pd.DataFrame()
    masterNArows_test = pd.DataFrame()

    #any preprocessing on assignparam done here
    assign_param = assignparam

    #if sampling_dict['sampling_type'] specification was recieved other than default
    #if sampling_dict didn't include a 'sampling_resource_dict' entry
    #one is populated with an internal automunge(.) call on a subset of entries
    sampling_dict = \
    self.__initialize_sampling_report_dict(df_train, assigncat, 
                                             assignparam, transformdict, processdict, 
                                             sampling_dict, printstatus, numbercategoryheuristic,
                                             powertransform, evalcat, logger)

    #initialize postprocess_dict
    #the dictionary / list values are populated through processing
    #and the passed variables may be accessed in various support functions
    #the remainder of returned postprocess_dict entries 
    #are populated at completion of processing and infill
    #and used to store validaiton results internal to various support functions

    #this is the same postprocess_dict returned from automunge(.) and used as a key for postmunge(.)
    #after this point we only inspect process_dict as postprocess_dict['process_dict']
    #some entries will be reset before return, including entropy_seeds, random_generator, sampling_dict

    postprocess_dict = {'column_dict' : {}, #data structures populated during automunge processing functions with an entry for each intermediate and returned column.
                        'columnkey_dict' : {}, #data structure  that maps each input column with transformtion applied to any applied transforms and the associated returned columns
                        'origcolumn' : {}, #data structure that maps each origcolumn to the applied root category, lists of derived columns, and a single arbitrary column key. 
                        'orig_noinplace' : set(), #set of origcolumns where no inplace trasnform was applied so subject to a deletion operation in circle of life
                        'process_dict' : process_dict, #the full procss_dict including user passed processdict, later returned to just include inspected entries
                        'mlti_categories' : set(), #categories applied as part of the mlti trasnform. Since they are routed differently than other transforms we log them seperately for use to populate the mirror_dict for the returned transform_dict and process_dict
                        'printstatus' : printstatus, #printstatus parameter as passed to automunge(.). activates printouts. (note that error message printouts are active for False case and muted for 'silent'
                        'randomseed' : randomseed, #If user passed a randomseed to automunge(.) this is value as passed, otherwise if user did not specify this is a random seed based on a random draw.
                        'application_number' : application_number, #selected based on a random draw independant of randomseed as random.randint(100000000000,999999999999), meant for use to distinguish between multiple automunge(.) applications to the same data
                        'autoMLer' : autoMLer, #data structure supporting ML infill that is initialized internally (not currenlty custom configurable) that designates which parent training / inference function is to be applied for different data type scenarios (e.g. continuous, ordinal, onehot, etc) as based on which autoML_type selected in ML_cmnd
                        'assign_param' : assign_param, #value of the assignparam parameter as passed to automunge(.)
                        'customML_inference_support' : {}, #comparable to the final form returned in ML_cmnd, used to support populating ML_cmnd, not returned in the final postprocess_dict to reduce memory overhead since is redundant with ML_cmnd
                        'test_plug_marker' : test_plug_marker, #marker for cases where df_test passed as False
                        }

    #perform any preparations on entropy_seeds and populate postprocess_dict with entropy_seed/sampling_dict/random_generator
    rowcount_train = df_train.shape[0]
    rowcount_test = df_test.shape[0]
    postprocess_dict = \
    self.__prepare_seeds(postprocess_dict, sampling_dict, random_generator, entropy_seeds, rowcount_train, rowcount_test, 'traintest', test_plug_marker)

    #now if we had a random random seed for pandas seeding, repopulate using entropy seeds in case we didn't have sufficient seeds earlier in workflow
    if randomrandomseed is True and sampling_dict['sampling_type'] != 'default' and populate_randomseed_expended_seed_count == 0:

      randomseed, randomrandomseed, entropy_seeds, _4 = \
      self.__populate_randomseed(False, postprocess_dict['entropy_seeds'], 
                                random_generator, sampling_dict)

      postprocess_dict.update({'randomseed' : randomseed})

    #we'll check if there are any protected features and process those columns in a seperate for loop
    #To ensure the features needing their basis have access in the input form
    #(in an alternate configuration the protected features could just be placed on far right of dataframe
    #this way works better when parallelizing the for loops)
    protected_features = self.__check_for_protected_features(assign_param, processdict)

    #we'll add to the protected_features any input column sets and update assigncat to treta them as null
    #and also log them in assign_param['messy_data_headers']
    protected_features, assigncat, assign_param, messy_set_overlap_valresult = \
    self.__messy_data_prep(protected_features, assigncat, assign_param, postprocess_dict, printstatus)

    logger_dict['validations'].update({'messy_set_overlap_valresult' : messy_set_overlap_valresult})
    logger = self.__logger_append(logger)

    #since some of __messy_data_prep appends set entries to protected_features
    #we'll have convention of sorting protected_features to ensure consistent order between automunge calls
    protected_features = sorted(protected_features)

    #quick validagtion result that a specified protected feature is foudn in data
    protected_features_in_headers_valresult = False
    if len(set(protected_features) - set(columns_train)) > 0:
      protected_features_in_headers_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error channel: a protected feature appears to have been specified in assignparam that was not found in df_train headers")
      self.__autoprint('warning', "this may also be a result of set bracket specification to assigncat having entries not found in df_train")
    logger_dict['validations'].update({'protected_features_in_headers_valresult' : protected_features_in_headers_valresult})
    logger = self.__logger_append(logger)
    
    #mirror assigncat which will populate the returned categories from eval function
    final_assigncat = self.__autocopy(assigncat)

    inverse_assigncat = self.__create_inverse_assigncat(assigncat)
    
    #create empty dictionary to serve as store for drift metrics
    drift_dict = {}

    #create mirror dictionary which will be populated with any inspected transform_dict and process_dict entries
    #so that the returned transform_dict and process_dict versions in postprocess_dict only contain inspected entries
    mirror_dict = {}

    #_________________________________________________________
    #__WorkflowBlock: automunge column processing
    #this for loop through the set of input feature columns applies transformations to train and test data
    #and includes deriving a root category based on either assigncat specification or evalcategory derivation
    #includes missing data injection for assignnan and consolidation to NaN representation
    #the NArows are aggregated based on the input column and root category and populated in masterNArows_train and masterNArows_test
    #the processfamily function applies transormations based on the root category's family tree
    #the circleoflife function manages data structures associated with and performs column replacements
    #the postprocess_dict['origcolumn'] entry is populated based on input column, derived columns, and category
    #the mirror_dict is populated for use to populate the returned form of transform_dict and process_dict in postprocess_dict
    
    #First we'll check if there are any protected features and process those columns in a seperate for loop
    #To ensure the features needing their basis have access in the input form
    #(in an alternate configuration the protected features could just be placed on far right of dataframe
    #this way works better when parallelizing the for loops)

    #columns_train_copy will strike protected features
    columns_train_copy = self.__autocopy(columns_train)
    if len(protected_features) > 0:
      for protected_feature in protected_features:
        columns_train_copy.remove(protected_feature)

    #__apply_processfamily aggregates the loop through column specific processing trees
    #including processfamily, evalcategory, getNArows, etc
    df_train, \
    df_test, \
    postprocess_dict, \
    mirror_dict, \
    final_assigncat, \
    masterNArows_train, \
    masterNArows_test = \
    self.__apply_processfamily(df_train,
                                df_test,
                                columns_train_copy,
                                postprocess_dict,
                                transform_dict,
                                mirror_dict,
                                assigncat,
                                inverse_assigncat,
                                final_assigncat,
                                assignnan,
                                masterNArows_train,
                                masterNArows_test,
                                randomseed,
                                eval_ratio,
                                numbercategoryheuristic,
                                powertransform,
                                evalcat,
                                drift_dict,
                                assign_param,
                                printstatus,
                                inplace,
                                logger,
                                labels = False,
                              )

    del columns_train_copy

    #_________________________________________________________
    #__WorkflowBlock: automunge label column processing
    #this for loop through the set of input label columns applies transformations to train and test data
    #and includes deriving a root category based on either assigncat specification or evalcategory derivation
    #includes missing data injection for assignnan and consolidation to NaN representation
    #does not include NArows aggregation (unless NArw included in family tree)
    #applies transformations and maintenance with processfamily and circleoflife
    #populates a postprocess_dict['origcolumn'] entry
    #populates a mirror_dict
    #also populates a labelsencoding_dict for potential use for public label inversion with encryption

    df_labels, \
    df_testlabels, \
    postprocess_dict, \
    mirror_dict, \
    final_assigncat, \
    masterNArows_train, \
    masterNArows_test = \
    self.__apply_processfamily(df_labels,
                                df_testlabels,
                                labels_column_listofcolumns, #when labels_column = False, labels_column_listofcolumns will be empty list []
                                postprocess_dict,
                                transform_dict,
                                mirror_dict,
                                assigncat,
                                inverse_assigncat,
                                final_assigncat,
                                assignnan,
                                masterNArows_train,
                                masterNArows_test,
                                randomseed,
                                eval_ratio,
                                numbercategoryheuristic,
                                powertransform,
                                evalcat,
                                drift_dict,
                                assign_param,
                                printstatus,
                                inplace,
                                logger,
                                labels = True,
                              )

    #prepare labelsencoding_dict for public label inversion with encryption in privacy_encode = True case
    #this also updates mirror_dict to include labelctgy
    labelsencoding_dict, mirror_dict, postprocess_dict  = \
    self.__prepare_labelsencoding_dict(postprocess_dict, processdict, transform_dict, mirror_dict, labels_column_listofcolumns, printstatus, logger)

    #_________________________________________________________
    #__WorkflowBlock: automunge protected feature processing
    #If any protected features were specified they are prepared here (after features and labels)
    #which serves purposes of ensuring they have a known configuration 
    #when other features that might inspect them are applied

    #by preparing protected_features in a seperate loop
    #it ensures they have a known configuration when being accessed in associated noise transforms
    if len(protected_features) > 0:

      df_train, \
      df_test, \
      postprocess_dict, \
      mirror_dict, \
      final_assigncat, \
      masterNArows_train, \
      masterNArows_test = \
      self.__apply_processfamily(df_train,
                                  df_test,
                                  protected_features,
                                  postprocess_dict,
                                  transform_dict,
                                  mirror_dict,
                                  assigncat,
                                  inverse_assigncat,
                                  final_assigncat,
                                  assignnan,
                                  masterNArows_train,
                                  masterNArows_test,
                                  randomseed,
                                  eval_ratio,
                                  numbercategoryheuristic,
                                  powertransform,
                                  evalcat,
                                  drift_dict,
                                  assign_param,
                                  printstatus,
                                  inplace,
                                  logger,
                                  labels = False,
                                )

    #_________________________________________________________
    #__WorkflowBlock: automunge infill derivation and insertion
    #Now that feature transformations have been applied, this segment performs infill
    #including ML infill or other assigninfill assignments
    #(note that an initial infill in most cases will already have been performed with processing functions)
    #postprocess_assigninfill_dict aggregates derived columns by infill type, such as based on parameters assigninfill, MLinfill
    #we then perform some data leakage mitigations for ML ifnill basis exclusions
    #including based on an evaluation of missing data correlation
    #or based on user specified exclusions
    #the infill is applied in __apply_am_infill
    #afterwards we do a little data structure cleanup for autoMLer and intialize some variables

    #now that we've pre-processed all of the columns, let's apply infill
    
    #printout display progress
    # if printstatus is True:
    self.__autoprint('debug', "______")
    self.__autoprint('debug', "")

    #This assembles the assignments of posttransform column headers for each infill type
    #(user is able to assign column headers in assigninfill without or without suffix appenders)
    #as used here columns_train is input columns, list(df_train) includes suffix appenders
    postprocess_assigninfill_dict = \
    self.__assemblepostprocess_assigninfill(assigninfill, list(df_train), \
                                          columns_train, postprocess_dict, MLinfill)

    #now we'll check for any signs of data leakage across features
    #as evidenced by high correlation of missing entries accross rows
    #these results will be appended to any user passed leakage_dict
    #where leakage_dict is for unidirectional specificaiton of ML infill basis exclusions
    ML_cmnd = self.__check_for_leakage(ML_cmnd, postprocess_dict, masterNArows_train, postprocess_assigninfill_dict)

    #now we convert leakage_dict to returned header convention, including derived and user defined entries
    #note this includes handling of ML_cmnd['full_exclude'] for ML infill basis exclusions from specification or MLinfilltype
    ML_cmnd = self.__convert_leakage_dict(ML_cmnd, postprocess_dict)

    #then we'll populate additional leakage_dict entries associated with bidirectional leakage_sets specification
    ML_cmnd = self.__convert_leakage_sets(ML_cmnd, postprocess_dict)

    #now apply infill
    df_train, df_test, postprocess_dict, infill_validations, sorted_columns_by_NaN_list, stop_count = \
    self.__apply_am_infill(df_train, df_test, postprocess_assigninfill_dict, \
                        postprocess_dict, infilliterate, printstatus, list(df_train), \
                        masterNArows_train, masterNArows_test, randomseed, ML_cmnd, logger)

    logger_dict['validations'].update(infill_validations)
    logger = self.__logger_append(logger)

    #now a cleanup to the postprocess_dict['autoMLer'] data structure to only record entries that were inspected for infill
    postprocess_dict, ML_cmnd = \
    self.__autoMLer_cleanup(postprocess_dict, ML_cmnd)

    #quickly gather a list of columns before any dimensionalioty reductions for populating mirror trees
    pre_dimred_finalcolumns_train = list(df_train)
    pre_dimred_finalcolumns_labels = list(df_labels)

    #_________________________________________________________
    #__WorkflowBlock: automunge feature importance dimensionality reduction
    #when featureselection was performed with a specified featurethreshold 
    #this triggers a sorting of columns by importance metric with prior feature importance application
    #the list of retained columns are provided as madethecut
    #and other columns are struck from the feature sets
    
    #Here's where we'll trim the columns that were stricken as part of featureselection method
    #if feature importance dimensionality reduction is applied based on featureselection and featurethreshold parameters
    
    if featureselection is not False:

      #get list of columns currently included
      currentcolumns = set(df_train)
      
      #this is to address an edge case for featureselection without diminsionality reduction
      if featureselection in {True, 'report'} or FSmodel is False \
      or (featureselection in {'pct'} and featurethreshold == 1.0) \
      or (featureselection in {'metric'} and featurethreshold == 0.0):
        madethecut = set(currentcolumns)
      
      madethecut = set(madethecut)
      
      #get list of columns to trim
      trimcolumns = currentcolumns - madethecut

      if len(trimcolumns) > 0:
        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Begin feature importance dimensionality reduction")
        self.__autoprint('debug', "")
        self.__autoprint('debug', "   method: ", featureselection)
        self.__autoprint('debug', "threshold: ", featurethreshold)
        self.__autoprint('debug', "")
        self.__autoprint('debug', "trimmed columns: ")
        self.__autoprint('debug', trimcolumns)
        self.__autoprint('debug', "")

      #trim columns
      for trimmee in trimcolumns:

        del df_train[trimmee]
        del df_test[trimmee]
        
      if len(trimcolumns) > 0:
        # if printstatus is True:
        self.__autoprint('debug', "returned columns: ")
        self.__autoprint('debug', list(df_train))
        self.__autoprint('debug', "")

    #_________________________________________________________
    #__WorkflowBlock: automunge PCA dimensionality reduction
    #PCA is performed based on automunge(.) parameter PCAn_components
    #which when activated is considered for selection of a final n_components through __evalPCA
    #applied towards numeric sets which are aggregated in __createPCAsets
    #validated as all valid numeric
    #and then trained in __PCAfunction
    #and concatinated

    #begin PCA dimensionality reduction if elected
    prePCAcolumns = list(df_train)
    
    #marker if PCA applied
    PCA_applied = False

    #PCA_transformed_columns are those that are fed to PCA
    PCA_transformed_columns = []

    #if user passed anything to automunge argument PCAn_components 
    #PCAn_components can be passed as float between 0-1, integer, False, or None
    #when PCAn_components is False _PCA not applied
    #when PCAn_components is None PCA only applied based on heuristic (as a funciton of # rows and # columns)

    PCAn_components_orig = PCAn_components

    if PCAn_components is not False:

      #run validation to ensure the df_train contains all valid numeric entries
      #since a value comparison is performed as part of _evalPCA
      PCA_train_first_numeric_data_result, PCA_train_first_all_valid_entries_result = \
      self.__validate_allvalidnumeric(df_train, printstatus)

      logger_dict['validations'].update({'PCA_train_first_numeric_data_result': PCA_train_first_numeric_data_result})
      logger_dict['validations'].update({'PCA_train_first_all_valid_entries_result': PCA_train_first_all_valid_entries_result})
      logger = self.__logger_append(logger)
        
      #also note for cases of heuristic inspection based on PCAn_components as None
      #if conditions of heuristic are met _evalPCA will derive a new n_components as integer
      #or otherwise return n_components as None indicating no PCA to be performed
      #(If heuristic not applied this will return n_components consistent with PCAn_components)
      PCActgy, n_components = \
      self.__evalPCA(df_train, PCAn_components, ML_cmnd)

    else:
      n_components = None

    #if PCAn_components != None:
    if n_components != None:

      #reset PCAn_components if a new n_components was derived based on heuristic
      #this will be the PCAn_components saved in postprocess_dict and inspected in postmunge
      PCAn_components = n_components

      #default is that ordinal and boolean integer columns are excluded from PCA
      #based on the ML_cmnd['PCA_cmnd']['bool_ordl_PCAexcl'] inititialized in _check_ML_cmnd
      #or user can just exlcude boolean but not ordinal by activating ML_cmnd['PCA_cmnd']['bool_PCA_excl']
      #or can include all columns in PCA by deactivating ML_cmnd['PCA_cmnd']['bool_ordl_PCAexcl']

      #Here add excluded columns to the PCAexcl list of columns based on bool_ordl_PCAexcl or bool_PCA_excl
      #which will be in addition to any excluded columns passed by user in PCAexcl
      #and bool_PCAexcl just tracks what columns were added

      if 'bool_PCA_excl' in ML_cmnd['PCA_cmnd'] \
      or 'bool_ordl_PCAexcl' in ML_cmnd['PCA_cmnd']:
        PCAexcl, bool_PCAexcl = self.__boolexcl(ML_cmnd, df_train, PCAexcl, postprocess_dict)
      else:
        bool_PCAexcl = []

      #only perform PCA if the specified/defrived number of columns < the number of
      #columns after removing the PCAexcl columns
      if n_components < (len(list(df_train)) - len(PCAexcl)) \
      and (n_components != 0) \
      and (n_components != None) \
      and (n_components is not False):

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Applying PCA dimensionality reduction")
        self.__autoprint('debug', "")
        self.__autoprint('debug', "Before PCA train set column count:")
        self.__autoprint('debug', len(df_train.columns))
        self.__autoprint('debug', '')
        
        #PCA applied marker set to true
        PCA_applied = True

        #this is to carve the excluded columns out from the set
        PCAset_train, PCAset_test, PCAexcl_posttransform = \
        self.__createPCAsets(df_train, df_test, PCAexcl, postprocess_dict, ML_cmnd)

        PCA_transformed_columns = list(PCAset_train)

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "Columns served to PCA:")
        self.__autoprint('debug', PCA_transformed_columns)
        self.__autoprint('debug', '')

        #run validation to ensure the PCA sets contain all valid numeric entries
        PCA_train_numeric_data_result, PCA_train_all_valid_entries_result = \
        self.__validate_allvalidnumeric(PCAset_train, printstatus)
  
        logger_dict['validations'].update({'PCA_train_numeric_data_result': PCA_train_numeric_data_result})
        logger_dict['validations'].update({'PCA_train_all_valid_entries_result': PCA_train_all_valid_entries_result})
        logger = self.__logger_append(logger)
      
        PCA_test_numeric_data_result, PCA_test_all_valid_entries_result = \
        self.__validate_allvalidnumeric(PCAset_test, printstatus)
  
        logger_dict['validations'].update({'PCA_test_numeric_data_result': PCA_test_numeric_data_result})
        logger_dict['validations'].update({'PCA_test_all_valid_entries_result': PCA_test_all_valid_entries_result})
        logger = self.__logger_append(logger)

        #this is to train the PCA model and perform transforms on train and test set
        PCAset_train, PCAset_test, postprocess_dict, PCA_columns_valresult = \
        self.__PCAfunction(PCAset_train, PCAset_test, n_components, PCActgy, postprocess_dict, \
                         randomseed, ML_cmnd)

        logger_dict['validations'].update({'PCA_columns_valresult': PCA_columns_valresult})
        logger = self.__logger_append(logger)

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "PCA model applied: ")
        self.__autoprint('debug', PCActgy)
        self.__autoprint('debug', "")

        PCA_suffixoverlap_results = \
        self.__df_check_suffixoverlap(df_train, list(PCAset_train), suffixoverlap_results = {}, printstatus = postprocess_dict['printstatus'])

        logger_dict['validations'].update({'PCA_suffixoverlap_results':PCA_suffixoverlap_results})
        logger = self.__logger_append(logger)

        #PCA_retain can be activated to retain columns serving as PCA basis
        PCA_retain = False
        if 'PCA_retain' in ML_cmnd and ML_cmnd['PCA_retain'] is True:
          PCA_retain = True
              
        if PCA_retain is False:
          #reattach the excluded columns to PCA set (retaining the original index from df_train)
          df_train = pd.concat([PCAset_train.set_index(df_train.index), df_train[PCAexcl_posttransform]], axis=1)
          df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test[PCAexcl_posttransform]], axis=1)
        if PCA_retain is True:
          #reattach the excluded columns to PCA set (retaining the original index from df_train)
          df_train = pd.concat([PCAset_train.set_index(df_train.index), df_train], axis=1)
          df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test], axis=1)

        returned_PCA_columns = list(PCAset_train)

        del PCAset_train
        del PCAset_test

        #here PCA drift stats are recorded for potential inspection with a postmunge driftreport
        if 'dimensionality_reduction_driftstats' not in postprocess_dict:
          postprocess_dict.update({'dimensionality_reduction_driftstats' : {}})
        postprocess_dict['dimensionality_reduction_driftstats'].update({
          'PCA_drift' : {'orig_stats' : self.__PCA_drift_stats(df_train, returned_PCA_columns)}
        })

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "returned PCA columns: ")
        self.__autoprint('debug', returned_PCA_columns)
        self.__autoprint('debug', "")
        self.__autoprint('debug', "After PCA train set column count:")
        self.__autoprint('debug', len(df_train.columns))
        self.__autoprint('debug', '')

      else:
        #else we'll just populate the PCAmodel slot in postprocess_dict with a placeholder
        postprocess_dict.update({'PCAmodel' : None})
        returned_PCA_columns = []
        PCAn_components = False
        PCA_retain = True

        logger_dict['validations'].update({'PCA_train_numeric_data_result': False})
        logger_dict['validations'].update({'PCA_train_all_valid_entries_result': False})
        logger_dict['validations'].update({'PCA_test_numeric_data_result': False})
        logger_dict['validations'].update({'PCA_test_all_valid_entries_result': False})
        logger_dict['validations'].update({'PCA_suffixoverlap_results':{}})
        logger = self.__logger_append(logger)

        if 'dimensionality_reduction_driftstats' not in postprocess_dict:
          postprocess_dict.update({'dimensionality_reduction_driftstats' : {}})

    else:
      #else we'll just populate the PCAmodel slot in postprocess_dict with a placeholder
      postprocess_dict.update({'PCAmodel' : None})
      returned_PCA_columns = []
      PCAn_components = False
      PCA_retain = True

      logger_dict['validations'].update({'PCA_train_numeric_data_result': False})
      logger_dict['validations'].update({'PCA_train_all_valid_entries_result': False})
      logger_dict['validations'].update({'PCA_test_numeric_data_result': False})
      logger_dict['validations'].update({'PCA_test_all_valid_entries_result': False})
      logger_dict['validations'].update({'PCA_suffixoverlap_results':{}})
      logger = self.__logger_append(logger)

      if 'dimensionality_reduction_driftstats' not in postprocess_dict:
        postprocess_dict.update({'dimensionality_reduction_driftstats' : {}})

    #_____

    #_________________________________________________________
    #__WorkflowBlock: automunge Binary dimensionality reduction (categoric consolidations)
    #categoric consolidations are performed to features based on the automunge(.) Binary parameter
    #or performed to labels based on specifying labels as a list with first entry set bracket specification
    #with application by way of __BinaryConsolidate

    #Binary dimensionality reduction to train and test data goes here
    df_train, df_test, Binary_dict, postprocess_dict, final_returned_Binary_columns, final_returned_Binary_sets, Binary_orig, Binary = \
    self.__BinaryConsolidate(df_train, df_test, Binary, postprocess_dict, 'Binary', logger)

    #record Binary drift stats for potential inspection with postmunge driftreport
    if Binary is not False:

      postprocess_dict['dimensionality_reduction_driftstats'].update({
        'Binary_drift' : {'orig_stats' : self.__Binary_drift_stats(df_train, Binary_dict)}
      })

    #Binary dimensionality reduction to df_labels and df_testlabels data goes here
    if isinstance(labels_column, list) and len(labels_column) > 1 and isinstance(labels_column[0], set):
      df_labels, df_testlabels, labels_Binary_dict, postprocess_dict, final_returned_labelBinary_columns, final_returned_labelBinary_sets, labelBinary_orig, labelBinary = \
      self.__BinaryConsolidate(df_labels, df_testlabels, labels_column, postprocess_dict, 'LabelBinary', logger)
    else:
      labels_Binary_dict = {}
      labelBinary_orig = False
      labelBinary = False
      final_returned_labelBinary_columns = []
      final_returned_labelBinary_sets = {'type' : {},
                                         '1010' : [],
                                         'onht' : [],
                                         'ord3' : []}
    labelsencoding_dict.update({'labels_Binary_dict' : labels_Binary_dict,
                                'labelBinary_orig' : labelBinary_orig,
                                'final_returned_labelBinary_columns' : final_returned_labelBinary_columns,
                                'final_returned_labelBinary_sets' : final_returned_labelBinary_sets})

    #store labels_Binary_dict in postprocess_dict to support _LabelFrequencyLevelizer
    postprocess_dict.update({'labels_Binary_dict' : labels_Binary_dict})

    if 'Binary_suffixoverlap_results' not in logger_dict['validations']:
      logger_dict['validations'].update({'Binary_suffixoverlap_results' : {}})
      logger = self.__logger_append(logger)

    #_____

    #_________________________________________________________
    #__WorkflowBlock: automunge duplicate row consolidation
    #duplicated row consolidations are optionally performed by the dupl_rows paraemter

    #grab rowcount serving as basis of drift stats (here since prior to oversampling or consolidations)
    train_rowcount = df_train.shape[0]
    test_rowcount = df_test.shape[0]

    #this is operation to consolidate duplicate rows based on dupl_rows parameter
    #in other words, if multiple copies of same row present only returns one
    if dupl_rows in {True, 'traintest'}:
      df_train, df_trainID, df_labels = self.__dupl_rows_consolidate(df_train, df_trainID, df_labels)
    if dupl_rows in {'test', 'traintest'}:
      df_test, df_testID, df_testlabels = self.__dupl_rows_consolidate(df_test, df_testID, df_testlabels)

    #_________________________________________________________
    #__WorkflowBlock: automunge label rebalancing (e.g. oversampling)
    #oversampling is optionally performed based on inspection of the TrainLabelFreqLevel parameter
    #applied in function __LabelFrequencyLevelizer
    #and may optionally be applied to either or both of train and test data
    #note that oversampling is consistently applied between features, labels, and ID sets

    #here is the process to levelize the frequency of label rows in train data
    #based on TrainLabelFreqLevel parameter
    #consolidated label support pending
    if TrainLabelFreqLevel in {True, 'traintest'} \
    and labels_column is not False:

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin label rebalancing")
      self.__autoprint('debug', "")
      self.__autoprint('debug', "Before rebalancing train set row count = ")
      self.__autoprint('debug', df_labels.shape[0])
      self.__autoprint('debug', "")

      if trainID_column is not False:
        df_train = pd.concat([df_train, df_trainID], axis=1)                        
        
      #apply LabelFrequencyLevelizer defined function
      df_train, df_labels = \
      self.__LabelFrequencyLevelizer(df_train, df_labels, postprocess_dict)

      #extract trainID
      if trainID_column is not False:

        df_trainID = pd.DataFrame(df_train[trainID_column])

        if isinstance(trainID_column, str):
          tempIDlist = [trainID_column]
        elif isinstance(trainID_column, list):
          tempIDlist = trainID_column
        for IDcolumn in tempIDlist:
          del df_train[IDcolumn]

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "")
      self.__autoprint('debug', "After rebalancing train set row count = ")
      self.__autoprint('debug', df_labels.shape[0])
      self.__autoprint('debug', "")

    if TrainLabelFreqLevel in {'test', 'traintest'} \
    and labelspresenttest is True:

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin test set label rebalancing")
      self.__autoprint('debug', "")
      self.__autoprint('debug', "Before rebalancing test set row count = ")
      self.__autoprint('debug', df_testlabels.shape[0])
      self.__autoprint('debug', "")

      if testID_column is not False:
        df_test = pd.concat([df_test, df_testID], axis=1)                        

      #apply LabelFrequencyLevelizer defined function
      df_test, df_testlabels = \
      self.__LabelFrequencyLevelizer(df_test, df_testlabels, postprocess_dict)
        
      #extract testID
      if testID_column is not False:

        df_testID = pd.DataFrame(df_test[testID_column])

        if isinstance(testID_column, str):
          tempIDlist = [testID_column]
        elif isinstance(testID_column, list):
          tempIDlist = testID_column
        for IDcolumn in tempIDlist:
          del df_test[IDcolumn]
          
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "")
      self.__autoprint('debug', "After rebalancing test set row count = ")
      self.__autoprint('debug', df_testlabels.shape[0])
      self.__autoprint('debug', "")

    #_________________________________________________________
    #__WorkflowBlock: automunge row shuffling
    #row shuffling is performed based on shuffletrain parameter
    #or amy also be performed in a privacy_encode scenario
    #and may optionally be applied to either or both of train and test data
    #note that shuffling is consistently applied between features, labels, and ID sets

    #then if shuffle was elected perform here
    if shuffletrain is True or shuffletrain == 'traintest' or privacy_encode == 'private':
      
      #shuffle training set and labels
      df_train = self.__df_shuffle(df_train, randomseed)
      
      if labels_column is not False:
        df_labels = self.__df_shuffle(df_labels, randomseed)

      if trainID_column is not False:
        df_trainID = self.__df_shuffle(df_trainID, randomseed)
      
    if shuffletrain == 'traintest' or privacy_encode == 'private':
      
      df_test = self.__df_shuffle(df_test, randomseed)
      
      if labelspresenttest is True:
        df_testlabels = self.__df_shuffle(df_testlabels, randomseed)

      if testID_column is not False:
        df_testID = self.__df_shuffle(df_testID, randomseed)

    #great the data is processed now let's do a few more global training preps

    #_________________________________________________________
    #__WorkflowBlock: automunge data type management
    #unlike integer types which are managed in transformation functions
    #float data types are collectively managed by the floatprecision parameter
    #unless deactivated for a category in dtype_convert process_dict entry
    #float transformations are applied by way of __floatprecision_transform
    #also any conversion to pandas categorical dtype if elected based on cat_type parameter performed here

    #now we'll apply the floatprecision transformation    
    #floatprecision adjustment only applied to columns returned from transforms
    #with mlinfilltype in {'numeric', 'concurrent_nmbr', 'exclude'}
    #when dtype_convert not deactivated for the associated category in processdict
    floatcolumns_train = list(df_train)
    floatcolumns_labels = list(df_labels)
    floatcolumns_train_copy = floatcolumns_train.copy()
    for floatcolumn in floatcolumns_train_copy:
      if floatcolumn not in returned_PCA_columns and \
      floatcolumn in postprocess_dict['column_dict'] and \
      (postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['MLinfilltype'] \
      not in {'numeric', 'concurrent_nmbr', 'exclude'} \
      or 'dtype_convert' in postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']] \
      and postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['dtype_convert'] is False):
        floatcolumns_train.remove(floatcolumn)
        
    floatcolumns_labels_copy = floatcolumns_labels.copy()
    for floatcolumn in floatcolumns_labels_copy:
      if floatcolumn not in returned_PCA_columns and \
      floatcolumn in postprocess_dict['column_dict'] and \
      (postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['MLinfilltype'] \
      not in {'numeric', 'concurrent_nmbr', 'exclude'} \
      or 'dtype_convert' in postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']] \
      and postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['dtype_convert'] is False):
        floatcolumns_labels.remove(floatcolumn)

    df_train = self.__floatprecision_transform(df_train, floatcolumns_train, floatprecision)
    if test_plug_marker is False:
      df_test = self.__floatprecision_transform(df_test, floatcolumns_train, floatprecision)
    if labels_column is not False:
      # finalcolumns_labels = list(df_labels)
      df_labels = self.__floatprecision_transform(df_labels, floatcolumns_labels, floatprecision)
      if labelspresenttest is True:
        df_testlabels = self.__floatprecision_transform(df_testlabels, floatcolumns_labels, floatprecision)
        
    #if cat_type was activated, convert df_train to pandas categorical based on MLinfilltype
    if cat_type is True:
      cat_columns_train = []
      for column in df_train.columns:
        if column in postprocess_dict['column_dict'] \
        and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
        in {'singlct', 'binary', 'multirt', '1010', 'concurrent_act', 'concurrent_ordl', 'boolexclude', 'ordlexclude'} \
        or column in final_returned_Binary_columns:
          
          cat_columns_train.append(column)
          df_train[column] = df_train[column].astype('category')
          
    #similar address for labels
    cat_columns_labels = []
    if cat_type is True:
      for column in df_labels.columns:
        if column in postprocess_dict['column_dict'] \
        and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
        in {'singlct', 'binary', 'multirt', '1010', 'concurrent_act', 'concurrent_ordl', 'boolexclude', 'ordlexclude'} \
        or column in final_returned_labelBinary_columns:
          
          cat_columns_labels.append(column)
          df_labels[column] = df_labels[column].astype('category')
    
    if cat_type is True:
      #now that df_train has final dtype configuration (other than privacy_encoding conversions to headers and order)
      #copy the dataframe with zero rows to store the dtype bassis in postprocess_dict
      dtype_df = df_train[cat_columns_train][:0].copy()
      dtype_labels_df = df_labels[cat_columns_labels][:0].copy()
    else:
      dtype_df = False
      dtype_labels_df = False
    
    #now if cat_type activated convert df_test and df_testlabels to comparable types with same categoric basis
    if cat_type is True:
      
      df_test[cat_columns_train] = df_test[cat_columns_train].astype(dtype_df[cat_columns_train].dtypes)
      df_testlabels[cat_columns_labels] = df_testlabels[cat_columns_labels].astype(dtype_labels_df[cat_columns_labels].dtypes)
    
    #_________________________________________________________
    #__WorkflowBlock: automunge excl suffix management
    #the excl (full passthrough) transform has a unique suffix convention in library
    #in that a suffix is appended in transformations (for data structure management)
    #and then removed in the returned data (signifying a full passthrough)
    #in addition to this header conversion, a series of conversion dictionaries are also populated
    #for use in header management associated with this special case
    #(note the suffix removal convention can be deactivated with excl_suffix parameter)

    #a special case, those columns that we completely excluded from processing and infill via 'excl' category
    #we'll scrub the suffix appender (unless user passed parameter excl_suffix=True)

    #Note that after this step any methods that use column headers to navigate data structures
    #will need to account fordiscrepency between returned headers without suffix
    #and data structure entries with suffix

    #to accomodate we assemble postprocess_dict entries of excl_columns_with_suffix and excl_columns_without_suffix
    #which are assembled independant of the excl_suffix parameter for reference
    #note that label columns are included in these lists

    #similarly we assemble conversion dictionaries in postprocess_dict
    #as excl_suffix_conversion_dict and excl_suffix_inversion_dict
    #these entries masy be applied in conjunction with self.__list_replace 
    #to convert a list of column headers to add the excl suffix (excl_suffix_conversion_dict) 
    #or to remove the excl suffix (excl_suffix_inversion_dict)
    #note that label columns are included in these dictionaries

    #we also create a redundant column_dict entry for header version without suffix to serve as reference
    #still if intending to build anything on top of returned data structures recomend just setting excl_suffix = True
    #which greatly simplifies things
    
    #first let's create a list of excl columns with and without suffix, just in case might come in handy
    #by convention excl is unique in that it has a hard coded suffix appender designation in the transformation function
    postprocess_dict.update({'excl_columns_with_suffix':[], 'excl_columns_without_suffix':[]})
    for cd_column in postprocess_dict['column_dict']:
      if postprocess_dict['column_dict'][cd_column]['category'] == 'excl':
        postprocess_dict['excl_columns_with_suffix'].append(cd_column)
        postprocess_dict['excl_columns_without_suffix'].append(cd_column[:-5])

    #these entries may be applied in conjunction with self.__list_replace 
    #to convert a list of column headers to add the excl suffix (excl_suffix_conversion_dict) or to remove the excl suffix (excl_suffix_inversion_dict)
    postprocess_dict.update({'excl_suffix_conversion_dict' : dict(zip(postprocess_dict['excl_columns_without_suffix'], postprocess_dict['excl_columns_with_suffix']))})
    postprocess_dict.update({'excl_suffix_inversion_dict' : dict(zip(postprocess_dict['excl_columns_with_suffix'], postprocess_dict['excl_columns_without_suffix']))})

    if excl_suffix is False:
      #run a quick suffix overlap validation before removing excl suffix
      excl_suffixoverlap_results = \
      self.__df_check_suffixoverlap(df_train, postprocess_dict['excl_columns_without_suffix'], suffixoverlap_results = {}, printstatus = postprocess_dict['printstatus'])
      logger_dict['validations'].update({'excl_suffixoverlap_results' : excl_suffixoverlap_results})
      logger = self.__logger_append(logger)
    else:
      logger_dict['validations'].update({'excl_suffixoverlap_results' : {}})
      logger = self.__logger_append(logger)

    #if excl_suffix is False we'll convert returned column headers to remove the '_excl' suffix
    if excl_suffix is False:

      df_train_columns = list(df_train)
      self.__list_replace(df_train_columns, postprocess_dict['excl_suffix_inversion_dict'])

      df_train.columns = df_train_columns
      df_test.columns = df_train_columns
      
      if labels_column is not False:

        df_labels_columns = list(df_labels)
        self.__list_replace(df_labels_columns, postprocess_dict['excl_suffix_inversion_dict'])

        df_labels.columns = df_labels_columns

        if labelspresenttest is True:
          df_testlabels.columns = df_labels_columns

    #_________________________________________________________
    #__WorkflowBlock: automunge populate postprocess_dict
    #the various returned entries are now fully populated in the postprocess_dict
    #in a few cases some entries will first need to be derived
    #including excluded_from_postmunge_getNArows, categorytree, inverse_categorytree, inputcolumn_dict
    #we also populate the returned column reports here including columntype_report, label_columntype_report, column_map
    #and strike and assignparam entries associated with entropy_seeds and random_generator

    #here's a list of final column names saving here since if a translation to 
    #numpy arrays is performed below based on pandasoutput it scrubs the column headers
    finalcolumns_train = list(df_train)
    finalcolumns_test = list(df_test)
    finalcolumns_labels = list(df_labels)

    #we'll create some tags specific to the application to support postprocess_dict versioning
    #note that we follow convention of using float equivalent strings as version numbers
    #to support backward compatibility checks
    #thus when reaching a round integer, the next version should be selected as int + 0.10 instead of 0.01
    automungeversion = '8.33'
#     application_number = random.randint(100000000000,999999999999)
#     application_timestamp = dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
    version_combined = '_' + str(automungeversion) + '_' + str(application_number)
                       #+ str(application_timestamp)

    #add automungeversion to ML_cmnd for potential access in inference functions
    ML_cmnd.update({'automungeversion' : automungeversion})

    #and consolidated all customML_inference_support entries from postprocess_dict
    ML_cmnd['customML_inference_support'].update(postprocess_dict['customML_inference_support'])
    #reset to empty dictionary
    postprocess_dict['customML_inference_support'] = {}

    #here we'll finish populating the postprocess_dict that is returned from automunge
    #as it will be used in the postmunge call below to process validation sets
    #note that some of the data structures will have been populated earlier such as column_dict etc
    postprocess_dict.update({'origtraincolumns' : columns_train, #list of origcolumns excluding any carved out as label or ID columns
                             'protected_features' : protected_features, #any specified protected_features (of input column headers) recieved in assign_param or processdict / defaultparams for the noise transform option
                             'origcolumns_all' : origcolumns_all, #list of origcolumns including any carved out as label or ID columns
                             'pre_dimred_finalcolumns_train' : pre_dimred_finalcolumns_train, #list of returned columns preceding any dimensionality reduction (as may be conducted by PCA or Binary), if no dimensionality reduction performed this will be same as finalcolumns_train
                             'finalcolumns_train' : finalcolumns_train, #list of returned columns. Note that excl columns matches convention of returned set (as may vary based on excl_suffix parameter). Excludes columns returned in label or ID sets.
                             'sorted_columns_by_NaN_list' : sorted_columns_by_NaN_list, #list of returned columns sorted by prevelance of missing data (from most to least). Used to determine order of ML infill application.
                             'labels_column' : labels_column, #labels_column automunge(.) parameter but with updates associated with string conversion and extraction of appropriate column for cases where labels_column passed as True (which extracts final column in set), otherwise returns as False. When labels_column was passed as a list it is returned here as a list.
                             'labels_column_listofcolumns' : labels_column_listofcolumns, #a list of label columns, for the single label column case this list has one entry. When labels_column was passed as a list this will be same except for omission of any set bracket specifications. This list is in the input header basis (without suffix appenders, preceding derivations).
                             'pre_dimred_finalcolumns_labels' : pre_dimred_finalcolumns_labels, #list of returned label columns preceding any dimensionality reduction (as may be conducted by categoric consolidations), if no dimensionality reduction performed this will be same as finalcolumns_labels. This list differs from labels_column_listofcolumns as it is in returned column basis (with suffix appenders, as returned after derivations).
                             'finalcolumns_labels' : finalcolumns_labels, #columns returned in the labels set
                             'single_train_column_labels_case' : single_train_column_labels_case, #boolean marker for cases where df_train was passed as a single column which was designqted as a label, otherwise returns False
                             'trainID_column_orig' : trainID_column_orig, #value of trainID_column as passed to automunge (as may be False, string, int, or list)
                             'trainID_column' : trainID_column, #column headers of the returned train ID sets (which in addition to specified ID columns includes the Automunge_index and any extracted index columns from feature sets)
                             'trainID_struckfeatures' : trainID_struckfeatures, #list of features included in train ID sets struck from features
                             'trainID_retainedfeatures' : trainID_retainedfeatures, #list of features included in train ID sets retained in features
                             'trainID_column_features' : trainID_column_features, #list of features included in ID sets (excluding index columns)
                             'finalcolumns_trainID' : list(df_trainID), #column headers of the returned ID sets (which in addition to specified ID columns includes the Automunge_index). This is redundant with trainID_column entry
                             'testID_column_orig' : testID_column_orig, #value of testID_column as passed to automunge (as may be string or list)
                             'testID_column' : testID_column, #value of testID_column as passed to automunge (as may be string or list)
                             'testID_struckfeatures' : testID_struckfeatures, #list of features included in test ID sets struck from features
                             'testID_retainedfeatures' : testID_retainedfeatures, #list of features included in test ID sets retained in features
                             'indexcolumn' : indexcolumn, #the new column created to serve as index column in the ID sets. Usually this is the string 'Automunge_index' except for cases where that string was already included in the ID sets
                             'origindexcolumn' : origindexcolumn, #when df_train had an unnamed non ranged integer index this is header in the returned ID sets
                             'nonrange_extract_marker' : nonrange_extract_marker, #for cases where automunge carried index columns over to ID set this marker tells postmunge to attempt as well
                             'valpercent' : valpercent, #value of the valpercent parameter as passed to automunge(.)
                             'floatprecision' : floatprecision, #value of the floatprecision parameter as passed to automunge(.)
                             'shuffletrain' : shuffletrain, #value of the shuffletrain parameter as passed to automunge(.)
                             'TrainLabelFreqLevel' : TrainLabelFreqLevel, #value of the TrainLabelFreqLevel parameter as passed to automunge(.)
                             'MLinfill' : MLinfill, #value of the MLinfill parameter as passed to automunge(.)
                             'infilliterate' : infilliterate, #value of the infilliterate parameter as passed to automunge(.)
                             'stop_count' : stop_count, #the final number of infilliterate iterations performed, may be different than infilliterate in cases where early stopping was evlauated based on ML_cmnd['halt_iterate'] (will always be <= infilliterate). This is the count of iterations for infilliterate performed in postmunge.
                             'eval_ratio' : eval_ratio, #value of the eval_ratio parameter as passed to automunge(.)
                             'powertransform' : powertransform, #value of the powertransform parameter as passed to automunge(.)
                             'binstransform' : binstransform, #value of the binstransform parameter as passed to automunge(.)
                             'numbercategoryheuristic' : numbercategoryheuristic, #value of the numbercategoryheuristic heuristic as passed to automunge(.)
                             'noise_augment' : noise_augment, #value of the noise_augment parameter as passed to automunge(.)
                             'pandasoutput' : pandasoutput, #value of the pandasoutput parameter as passed to automugne(.)
                             'orig_headers' : orig_headers, #value of orig_headers paraemter passed to automunge(.), True results in returned dataframes with input column headers (and may include redundent headers)
                             'privacy_encode' : privacy_encode, #value of the privacy_encode parameter passed to automunge(.), True or 'private' results in feature anonymizing steps (e.g. shuffling, masked column headers)
                             'NArw_marker' : NArw_marker, #value of the NArw_marker parameter as passed to automunge(.)
                             'featureselection' : featureselection, #value of the featureselection parameter as passed to automunge(.)
                             'featurethreshold' : featurethreshold, #value of the featurethreshold parameter as passed to automunge(.)
                             'FSmodel' : FSmodel, #When a feature selection model is trained, it is stored here, else False
                             'FScolumn_dict' : FScolumn_dict, #data structure that may serve as alternative to FS_sorted for returned feature selection results (when feature selection activated), as contains additional detail beyond the metric values reported in FS_sorted
                             'FS_sorted' : FS_sorted, #data structure for reporting feature selection results, including sorted results for metric by origcolumn and mewtric2 by returned column
                             'inplace' : inplace, #value of the inplace parameter as passed to automunge(.)
                             'drift_dict' : drift_dict, #data structure that supports the driftreport option in postmunge, which contains recorded drift metrics for both origcolumns and returned columns based on properties of the train set
                             'train_rowcount' : train_rowcount, #row count of df_train set (if dupl_rows option selected this is the row count before consolidation of rows)
                             'test_rowcount' : test_rowcount, #row count of df_test set (if dupl_rows option selected this is the row count before consolidation of rows)
                             'Binary' : Binary, #a Binary support data structure from converting all Binary scenarios to a common form of lists of lists (excluding the fist entry specification)
                             'Binary_orig' : Binary_orig, #value of the Binary parameter as passed to automunge(.)
                             'Binary_dict' : Binary_dict, #data structure supporting the consistent application of Binary dimnensionality reduction in postmunge. Contains first tier key of incremented integer and second tier a Binary_dict corresponding to each specified set of consolidation targets.
                             'returned_Binary_columns' : final_returned_Binary_columns, #list of columns returned from Binary dimensionality reduction, inclusive of all consolidation sets
                             'returned_Binary_sets' : final_returned_Binary_sets, #a dictionary that supports classificaiton of Binary returned columns in teh columntype_report
                             'labelBinary' : labelBinary, #comparable to Binary but for label consolidations, if labels_column did not include a set bracket specificaiton this will return as False
                             'labelBinary_orig' : labelBinary_orig, #when labels_column included a set bracket specificaiton this will match labels_column, otherwise returned as False
                             'final_returned_labelBinary_columns' : final_returned_labelBinary_columns, #comparable to returned_Binary_columns but for label consolidations
                             'final_returned_labelBinary_sets' : final_returned_labelBinary_sets, #comparable to returned_Binary_sets but for label consolidations
                             'PCA_applied' : PCA_applied, #boolean marker indicating if PCA dimensionality reduction was performed
                             'PCA_retain' : PCA_retain, #value initialized as False unless accessed from ML_cmnd['PCA_retain'], associated with specification for PCA with source column retention
                             'PCAn_components' : PCAn_components, #value of the PCAn_components parameter after taking account for any heuristic inspection applied when PCAn_components = None
                             'PCAn_components_orig' : PCAn_components_orig, #value of the PCAn_components parameter as originally passed to automunge(.)
                             'PCAexcl' : PCAexcl, #a list of columns excluded from PCA when applied based on PCAn_components. Note that list is automatically initialized with returned boolean integer and ordinal columns, and user can also pass additional columns to PCAexl parameter to include in this list
                             'prePCAcolumns' : prePCAcolumns, #a list of columns in returned train set preceding any applicaiton of PCA dimensionality reduction
                             'PCA_transformed_columns' : PCA_transformed_columns, #a list of columns fed to the PCA transform
                             'returned_PCA_columns' : returned_PCA_columns, #a list of columns returned as a part of PCA dimensionality reudction. (Basically, final returned columns = prePCAcolumns - PCAexcl + returned_PCA_columns), or for the PCA_retain scenario (final returned columns = prePCAcolumns + returned_PCA_columns)
                             'madethecut' : madethecut, #for cases when feature importance dimensionality reduction performed based on featureselection and featurethreshold parameters, this is the list of columns that made the cut and thus returned in the returned set
                             'excl_suffix' : excl_suffix, #value of the excl_suffix parameter as passed to automunge(.)
                             'traindata' : False, #This is a slot for use in postmunge. (traindata is a temporary entry in postmunge reverted to False upon return)
                             'assigncat' : assigncat, #value of the assigncat parameter as passed to automunge(.). (Note that if any formatting updates are made as part of validations (such as incorporation of list brackets for column assignments) they are included in this version.)
                             'inverse_assigncat' : inverse_assigncat, #inversion of the assigncat data structure provided for informational purposes (where column is key and category is value)
                             'final_assigncat' : final_assigncat, #value of assigncat after incorporating the category assignments performed under automation for those columns not otherwise assigned in initial assigncat
                             'assigninfill' : assigninfill, #value of the assigninfill parameter as passed to automunge(.)
                             'transform_dict' : mirror_dict['transform_dict'], #resulting from transform_dict after consolidating the internally intialized transform_dict with the user passed transformdict. Only returns entries that are inspected to apply transforms in the automunge(.) application.
                             'process_dict' : mirror_dict['process_dict'], #resulting from process_dict after consolidating the internally intialized process_dict with the user passed processdict. Only returns entries that are inspected to apply transforms in the automunge(.) application.
                             'postprocess_assigninfill_dict' : postprocess_assigninfill_dict, #final assigninfill after consolidating user passed assigninfill as well as addressing any unspecified infill sets to by applied with standardinfill or MLinfill basewd on MLinfill parameter. Note that includes an extra value in comparison to assigninfill as 'unspecified' which is list of columns that weren't otherwise assigned in user passed assigncat
                             'assign_param' : assign_param, #value of the assignparam parameter as passed to automunge(.)
                             'assignnan' : assignnan, #value of the assignnan parameter as passed to automunge(.)
                             'ML_cmnd' : ML_cmnd, #value of the ML_cmnd parameter after any initializations and conversions (such as based on _check_ML_cmnd or leakage_tolerance operations). This is recorded after ML infill.
                             'ML_cmnd_orig' : ML_cmnd_orig, #value of the ML_cmnd parameter as originally passed to automunge(.)
                             'cat_type' : cat_type, #value of the cat_type parameter as passed to automunge
                             'dtype_df' : dtype_df, #a copy of df_train with zero rows that stores pandas returned data type information (prior to any privacy encoding)
                             'dtype_labels_df' : dtype_labels_df, #a copy of df_labels with zero rows that stores pandas returned data type information (prior to any privacy encoding)
                             'randomrandomseed' : randomrandomseed, #boolean marker indicating if a random initialization of random seed was performed (True when user does not designate a randomseed)
                             'printstatus' : printstatus, #printstatus parameter as passed to automunge(.). activates printouts. (note that error message printouts are active even when this is off.)
                             'new_feature_ppd' : {}, #may be returned with one or more additional postprocess_dict's populated for new features added to a prior populated postprocess_dict based on the ppd_append parameter
                             'automungeversion' : automungeversion, #version number of Automunge library used to generate this postprocess_dict. Aligns with version numbers captured on github such as in rollout notes etc
                             'application_number' : application_number, #selected based on a random draw independant of randomseed as random.randint(100000000000,999999999999), meant for use to distinguish between multiple automunge(.) applications to the same data
                            #  'application_timestamp' : application_timestamp, #a string representation of the data and time of the automunge(.) application that generated this postprocess_dict. Useful to distinguish between cases when by fluke application_number is redundant between applications.
                             'version_combined' : version_combined}) #a single string concatinating the automungeversion and application_number

    #note that encrypt_key value is not stored, when encryption performed final returned postprocess_dict records encryption as True
    postprocess_dict.update({'encryption' : False}) #marker for whether encryption took place in automunge based on the encrypt_key parameter

    #redundant validation results for troubleshooting also available externally when user passes a logger_dict dictionary
    postprocess_dict['miscparameters_results'] = self.__autocopy(logger_dict['validations']) #data structure reporting results of all validation checks performed during automunge, such as to validate legal parameter entries or properties of passed data. For details see "validation results" tab in this same spreadsheet

    #support function to speed up postmunge when calling getNArows not needed
    excluded_from_postmunge_getNArows = \
    self.__assemble_excluded_from_postmunge_getNArows(postprocess_dict)

    postprocess_dict.update({'excluded_from_postmunge_getNArows' : excluded_from_postmunge_getNArows}) #a list of inputcolumns where postmunge does not need to apply getNArows for infill applicaiton. (included to speed up postmunge for cases when this support function not needed for infill)

    #mirror tree assembly functions go here, these mirror the progression of transformation functions
    #where categorytree is forward pass and inverse_categorytree is backward pass
    
    #don't currently use categorytree explicitly but populating in case downstream users find use
    categorytree = self.__populate_categorytree(postprocess_dict)
    
    #the inverse tree supports inversion in postmunge
    inverse_categorytree = self.__populate_inverse_categorytree(postprocess_dict)
    
    #we'll create another structure, this one flattened, similar to origcolumn or column_dict
    #this one as inputcolumn_dict
    inputcolumn_dict = self.__populate_inputcolumn_dict(postprocess_dict)
    
    #the trees are returned in postprocess_dict
    postprocess_dict.update({'categorytree' : categorytree, #an data structure that follows the generations of transformation categories and their associated columns realized through a forward pass of transformation applications. Currently this is just included as an informational resource.
                             'inverse_categorytree' : inverse_categorytree, #a data structure similar to an inversion of the categorytree, which follows in reverse the generations of transformation categories and associated columns realized thorugh a backward pass (inversion) to translate returned data back to the original form of data passed to automunge(.). Used in the postmunge inversion operation.
                             'inputcolumn_dict' : inputcolumn_dict}) #somewhat similar to columnkey_dict, currently this is just included as an informational resource, not used in workflow anywhere

    #populate a report for column types of returned set
    columntype_report = \
    self.__populate_columntype_report(postprocess_dict, postprocess_dict['finalcolumns_train'])
    postprocess_dict.update({'columntype_report' : columntype_report})

    label_columntype_report = \
    self.__populate_columntype_report(postprocess_dict, postprocess_dict['finalcolumns_labels'])
    postprocess_dict.update({'label_columntype_report' : label_columntype_report})

    #populate a map from received column headers to associated returned column headers
    column_map = \
    self.__populate_column_map_report(postprocess_dict)
    postprocess_dict.update({'column_map' : column_map})

    #populate a noise sampling report for use to determine entropy seed requirements
    postprocess_dict = self.__populate_sampling_report_dict(postprocess_dict, sampling_dict)

    #_________________________________________________________
    #__WorkflowBlock: automunge privacy encoding
    #privacy encode application is now performed based on privacy_encode parameter
    #this includes aggregation of the private headers (as integers) and conversion dictionaries
    #note that even when privacy encoding isn't performed the conversion dictionaries are still aggrevated for reference
    #if activated the encoding includes column shuffling (row shuffling may have been performed in the shuffling block)
    #anonymization of the columntype_report and label_columntype_report is also performed
    #and in 'private' scenario the indexes are reset (index even reset for ID sets, the original index can be recovered in ID set columns)
    #as an aside also perform a final labelsencoding_dict prep here

    #now if user selects privacy preserving encodings, we'll update the headers
    #since any received integers will have been converted to string, we'll use integers
    #for privacy headers to avoid overlap
    #and shuffle order of features in returned dataframe
    #alternate columntype_report available as private_columntype_report

    #if orig_headers was selected it is also performed here
    #note that privacy_encode takes precendence over orig_headers

    #if orig_headers parameter activated strip suffix appenders (privacy_encode takes precedence)
    #this primarily intended to support use case of integration into existing data pipeline
    if orig_headers is True and privacy_encode is False:
      
      df_train = self.__orig_headers_support(df_train, postprocess_dict)
      df_labels = self.__orig_headers_support(df_labels, postprocess_dict)
      
      df_test = self.__orig_headers_support(df_test, postprocess_dict)
      df_testlabels = self.__orig_headers_support(df_testlabels, postprocess_dict)

    if privacy_encode is not False:
      df_train = self.__df_shuffle(df_train, randomseed, axis=1)
      df_test = self.__df_shuffle(df_test, randomseed, axis=1)

    if privacy_encode == 'private':
      df_labels = self.__df_shuffle(df_labels, randomseed, axis=1)
      df_testlabels = self.__df_shuffle(df_testlabels, randomseed, axis=1)
    
    finalcolumns_train = list(df_train)
    finalcolumns_labels = list(df_labels)
    finalcolumns_trainID = list(df_trainID)
    finalcolumns_testID = list(df_testID)    
    
    privacy_headers_train = list(range(len(finalcolumns_train)))
    
    privacy_headers_train_dict = dict(zip(finalcolumns_train, privacy_headers_train))
    inverse_privacy_headers_train_dict = {value:key for key,value in privacy_headers_train_dict.items()}
    
    len_privacy_headers_train = len(privacy_headers_train)
    
    if postprocess_dict['labels_column'] is not False:

      privacy_headers_labels = list(range(len(finalcolumns_labels)))
      len_privacy_headers_labels = len(privacy_headers_labels)
      privacy_headers_labels = [x + len_privacy_headers_train for x in privacy_headers_labels]

      privacy_headers_labels_dict = dict(zip(finalcolumns_labels, privacy_headers_labels))
      inverse_privacy_headers_labels_dict = {value:key for key,value in privacy_headers_labels_dict.items()}
    
    else:
      
      privacy_headers_labels = []
      len_privacy_headers_labels = 0
      privacy_headers_labels_dict = {}
      inverse_privacy_headers_labels_dict = {}
    
    if privacy_encode is not False:
      df_train = df_train.rename(columns = privacy_headers_train_dict)
      df_test = df_test.rename(columns = privacy_headers_train_dict)
      
    if privacy_encode == 'private':
      df_labels = df_labels.rename(columns = privacy_headers_labels_dict)
      df_testlabels = df_testlabels.rename(columns = privacy_headers_labels_dict)
      
    #now record privacy encoding conventions in postprocess_dict
    # postprocess_dict.update({'privacy_encode' : privacy_encode}) #value of the privacy_encode parameter passed to automunge(.), True or 'private' results in feature anonymizing steps (e.g. shuffling, masked column headers)
    postprocess_dict.update({'privacy_headers_train' : privacy_headers_train}) #a list of columns that would be returned in train set if privacy encoding is elected based on privacy_encode parameter (where these are basically just sequential integers)
    postprocess_dict.update({'privacy_headers_train_dict' : privacy_headers_train_dict}) #a dictionary mapping returned columns without privacy_encode to returned columns with privacy_encode
    postprocess_dict.update({'inverse_privacy_headers_train_dict' : inverse_privacy_headers_train_dict}) #a dictionary mapping returned columns with privacy encode to returned columns without privacy encode (the inverse of privacy_headers_train_dict)
    postprocess_dict.update({'privacy_headers_labels' : privacy_headers_labels}) #comparable to privacy_headers_train but for columns in labels set
    postprocess_dict.update({'privacy_headers_labels_dict' : privacy_headers_labels_dict}) #comparable to privacy_headers_train_dict but for columns in labels set
    postprocess_dict.update({'inverse_privacy_headers_labels_dict' : inverse_privacy_headers_labels_dict}) #comparable to inverse_privacy_headers_train_dict but for columns in labels set

    #now generate a privacy_encoded verison of columntype_report
    private_columntype_report = self.__autocopy(postprocess_dict['columntype_report'])
    
    for key in columntype_report:

      if key not in {'onehot_sets', 'binary_sets'}:
        self.__list_replace(private_columntype_report[key], postprocess_dict['privacy_headers_train_dict'])
        random.shuffle(private_columntype_report[key])
        
      elif key in {'onehot_sets', 'binary_sets'}:  
        for sublist in private_columntype_report[key]:
          self.__list_replace(sublist, postprocess_dict['privacy_headers_train_dict'])
          random.shuffle(sublist)

    postprocess_dict.update({'private_columntype_report' : private_columntype_report})

    #now generate a privacy_encoded verison of label_columntype_report
    private_label_columntype_report = self.__autocopy(label_columntype_report)

    for key in private_label_columntype_report:

      if key not in {'onehot_sets', 'binary_sets'}:
        self.__list_replace(private_label_columntype_report[key], postprocess_dict['privacy_headers_labels_dict'])
        random.shuffle(private_label_columntype_report[key])

      elif key in {'onehot_sets', 'binary_sets'}:  
        for sublist in private_label_columntype_report[key]:
          self.__list_replace(sublist, postprocess_dict['privacy_headers_labels_dict'])
          random.shuffle(sublist)

    postprocess_dict.update({'private_label_columntype_report' : private_label_columntype_report})

    #if privacy_encode is not False, overwrite the columntype_report and label_columntype_report to the private version
    #and erase the column_map
    if privacy_encode is not False:
      postprocess_dict['columntype_report'] = private_columntype_report
      postprocess_dict['column_map'] = {}
    if privacy_encode == 'private':
      postprocess_dict['label_columntype_report'] = private_label_columntype_report

    #for the privacy_encode == 'private' scenario, row shuffling will have already been performed at shuffletrain application
    #now we'll reset index and reset Automunge_index in the ID set
    if privacy_encode == 'private':
      
      #this resets the dataframe index
      #note we are resetting the ID index as well so that it will match the feature and label indexes, 
      #with the ID set Automugne_index column available for original index information
      df_train = df_train.reset_index(drop=True)
      df_trainID = df_trainID.reset_index(drop=True)
      df_labels = df_labels.reset_index(drop=True)
      df_test = df_test.reset_index(drop=True)
      df_testID = df_testID.reset_index(drop=True)
      df_testlabels = df_testlabels.reset_index(drop=True)

      df_validation1 = df_validation1.reset_index(drop=True)
      df_validationID1 = df_validationID1.reset_index(drop=True)

      # #this resets the Automunge_index column returned in ID sets 
      # df_trainID[indexcolumn] = pd.DataFrame({indexcolumn:range(0,df_trainID.shape[0])})
      # df_testID[indexcolumn] = pd.DataFrame({indexcolumn:range(0,df_testID.shape[0])})
      # if totalvalidationratio > 0:
      #   df_validationID1[indexcolumn] = pd.DataFrame({indexcolumn:range(0,df_validationID1.shape[0])})

    #final labelsencoding_dict prep to support label inversion when other entries encrypted
    labelsencoding_dict = \
    self.__populate_labelsencoding_dict_support3(labelsencoding_dict, postprocess_dict)

    postprocess_dict.update({'labelsencoding_dict' : labelsencoding_dict})

    #_________________________________________________________
    #__WorkflowBlock: automunge noise_augment prep duplicate sets
    #when noise_augment is passed as integer greater than 0
    #the duplicate set that was extracted in #__WorkflowBlock: automunge noise_augment prep duplicate sets
    #is prepared in postmunge and concatinated onto the train, ID, and label sets
    #the convention is the first duplicate is prepared without noise
    #unless noise_augment is float(int) type
    #in which case all duplicates are prepared with noise
    #after concatination the duplicates are collectively shuffled based on shuffletrain parameter

    if noise_augment > 0:
      
      for i in range(int(noise_augment)):
        
        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Preparing noise_augment duplicate number ", i+1)
        self.__autoprint('debug', "")
      
        #if noise_augment was passed as integer dtype 
        #then first duplicate is prepared without noise
        #(if noise_augment is a float then all sets have noise)
        if i == 0 and isinstance(noise_augment, int):
          traindata = 'train_no_noise'
        else:
          traindata = True
          
        #note that np.random used for noise doesn't accept a random seed, this seed is for other operations
        augrandomseed = False
        if randomrandomseed is False:
          augrandomseed = randomseed

        #following postmunge call will return validation results in original bucket
        #which we'll store in noise_augment_validations
        orig_validations_dict = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}

        duplicate_i, duplicate_i_ID, duplicate_i_labels, \
        _1 = \
        self.postmunge(postprocess_dict, df_augment_duplicate,
                      printstatus = printstatus,
                      dupl_rows = dupl_rows,
                      traindata = traindata,
                      entropy_seeds = postprocess_dict['entropy_seeds'],
                      random_generator = random_generator,
                      sampling_dict = sampling_dict,
                      pandasoutput = 'dataframe',
                      randomseed = augrandomseed,
                      logger = logger)

        #logger_dict validation results stored in logger_dict['noise_augment_validations']
        logger_dict['noise_augment_validations'] = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}
        logger_dict['validations'] = self.__autocopy(orig_validations_dict)

        #this adjusts Automunge_index to avoid duplicates
        #indexcolumn is 'Automunge_index' from automunge or may be adjusted in cases of overlap
        #indexcolumn will be same in privacy_encode scenario (ID sets not anonymized)
        duplicate_i_ID[indexcolumn] += (df_trainID[indexcolumn].max() + 1)

        df_train = pd.concat([df_train, duplicate_i], axis=0, ignore_index=True)
        df_trainID = pd.concat([df_trainID, duplicate_i_ID], axis=0, ignore_index=True)

        if labels_column is not False:
          #single column labels returned from postmunge will be a dataframe
          df_labels = pd.concat([df_labels, duplicate_i_labels], axis=0, ignore_index=True)

        del duplicate_i, duplicate_i_ID, duplicate_i_labels, _1

      del df_augment_duplicate
      
      #this is an additional shuffle to the one performed in #__WorkflowBlock: automunge row shuffling
      #it results in the duplicates being intermingled
      #to maintain distinct sets can turn off shuffletrain
      #can still distinguish with shuffling by df_train.shape in comparison to Automunge_index
      if shuffletrain in {True, 'traintest'}:
        
        df_train = self.__df_shuffle(df_train, randomseed)
        df_trainID = self.__df_shuffle(df_trainID, randomseed)
        
        if labels_column is not False:
          df_labels = self.__df_shuffle(df_labels, randomseed)

    #_________________________________________________________
    #__WorkflowBlock: automunge validation data prep
    #when a validation set was extracted per valratio parameter
    #it is passed to postmunge to prepare using the recently populated postprocess_dict as the basis
    #note that if train data included labels they will still be included in df_validation1 prior to this postmunge application
    #we then perform some index management
    #as an aside we also adjacent here perform some cleanups 
    #to dataframes with scenario as populated with a dummy row, like df_test or in some cases the validaiton sets

    if totalvalidationratio > 0:

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin validation set processing with Postmunge")
      self.__autoprint('debug', "")

      #(postmunge shuffletrain not needed since validation data was already shuffled in __df_split if shuffletrain was activated)

      #to recover index (postmunge partitions non-range index into ID set)
      temp_valindex = df_validation1.index

      valrandomseed = False
      if randomrandomseed is False:
        valrandomseed = randomseed

      #following postmunge call will return validation results in original bucket
      #which we'll store in val_validations
      orig_validations_dict = self.__autocopy(logger_dict['validations'])
      logger_dict['validations'] = {'suffixoverlap_results':{}}

      #process validation set consistent to train set with postmunge here
      #note that traindata parameter consistent with postmunge(.) default (False)
      #and pandasoutput intentionally set to 'dataframe'
      df_validation1, _2, df_validationlabels1, _4 = \
      self.postmunge(postprocess_dict, df_validation1, testID_column = False, \
                     pandasoutput = 'dataframe', printstatus = printstatus, \
                     shuffletrain = False, dupl_rows = dupl_rows,
                     entropy_seeds = postprocess_dict['entropy_seeds'],
                     random_generator = random_generator,
                     sampling_dict = sampling_dict,
                     randomseed = valrandomseed,
                     logger=logger)

      #logger_dict validation results stored in logger_dict['val_validations']
      logger_dict['val_validations'] = self.__autocopy(logger_dict['validations'])
      logger_dict['validations'] = {'suffixoverlap_results':{}}
      logger_dict['validations'] = self.__autocopy(orig_validations_dict)

      df_validation1.index = temp_valindex
      if len(list(df_validationlabels1)) > 0:
        df_validationlabels1.index = temp_valindex

      del temp_valindex

    if totalvalidationratio <= 0.0:
      df_validation1 = pd.DataFrame()
      df_validationlabels1 = pd.DataFrame()
      df_validationID1 = pd.DataFrame()

    if testID_column is not False:
      df_testID = df_testID
    else:
      df_testID = pd.DataFrame()

    #now if user never passed a test set and we just created a dummy set 
    #then reset returned test sets to empty
    if test_plug_marker is True:
      df_test = pd.DataFrame()
      df_testID = pd.DataFrame()

    #if we created an temporary sets for test data we'll reset as empty here 
    elif labelspresenttest is False:
      df_testlabels = pd.DataFrame()

    #_________________________________________________________
    #__WorkflowBlock: automunge final preps and return
    #final preps include any numpy conversion based on pandasoutput parameter
    #any flattenting (like with ravel or casting as series) for single column sets
    #if ppd_append received a prior populated postprocess_dict the new version is combined
    #and any encryption of the postprocess_dict based on encrypt_key parameter
    #Automunge Complete

    #this defragments the dataframes
    if inplace is not True:
      df_train = df_train.copy()
      df_test = df_test.copy()

    #printout display progress
    # if printstatus in {True, 'summary'}:

    self.__autoprint('info', "______")
    self.__autoprint('info', "")
    self.__autoprint('info', "versioning serial stamp:")
    self.__autoprint('info', version_combined)
    self.__autoprint('info', "")

    self.__autoprint('info', "Automunge returned train column set: ")
    self.__autoprint('info', list(df_train))
    self.__autoprint('info', "")

    if df_trainID.empty is False:
      self.__autoprint('info', "Automunge returned ID column set: ")
      self.__autoprint('info', list(df_trainID))
      self.__autoprint('info', "")

    if df_labels.empty is False:
      self.__autoprint('info', "Automunge returned label column set: ")
      self.__autoprint('info', list(df_labels))
      self.__autoprint('info', "")
          
    #else set output to numpy arrays
    if pandasoutput is False:

      df_train = df_train.to_numpy()
      df_trainID = df_trainID.to_numpy()
      df_labels = df_labels.to_numpy()
      df_validation1 = df_validation1.to_numpy()
      df_validationID1 = df_validationID1.to_numpy()
      df_validationlabels1 = df_validationlabels1.to_numpy()
      df_test = df_test.to_numpy()
      df_testID = df_testID.to_numpy()
      df_testlabels = df_testlabels.to_numpy()

      #apply ravel to labels if appropriate - converts from eg [[1,2,3]] to [1,2,3]
      if df_labels.ndim == 2 and df_labels.shape[1] == 1:
        df_labels = np.ravel(df_labels)
      if df_validationlabels1.ndim == 2 and df_validationlabels1.shape[1] == 1:
        df_validationlabels1 = np.ravel(df_validationlabels1)
      if df_testlabels.ndim == 2 and df_testlabels.shape[1] == 1:
        df_testlabels = np.ravel(df_testlabels)

    #else flatten any single column label dataframes to series except for pandasoutput = 'dataframe' case
    elif pandasoutput is True:
      if len(df_labels.shape) > 1 and df_labels.shape[1] == 1:
        df_labels = df_labels[df_labels.columns[0]]
      if len(df_validationlabels1.shape) > 1 and df_validationlabels1.shape[1] == 1:
        df_validationlabels1 = df_validationlabels1[df_validationlabels1.columns[0]]
      if len(df_testlabels.shape) > 1 and df_testlabels.shape[1] == 1:
        df_testlabels = df_testlabels[df_testlabels.columns[0]]

    #then at completion of automunge(.), aggregate the suffixoverlap results
    #and do an additional printout if any column overlap error to be sure user sees message
    postprocess_dict = self.__suffix_overlap_final_aggregation_and_printouts(postprocess_dict, logger_dict, logger)

    #reset any entropy seeding entries which are specific to an automunge(.) or postmunge(.) call
    for entry in {'entropy_seeds', 'random_generator', 'sampling_dict'}:
      postprocess_dict[entry] = False

    #a cleanup to the validation results in cases of privacy encoding to remove leakage channel
    #note that the aggregated result is still available as suffixoverlap_aggregated_result
    if privacy_encode in {True, 'private'}:

      for valresult_leakage_channel in ['suffixoverlap_results', 
                                        'PCA_suffixoverlap_results', 
                                        'Binary_suffixoverlap_results',
                                        'excl_suffixoverlap_results']:

        if valresult_leakage_channel in postprocess_dict['miscparameters_results']:

          postprocess_dict['miscparameters_results'][valresult_leakage_channel] = {}
          #these are retained in logger_dict['validations']

    #if the prepared features are to be appended onto a prior populated postprocess_dict based on ppd_append
    if ppd_append is not False and isinstance(ppd_append, dict):
      
      postprocess_dict = \
      self.__ppd_append_support(ppd_append, postprocess_dict, encrypt_key, printstatus)

    #perform postprocess_dict encryption if elected by encrypt_key
    if encrypt_key is not False:
      postprocess_dict = \
      self.__encrypt_postprocess_dict(postprocess_dict, encrypt_key, privacy_encode, printstatus)

    #final consolidation of logger_dict with any externally initialized logger diciotnary
    logger.update(logger_dict)

    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Automunge Complete")
    self.__autoprint('info', "")

    return df_train, df_trainID, df_labels, \
    df_validation1, df_validationID1, df_validationlabels1, \
    df_test, df_testID, df_testlabels, \
    postprocess_dict

  #________________________________________________
  #02 - postmunge(.) related Function Blocks
  #________________________________________________

  #__FunctionBlock: postmunge process family functions

  def __postprocessfamily(self, df_test, column, origcategory, \
                        transform_dict, postprocess_dict, assign_param, logger):
    '''
    #as postmunge runs a for loop through each column, this is the  
    #first tier processing function applied which runs through the family primitives
    #populated in the transform_dict by assembletransformdict.
    #a little simpler than the processfamily version in automunge
    #since doesn't have to populate data structures, just applied transforms
    
    #we will run in order of
    #siblings, cousins, parents, auntsuncles
    '''

    inplaceperformed = False
    
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #when a replacement transform is applied
    final_upstream = False
    if len(transform_dict[origcategory]['auntsuncles']) == 0:
      if len(transform_dict[origcategory]['parents']) > 0:
        final_upstream = transform_dict[origcategory]['parents'][-1]
    else:
      if len(transform_dict[origcategory]['auntsuncles']) > 0:
        final_upstream = transform_dict[origcategory]['auntsuncles'][-1]
        
    #process the siblings (with downstream, supplemental)
    for sibling in transform_dict[origcategory]['siblings']:

  #       print("sibling = ", sibling)

      if sibling != None:
        #note we use the processparent function here
        df_test = \
        self.__postprocessparent(df_test, column, sibling, origcategory, final_upstream, \
                              transform_dict, postprocess_dict, assign_param, logger)
  
    #process the cousins (no downstream, supplemental)
    for cousin in transform_dict[origcategory]['cousins']:

  #       print("cousin = ", cousin)

      if cousin != None:
        #note we use the processsibling function here
        df_test = \
        self.__postprocesscousin(df_test, column, cousin, origcategory, final_upstream, \
                                transform_dict, postprocess_dict, assign_param, logger)
  
    #process the parents (with downstream, with replacement)
    for parent in transform_dict[origcategory]['parents']:

  #       print("parent = ", parent)

      if parent != None:
        df_test = \
        self.__postprocessparent(df_test, column, parent, origcategory, final_upstream, \
                              transform_dict, postprocess_dict, assign_param, logger)
        
    #process the auntsuncles (no downstream, with replacement)
    for auntuncle in transform_dict[origcategory]['auntsuncles']:

  #       print("auntuncle = ", auntuncle)

      if auntuncle != None:
        df_test = \
        self.__postprocesscousin(df_test, column, auntuncle, origcategory, final_upstream, \
                                transform_dict, postprocess_dict, assign_param, logger)

    return df_test

  def __postcircleoflife(self, df_test, column, category, \
                        transform_dict, postprocess_dict):
    '''
    This functino deletes source columns for family primitives that included replacement.
    '''

    #if we had replacement transformations performed on first generation \
    #then delete the original column
    if len(transform_dict[category]['auntsuncles']) \
    + len(transform_dict[category]['parents']) > 0:
      if column in postprocess_dict['orig_noinplace']:
        del df_test[column]

    #if we had replacement transformations performed on downstream generation \
    #then delete the associated parent column 
    for columndict_column in postprocess_dict['column_dict']:
      if postprocess_dict['column_dict'][columndict_column]['deletecolumn'] is True:

        #now we'll delete column
        #note this only worksa on single column  parents, need to incioroprate categorylist
        #for multicolumn parents (future extension)
        if columndict_column in df_test.columns:
          del df_test[columndict_column]

    return df_test

  def __postprocesscousin(self, df_test, column, cousin, origcategory, final_upstream, \
                       transform_dict, postprocess_dict, assign_param, logger):
    """
    #postprocesscousin is comparable to processcousin but applied in postmunge instead of automunge
    #a little simpler in that doesn't have to populate data structures, just applies transform functions
    #for dualprocess case applies postprocess instead of dualprocess
    
    #note that the presence of a custom_train processing function for a tree category
    #takes precedence over a postprocess function
    #which takes precedence over a singleprocess function
    """

    process_dict = postprocess_dict['process_dict']

    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == cousin:
      inplacecandidate = True
    
    params = self.__grab_params(assign_param, cousin, column, process_dict[cousin], postprocess_dict)

    #now populate params entries associated with entropy seeding for noise transforms
    params, postprocess_dict = \
    self.__random_parameters_params_append(params, postprocess_dict, \
                                          column, cousin, 'test', \
                                          df_test.shape[0], df_test.shape[0], \
                                          False, False, df_test)
    
    if inplacecandidate is True:
      if 'inplace_option' not in process_dict[cousin] \
      or 'inplace_option' in process_dict[cousin] \
      and process_dict[cousin]['inplace_option'] is True:
        if 'inplace' not in params:
          inplaceperformed = True
          params.update({'inplace' : True})
        elif ('inplace' in params and params['inplace'] != False):
          inplaceperformed = True
          params.update({'inplace' : True})
        else:
          inplaceperformed = False
    else:
      #user cannot manually specify inplace as True by design
      if ('inplace' in params and params['inplace'] is True):
        inplaceperformed = False
        params.update({'inplace' : False})
        
    #columnkey_list is a list of columns returned from the transform in automunge(.)
    #which may be used as a key to access the normalization_dict and etc in postmunge(.) transforms
    columnkey_list = []
    if column in postprocess_dict['columnkey_dict']:
      if cousin in postprocess_dict['columnkey_dict'][column]:
        columnkey_list = postprocess_dict['columnkey_dict'][column][cousin]
        
    #if this is a custom process function
    #(convention is that 'custom_train' is populated in both scenarios for dualprocess or singleprocess)
    if 'custom_train' in process_dict[cousin] \
    and process_dict[cousin]['custom_train'] != None:
      
      df_test = \
      self.__custom_postprocess_wrapper(df_test, column, postprocess_dict, columnkey_list, params)
    
    #elif this is a dual process function
    elif 'postprocess' in process_dict[cousin] \
    and process_dict[cousin]['postprocess'] != None:
      
      df_test = \
      process_dict[cousin]['postprocess'](df_test, column, postprocess_dict, \
                                            columnkey_list, params)

    #else if this is a single process function
    elif 'singleprocess' in process_dict[cousin] \
    and process_dict[cousin]['singleprocess'] != None:
            
      df_test, _1 = \
      process_dict[cousin]['singleprocess'](df_test, column, origcategory, \
                                            cousin, postprocess_dict, params)

    return df_test

  def __postprocessparent(self, df_test, column, parent, origcategory, final_upstream, \
                      transform_dict, postprocess_dict, assign_param, logger):
    """
    #postprocessparent is comparable to processparent but applied in postmunge instead of automunge
    #a little simpler in that doesn't have to populate data structures, just applies transform functions
    #for dualprocess case applies postprocess instead of dualprocess

    #we want to apply in order of
    #upstream process, niecesnephews, friends, children, coworkers
    
    #note that the presence of a custom_train processing function for a tree category
    #takes precedence over a postprocess function
    #which takes precedence over a singleprocess function
    """

    process_dict = postprocess_dict['process_dict']
    
    #this is used to derive the new columns from the trasform
    origcolumnsset = set(df_test)
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == parent:
      inplacecandidate = True

    params = self.__grab_params(assign_param, parent, column, process_dict[parent], postprocess_dict)

    #now populate params entries associated with entropy seeding for noise transforms
    params, postprocess_dict = \
    self.__random_parameters_params_append(params, postprocess_dict, \
                                          column, parent, 'test', \
                                          df_test.shape[0], df_test.shape[0], \
                                          False, False, df_test)
    
    if inplacecandidate is True:
      if 'inplace_option' not in process_dict[parent] \
      or 'inplace_option' in process_dict[parent] \
      and process_dict[parent]['inplace_option'] is True:
        if 'inplace' not in params:
          inplaceperformed = True
          params.update({'inplace' : True})
        elif ('inplace' in params and params['inplace'] != False):
          inplaceperformed = True
          params.update({'inplace' : True})
        else:
          inplaceperformed = False
    else:
      #user cannot manually specify inplace as True by design
      if ('inplace' in params and params['inplace'] is True):
        inplaceperformed = False
        params.update({'inplace' : False})

    #columnkey_list is a list of columns returned from the transform in automunge(.)
    #which may be used as a key to access the normalization_dict and etc in postmunge(.) transforms
    columnkey_list = []
    if column in postprocess_dict['columnkey_dict']:
      if parent in postprocess_dict['columnkey_dict'][column]:
        columnkey_list = postprocess_dict['columnkey_dict'][column][parent]
        
    #if this is a custom process function
    #(convention is that 'custom_train' is populated in both scenarios for dualprocess or singleprocess)
    if 'custom_train' in process_dict[parent] \
    and process_dict[parent]['custom_train'] != None:
      
      df_test = \
      self.__custom_postprocess_wrapper(df_test, column, postprocess_dict, columnkey_list, params)
    
    #elif this is a dual process function
    elif 'postprocess' in process_dict[parent] \
    and process_dict[parent]['postprocess'] != None:
            
      df_test = \
      process_dict[parent]['postprocess'](df_test, column, postprocess_dict, \
                                            columnkey_list, params)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[parent] \
    and process_dict[parent]['singleprocess'] != None:
      
      df_test, _1 = \
      process_dict[parent]['singleprocess'](df_test, column, origcategory, \
                                            parent, postprocess_dict, params)

    #this is used to derive the new columns from the transform
    newcolumnsset = set(df_test)
    
    #the " - {column}" is to accomodate both inplace scenarios
    support_newcolumns = (origcolumnsset ^ newcolumnsset) - {column}
    
    #downstream transform only performed if upstream did not return empty set
    if len(support_newcolumns) > 0:
      support_newcolumn = list(support_newcolumns)[0]
      support_categorylist = postprocess_dict['column_dict'][support_newcolumn]['categorylist']

      #to be consistent with automunge(.), parentcolumn is selected as first entry in upstream categorylist
      parentcolumn = support_categorylist[0]

      #final downstream transform from coworkers or children is elligible for inplace
      #when a replacement transform is applied
      final_downstream = False
      if len(transform_dict[parent]['coworkers']) == 0:
        if len(transform_dict[parent]['children']) > 0:
          final_downstream = transform_dict[parent]['children'][-1]
      else:
        if len(transform_dict[parent]['coworkers']) > 0:
          final_downstream = transform_dict[parent]['coworkers'][-1]

      #process any niecesnephews
      for niecenephew in transform_dict[parent]['niecesnephews']:

        if niecenephew != None:

          #process the niecenephew
          #note the function applied is postprocessparent (using recursion)
          df_test = \
          self.__postprocessparent(df_test, parentcolumn, niecenephew, origcategory, final_downstream, \
                                  transform_dict, postprocess_dict, assign_param, logger)

      #process any friends
      for friend in transform_dict[parent]['friends']:

        if friend != None:

          #process the friend
          #note the function applied is processcousin
          df_test = \
          self.__postprocesscousin(df_test, parentcolumn, friend, origcategory, final_downstream, \
                                  transform_dict, postprocess_dict, assign_param, logger)

      #process any children
      for child in transform_dict[parent]['children']:

        if child != None:

          #process the child
          #note the function applied is postprocessparent (using recursion)
          #parent column
          df_test = \
          self.__postprocessparent(df_test, parentcolumn, child, origcategory, final_downstream, \
                                  transform_dict, postprocess_dict, assign_param, logger)

      #process any coworkers
      for coworker in transform_dict[parent]['coworkers']:

        if coworker != None:

          #process the coworker
          #note the function applied is processcousin
          df_test = \
          self.__postprocesscousin(df_test, parentcolumn, coworker, origcategory, final_downstream, \
                                  transform_dict, postprocess_dict, assign_param, logger)

    return df_test

  def __custom_postprocess_wrapper(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    A wrapper for custom transformation functions applied in postmunge
    Where custom transformations follow templates of custom_test_template
    Or if custom_test_template not populated in process_dict then follows custom_train_template
    
    The purpose of this wrapper is to extend the minimized versions of custom transforms
    To include other conventions of the library
    Such as default infill, populating data structures, inplace option, suffix overlap detection, and etc
    
    The form of the _custom_process_wrapper inputs/outputs 
    will be similar to what is applied for postprocess functions
    
    If the processdict only has an entry for custom_train
    custom_train will be applied to mdf_test instead of custom_test
    Which will be similar to the singleprocess convention in the library
    
    Receives dataframe of a test set as mdf_test
    column is the received column that will serve as target for the transformation
    postprocess_dict is the dictionary data structure passed between transforms
    columnkey is a list of columns returned from the correspondihng tranforms applied in _custom_process_wrapper
    params are the parameters passed through assignparam associated with this specific categoy and column
    
    Returns the resulting transformed dataframe mdf_test
    
    Note that this wrapper includes application of a default infill per any processdict entry for 'defaultinfill'
    Or otherwise performs adjinfill as default
    As well as other infill conventions determined based on the NArowtype associated with the treecategory
    
    We'll also create seperately a similar wrapper for process functions applied in automunge (_custom_process_wrapper)
    And likewise a wrapper for custom inversions that may be performed in postmunge (_custom_inverseprocess_wrapper)
    """

    #normkey used to retrieve the normalization_dict
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
    
    #First we'll grab inplace parameter common to transformation functions
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
        
    #___
      
    #normkey is False when process function applied in automunge returned an empty set
    if normkey is not False:

      suffix = postprocess_dict['column_dict'][normkey]['custom_process_wrapper_dict']['suffix']
    
      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
        
      #treecategory is the category entry to a family tree primitive that served as basis for this transform
      treecategory = postprocess_dict['column_dict'][normkey]['category']
      NArowtype = postprocess_dict['process_dict'][treecategory]['NArowtype']
      MLinfilltype = postprocess_dict['process_dict'][treecategory]['MLinfilltype']
      
      #___
      
      #now apply default infill based on NArowtype

      #first to apply default infill we'll convert any nonvalid entries to nan
      
      if NArowtype in {'numeric'}:

        #convert all values to either numeric or NaN
        mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

      elif NArowtype in {'integer'}:

        #convert all values to either numeric or NaN
        mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

        #non integers are subject to infill
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn].round(), alternative=np.nan, specified='alternative')

      elif NArowtype in {'positivenumeric'}:

        #convert all values to either numeric or NaN
        mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

        #values <= 0 subject to infill
        mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan

      elif NArowtype in {'nonnegativenumeric'}:

        #convert all values to either numeric or NaN
        mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

        #values < 0 subject to infill
        mdf_test.loc[mdf_test[suffixcolumn] < 0, (suffixcolumn)] = np.nan

      elif NArowtype in {'nonzeronumeric'}:

        #convert all values to either numeric or NaN
        mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

        #values == 0 subject to infill
        mdf_test.loc[mdf_test[suffixcolumn] == 0, (suffixcolumn)] = np.nan

      elif NArowtype in {'datetime'}:

        #convert values to datetime
        mdf_test[suffixcolumn] = pd.to_datetime(mdf_test[suffixcolumn], errors = 'coerce', utc=True, format='mixed')

      elif NArowtype in {'justNaN', 'binary', 'exclude', 'totalexclude', 'parsenumeric'}:

        #nonvalid entries are already nan
        pass
      
      #___
      
      #Now that all nonvalid entries are cast as nan, we'll perform our default infill

      custom_process_wrapper_dict = postprocess_dict['column_dict'][normkey]['custom_process_wrapper_dict']
      defaultinfill_dict = custom_process_wrapper_dict['defaultinfill_dict']
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #___
      
      #great now we can apply the custom_test_template to the test data
      
      #normalization_dict is consistent with what was populated in the automunge call to custom_train
      normalization_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]

      #We'll have convention that if a corresponding custom_test_template wasn't populated
      #custom_test entry will either not be included or cast as None, in which case we apply custom_train to test data
      if 'custom_test' in postprocess_dict['process_dict'][treecategory] \
      and postprocess_dict['process_dict'][treecategory]['custom_test'] != None:

        mdf_test = \
        postprocess_dict['process_dict'][treecategory]['custom_test'](mdf_test, suffixcolumn, normalization_dict)

      else:

        mdf_test, _1 = \
        postprocess_dict['process_dict'][treecategory]['custom_train'](mdf_test, suffixcolumn, params)
      
      #___
      
      #we'll perform one more adjinfill application in case user defined transform had any unforseen edge cases
      
      if MLinfilltype not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

        newcolumns_list = postprocess_dict['column_dict'][normkey]['categorylist']
        
        #apply ffill to replace NArows with value from adjacent cell in preceding row
        mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(method='ffill')

        #we'll follow with a bfill just in case first row had a nan
        mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(method='bfill')

        #finally if prior infill still resulted in nan we'll just plug with 0
        mdf_test[newcolumns_list] = mdf_test[newcolumns_list].fillna(0)
      
      #___

      #now perform a datatype conversion based on MLinfilltype
      if custom_process_wrapper_dict['dtype_convert'] is True:
        
        newcolumns_list = postprocess_dict['column_dict'][normkey]['categorylist']

        if MLinfilltype in {'numeric', 'concurrent_nmbr', 'exclude'}:
          #datatype conversion performed elsewhere based on floatprecision parameter
          pass

        if MLinfilltype in {'binary', 'multirt', '1010', 'concurrent_act', 'boolexclude'}:
          #datatype cast as np.int8 since entries are boolean integers
          mdf_test[newcolumns_list] = mdf_test[newcolumns_list].astype(np.int8)

        if MLinfilltype in {'singlct', 'ordlexclude', 'concurrent_ordl'}:
          #ordinal sets are given a conditional dtype based on max activation
          #this can be deactivated in processdict with dtype_convert if desired

          max_encoding_for_dtype_convert = custom_process_wrapper_dict['max_encoding_for_dtype_convert']

          for newcolumn in newcolumns_list:

            max_encoding = max_encoding_for_dtype_convert[newcolumn]

            if max_encoding <= 255:
              mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint8)
            elif max_encoding <= 65535:
              mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint16)
            else:
              mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.uint32)

        if MLinfilltype in {'integer', 'totalexclude'}:
          #no conversion, assumes any conversion takes place in transformation function
          pass

      #___
      
    #this else corresponds to cases where train processing returned an empty set
    else:
      
      #if we didn't have a normkey but this transform was designated as inplace we'll delete column
      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def __apply_postprocessfamily(self,
                                df_test,
                                columns_train,
                                postprocess_dict,
                                driftreport,
                                transform_dict,
                                assign_param,
                                printstatus,
                                inplace,
                                logger,
                                labels = False,
                                masterNArows_test = pd.DataFrame(),
                                postdrift_dict = {},
                               ):
    """
    Aggregates the POSTprocessfamily functions and surrouding actions
    Including evalcategory, getNArows, etc
    the labels case is to distinguish between applying function to features or labels
    """

    for column in columns_train:
      
      category = postprocess_dict['origcolumn'][column]['category']
      
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "______")
      self.__autoprint('debug', "")
      if labels is False:
        self.__autoprint('debug', "processing column: ", column)
      else:
        self.__autoprint('debug', "processing labels column: ", column)
      self.__autoprint('debug', "    root category: ", category)
      self.__autoprint('debug', "")
      
      #assignnan application
      df_test = self.__assignnan_convert(df_test, column, category, postprocess_dict['assignnan'], postprocess_dict)
      
      #we also have convention that infinity and None values are by default subjected to infill
      convert_to_nan_list = [np.inf, -np.inf, None, float("NaN")]
      df_test = self.__convert_to_nan(df_test, column, category, postprocess_dict, convert_to_nan_list)
      
      if labels is False:
        
        #aggregating NArows may also serve purpose of aggregating drift statistics based on driftreport setting
        
        if driftreport in {'efficient', True}:
          testNArows, postdrift_dict = \
          self.__getNArows(df_test, column, category, postprocess_dict, postdrift_dict, True)

          # if printstatus is True:
          self.__autoprint('debug', "original source column drift stats:")
          self.__autoprint('debug', postprocess_dict['drift_dict'][column])
          self.__autoprint('debug', "")
          self.__autoprint('debug', "new source column drift stats:")
          self.__autoprint('debug', postdrift_dict[column])
          self.__autoprint('debug', "")

          #now append that NArows onto a master NA rows df
          if column not in postprocess_dict['excluded_from_postmunge_getNArows']:
            masterNArows_test = pd.concat([masterNArows_test, testNArows], axis=1)

        else:
          #if drift statistics aren't desired, only aggreagte those NArows needed for infill support
          if column not in postprocess_dict['excluded_from_postmunge_getNArows']:
            testNArows = self.__getNArows(df_test, column, category, postprocess_dict)
            
            #now append that NArows onto a master NA rows df
            masterNArows_test = pd.concat([masterNArows_test, testNArows], axis=1)
          
      #process family
      df_test = \
      self.__postprocessfamily(df_test, column, category, \
                               transform_dict, postprocess_dict, assign_param, logger)
      
      #delete columns subject to replacement
      df_test = \
      self.__postcircleoflife(df_test, column, category, \
                            transform_dict, postprocess_dict)
      
      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', " returned columns:")
      self.__autoprint('debug', postprocess_dict['origcolumn'][column]['columnkeylist'])
      self.__autoprint('debug', "")
        
      #this defragments the dataframe
      if inplace is not True:
        df_test = df_test.copy()

    return df_test, \
           postprocess_dict, \
           masterNArows_test, \
           postdrift_dict

  #__FunctionBlock: postmunge postprocess and custom_test functions
  
  def _postprocess_numerical(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_numerical(mdf_test, column, postprocess_dict, columnkey)
    #function to normalize data to mean of 0 and standard deviation of 1 from training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the mean and std from the train set \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of remaining values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      std = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['std']
      offset = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
      multiplier = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
      cap = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      abs_zero = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['abs_zero']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
        = cap
      
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
        = floor

      #subtract mean from column
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - mean

      #get standard deviation of training data
      std = std

      #divide column values by std
      #offset, multiplier are parameters that defaults to zero, one
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / std * multiplier + offset

      #convert any negative zero to zero (negative zero reserved for default infill)
      if abs_zero is True:
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == 0, 0., specified='replacement')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
    
  def _postprocess_MADn(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_MADn(mdf_test, column, postprocess_dict, columnkey)
    #function to normalize data to mean of 0 and mean absolute deviation of 1 from training distribution
    #takes as arguement pandas dataframe of test data (mdf_test)\
    #and the name of the column string ('column'), and the mean and MAD from the train set \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of remaining values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      MAD = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      center = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['center']
      maximum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      if center == 'mean':

        #subtract mean from column
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - mean
        
      elif center == 'max':
        
        #subtract maximum from column
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - maximum

      #get mean absolute deviation of training data
      MAD = MAD

      #divide column values by std
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / MAD

  #     #change data type for memory savings
  #     mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
    
  def _postprocess_mnmx(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mnmx(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      minimum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
      maximum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
      cap = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #avoid outlier div by zero when max = min
      maxminusmin = maximum - minimum
      if maxminusmin == 0:
        maxminusmin = 1

      #perform min-max scaling to test set using values from train
      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - minimum) / \
                                  (maxminusmin)

      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[suffixcolumn] > (cap - minimum)/maxminusmin, (suffixcolumn)] \
        = (cap - minimum)/maxminusmin
      
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[suffixcolumn] < (floor - minimum)/maxminusmin, (suffixcolumn)] \
        = (floor - minimum)/maxminusmin

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_mnm3(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mnmx(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #quantiles with values exceeding quantiles capped
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      quantilemin = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemin']
      quantilemax = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemax']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

  #     #get mean of training data
  #     mean = mean    
      
      #replace values > quantilemax with quantilemax
      mdf_test.loc[mdf_test[suffixcolumn] > quantilemax, (suffixcolumn)] \
      = quantilemax
      #replace values < quantile10 with quantile10
      mdf_test.loc[mdf_test[suffixcolumn] < quantilemin, (suffixcolumn)] \
      = quantilemin
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #avoid outlier div by zero when max = min
      maxminusmin = quantilemax - quantilemin
      if maxminusmin == 0:
        maxminusmin = 1

      #perform min-max scaling to test set using values from train
      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - quantilemin) / \
                                  (maxminusmin)

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_mxab(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_mxab(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of -1 and maximum of 1 \
    #based on division by max absolute values from training set.
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      maxabs = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxabs']
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #avoid outlier div by zero 
      if maxabs == 0:
        maxabs = 1

      #perform max abs scaling to test set using values from train
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / \
                                  (maxabs)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_retn(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_retn(mdf_train, mdf_test, column, category)
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #replaces missing or improperly formatted data with mean of remaining values
    
    #returns same dataframes with new column of name suffixcolumn
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      minimum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
      maximum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
      scalingapproach = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scalingapproach']
      offset = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
      multiplier = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
      cap = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      divisor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
        = cap
      
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
        = floor

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #avoid outlier div by zero when max = min
      maxminusmin = maximum - minimum
      if maxminusmin == 0:
        maxminusmin = 1
      
      if scalingapproach == 'retn':
        
        mdf_test[suffixcolumn] = (mdf_test[suffixcolumn]) / \
                                      (divisor) * multiplier + offset
        
      elif scalingapproach == 'mnmx':
      
        #perform min-max scaling to test set using values from train
        mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - minimum) / \
                                    (divisor) * multiplier + offset
        
      elif scalingapproach == 'mxmn':
      
        #perform min-max scaling to test set using values from train
        mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - maximum) / \
                                    (divisor) * multiplier + offset

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_mean(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mean(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      minimum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
      maximum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
      maxminusmin = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
      offset = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
      multiplier = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
      cap = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[suffixcolumn] > cap, (suffixcolumn)] \
        = cap
      
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[suffixcolumn] < floor, (suffixcolumn)] \
        = floor

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #perform min-max scaling to test set using values from train
      mdf_test[suffixcolumn] = (mdf_test[suffixcolumn] - mean) / \
                                  (maxminusmin) * multiplier + offset

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_binary(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_binary(mdf, column, postprocess_dict, columnkey)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_test), \
    #the name of the column string ('column') \
    #and the string classification to assign to missing data ('missing')
    #saved in the postprocess_dict
    #replaces original specified column in dataframe
    #returns transformed dataframe
    #missing category must be identical to one of the two existing categories
    #returns error message if more than two categories remain
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      binary_missing_plug = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['missing']
      str_convert = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
      onevalue = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][1]
      zerovalue = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][0]
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      if 'invert' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        invert = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['invert']
      else:
        #backward compatibility preceding 8.18
        invert = False

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
      
      if str_convert is True:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #replace missing data with specified classification
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(binary_missing_plug)

      #this addressess issue where nunique for mdftest > than that for mdf_train
      #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
      #for user specified infill, and as currently addressed will default to infill with most common value
      #in the mean time a workaround could be for user to manually replace extra values with nan prior to
      #postmunge application such as if they want to apply ML infill
      #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
      #if len(mdf_test[suffixcolumn].unique()) > 2:
      uniqueintest = mdf_test[suffixcolumn].unique()
      for unique in uniqueintest:
        if unique not in {onevalue, zerovalue}:
          mdf_test = \
          self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == unique, binary_missing_plug, specified='replacement')
      
      #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
      if invert is False:
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == onevalue, 1, 0)
      elif invert is True:
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == onevalue, 0, 1)

      #create list of columns
      bnrycolumns = [suffixcolumn]

      #change data types to 8-bit (1 byte) integers for memory savings
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.int8)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _custom_test_onht(self, df, column, normalization_dict):
    """
    #rewrite of the onht trasnform
    #corresponding to _custom_train_onht
    """

    #access the train set properties from normalization_dict
    str_convert = normalization_dict['str_convert']
    inverse_consolidation_translate_dict = normalization_dict['inverse_consolidation_translate_dict']
    labels_dict = normalization_dict['labels_dict']
    null_activation = normalization_dict['null_activation']
    
    missing_marker = None
    if null_activation == 'Binary':
      missing_marker = normalization_dict['missing_marker']
    
    #setting to object allows mixed data types for .replace operations and removes complexity of pandas category dtype
    df[column] = df[column].astype('object')
    
    #if str_convert elected (for common encoding between e.g. 2=='2')
    if str_convert is True:
      df = \
      self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')
    
    #if a consolidated_activations parameter was performed, we'll consolidated here
    if inverse_consolidation_translate_dict != {}:
      #apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)
      
    if null_activation is True:
      #if the test set has entries without encodings, we'll replace with missing data marker
      extra_entries = set(df[column].unique()) - set(labels_dict)
      extra_entries = list({x for x in extra_entries if x==x})
      if len(extra_entries) > 0:
        plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
        df[column] = df[column].astype('object').replace(plug_dict)
      
    #one hot encoding support function
    df_cat = self.__onehot_support(df, column, scenario=1, activations_list=list(labels_dict))
    
    #change data types to int8 for memory savings
    for activation_column in labels_dict:
      df_cat[activation_column] = df_cat[activation_column].astype(np.int8)
      
    #concatinate the sparse set with the rest of our training data
    df = pd.concat([df, df_cat], axis=1)
    
    del df[column]
    
    #now convert coloumn headers from text convention to onht convention
    df = df.rename(columns=labels_dict)
    
    return df

  def _postprocess_smth(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_smth(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #preprocess column with one hot encoding
    #same as 'text' transform except labels returned column with integer instead of entry appender
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      textlabelsdict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
      textcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      inputtextcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
      LSfitparams_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['LSfitparams_dict']
      activation = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['activation']
      LSfit = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['LSfit']
      testsmooth = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testsmooth']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']

      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if inplace is not True:
        for inputtextcolumn in inputtextcolumns:
          mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()

      else:
        mdf_test.rename(columns = textlabelsdict, inplace = True)

      #label smoothing only applied if traindata postmunge(.) parameter is True
      if traindata in {True, 'train_no_noise'} or testsmooth is True:

        categorycomplete_test_dict = dict(zip(textcolumns, [False]*len(textcolumns)))

        for labelsmoothingcolumn in textcolumns:

          if categorycomplete_test_dict[labelsmoothingcolumn] is False:

            mdf_test, categorycomplete_dict = \
            self.__postapply_LabelSmoothing(mdf_test, 
                                          labelsmoothingcolumn, 
                                          categorycomplete_test_dict, 
                                          LSfitparams_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _custom_test_GPS1(self, df, column, normalization_dict):
    """
    #corresponding to _custom_train_GPS1

    #intended for use in with GPS_convention == 'nonunique'
    #for cases where the set of coordinates in the test set are expected to be the same as the train set
    #such that parsing is not performed on test set entries

    #although still supports GPS_convention == 'default'
    #in which case each row is parsed
    """

    #access parameters from normalization_dict
    GPS_convention = normalization_dict['GPS_convention']
    comma_addresses = normalization_dict['comma_addresses']
    comma_count = normalization_dict['comma_count']
    latt_column = normalization_dict['latt_column']
    long_column = normalization_dict['long_column']

    #_____

    def default_GPS_parse_latt(coordinates1):
      return self.__GPS_parse(coordinates1, comma_addresses[0], comma_addresses[1], 'S')

    def default_GPS_parse_long(coordinates1):
      return self.__GPS_parse(coordinates1, comma_addresses[2], comma_addresses[3], 'W')

    #_____

    #in the default scenario, each row is parsed, which may be appropriate for all unique entries
    if GPS_convention == 'default':

      # df[column] = df[column].astype(str)

      df[latt_column] = pd.Series(df[column].astype(str)).transform(default_GPS_parse_latt)

      df[long_column] = pd.Series(df[column].astype(str)).transform(default_GPS_parse_long)
      
      del df[column]

    #_____
    
    #in the nonunique scenario, entries are not parsed, relying on assumption test set has same unique entries as train
    if GPS_convention == 'nonunique':

      latt_replace_dict = normalization_dict['latt_replace_dict']
      long_replace_dict = normalization_dict['long_replace_dict']

      #entries that weren't present in train set are translated to NaN
      unique_entries_test = list(df[column].astype(str).unique())
      for unique_entry_test in unique_entries_test:
        if unique_entry_test not in latt_replace_dict:
          latt_replace_dict.update({unique_entry_test : np.nan})
          long_replace_dict.update({unique_entry_test : np.nan})

      #now populate latt and long columns
      df[latt_column] = df[column].astype(str).astype('object').replace(latt_replace_dict)
      df[long_column] = df[column].astype(str).astype('object').replace(long_replace_dict)

      del df[column]
    
    return df

  def _postprocess_mlti(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #intended for applicant downstream of a concurrent_nmbr MLinfilltype encoding 
    #coresponds to _process_mlti
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      norm_category = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_category']
      norm_params = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_params']
      textlabelsdict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
      textcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      inputtextcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
      norm_columnkey_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_columnkey_dict']
      norm_column_dict_list = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_column_dict_list']
      dtype = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['dtype']
      max_encoding_for_dtype_convert_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['max_encoding_for_dtype_convert_dict']
      final_returned_columns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['final_returned_columns']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      
      #________

      #note that random_generator accessed from automunge(.) parameter and not passed to postmunge
      #postmunge(.) has a corresponding parameter to support
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64

#       #note that in the random_generator == False scenario 
#       #__random_parameters_params_append will have set to np.random.PCG64
#       nprandom_dict = {'custom' : random_generator,
#                        'default': np.random.PCG64}

      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
  
      #for entropy seeding support for mlti postprocess, 
      #need to add random_generator and sampling_resource_dict to norm_params
      #which are stored as norm_params = {inputcolumn : params}
      #for inputcolumn in textcolumns
      
      #traindata inspection is applicable to postmunge, in automunge will not be present
      if 'noise_transform' in postprocess_dict['process_dict'][norm_category] \
      and postprocess_dict['process_dict'][norm_category]['noise_transform'] is not False \
      and ( 'traindata' not in postprocess_dict or 'traindata' in postprocess_dict \
      and postprocess_dict['traindata'] not in {'train_no_noise', 'test_no_noise'}):
        
        for inputcolumn in textcolumns:

          norm_params[inputcolumn].update({'random_generator' : random_generator,
                                           'sampling_resource_dict' : sampling_resource_dict})
      
      #________
      
      if inplace is not True:
        for inputtextcolumn in inputtextcolumns:
          mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()

      else:
        mdf_test.rename(columns = textlabelsdict, inplace = True)
        
      #since postprocess_dict won't have normalziation_dict saved in same place, we'll populate a mirror norm_postprocess_dict
      norm_postprocess_dict = {'process_dict' : postprocess_dict['process_dict'],
                               'column_dict' : {},
                               'traindata' : postprocess_dict['traindata'],
                               'printstatus' : postprocess_dict['printstatus']}
      
      for norm_column_dict in norm_column_dict_list:
        norm_postprocess_dict['column_dict'].update(norm_column_dict)

      #if this is a custom process function
      #(convention is that 'custom_train' is populated in both scenarios for dualprocess or singleprocess)
      if 'custom_train' in postprocess_dict['process_dict'][norm_category] \
      and callable(postprocess_dict['process_dict'][norm_category]['custom_train']):
        
        for inputcolumn in textcolumns:
          
          #columnkey_list is a list of columns returned from the transform in automunge(.)
          #which may be used as a key to access the normalization_dict and etc in postmunge(.) transforms
          columnkey_list = []
          if inputcolumn in norm_columnkey_dict['columnkey_dict']:
            if norm_category in norm_columnkey_dict['columnkey_dict'][inputcolumn]:
              columnkey_list = norm_columnkey_dict['columnkey_dict'][inputcolumn][norm_category]
              
          #for entropy seeding we'll shuffle for each application
          if 'sampling_resource_dict' in norm_params[inputcolumn]:
            for entry in norm_params[inputcolumn]['sampling_resource_dict']:
              if entry[-5:] == 'seeds' \
              and isinstance(norm_params[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
                np.random.shuffle(norm_params[inputcolumn]['sampling_resource_dict'][entry])

          mdf_test = \
          self.__custom_postprocess_wrapper(mdf_test, inputcolumn, norm_postprocess_dict, columnkey_list, norm_params[inputcolumn])

        if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
        and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
          for textcolumn in textcolumns:
            del mdf_test[textcolumn]
        
      #elif this is a dual process function
      elif 'postprocess' in postprocess_dict['process_dict'][norm_category] \
      and callable(postprocess_dict['process_dict'][norm_category]['postprocess']):
        
        for inputcolumn in textcolumns:
          
          #columnkey_list is a list of columns returned from the transform in automunge(.)
          #which may be used as a key to access the normalization_dict and etc in postmunge(.) transforms
          columnkey_list = []
          if inputcolumn in norm_columnkey_dict['columnkey_dict']:
            if norm_category in norm_columnkey_dict['columnkey_dict'][inputcolumn]:
              columnkey_list = norm_columnkey_dict['columnkey_dict'][inputcolumn][norm_category]
              
          #for entropy seeding we'll shuffle for each application
          if 'sampling_resource_dict' in norm_params[inputcolumn]:
            for entry in norm_params[inputcolumn]['sampling_resource_dict']:
              if entry[-5:] == 'seeds' \
              and isinstance(norm_params[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
                np.random.shuffle(norm_params[inputcolumn]['sampling_resource_dict'][entry])

          mdf_test = \
          postprocess_dict['process_dict'][norm_category]['postprocess'](mdf_test, inputcolumn, norm_postprocess_dict, \
                                                                         columnkey_list, norm_params[inputcolumn])

        if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
        and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
          for textcolumn in textcolumns:
            del mdf_test[textcolumn]
          
      #else if this is a single process function
      elif 'singleprocess' in postprocess_dict['process_dict'][norm_category] \
      and callable(postprocess_dict['process_dict'][norm_category]['singleprocess']):
        
        for inputcolumn in textcolumns:
          
          #columnkey_list is a list of columns returned from the transform in automunge(.)
          #which may be used as a key to access the normalization_dict and etc in postmunge(.) transforms
          columnkey_list = []
          if inputcolumn in norm_columnkey_dict['columnkey_dict']:
            if norm_category in norm_columnkey_dict['columnkey_dict'][inputcolumn]:
              columnkey_list = norm_columnkey_dict['columnkey_dict'][inputcolumn][norm_category]
              
          #for entropy seeding we'll shuffle for each application
          if 'sampling_resource_dict' in norm_params[inputcolumn]:
            for entry in norm_params[inputcolumn]['sampling_resource_dict']:
              if entry[-5:] == 'seeds' \
              and isinstance(norm_params[inputcolumn]['sampling_resource_dict'][entry], type(np.array([]))):
                np.random.shuffle(norm_params[inputcolumn]['sampling_resource_dict'][entry])
              
          if inputcolumn in postprocess_dict['origcolumn']:
            origcategory = postprocess_dict['origcolumn'][inputcolumn]['category']
          else:
            origcategory = postprocess_dict['column_dict'][inputcolumn]['origcategory']

          mdf_test, _1 = \
          postprocess_dict['process_dict'][norm_category]['singleprocess'](mdf_test, inputcolumn, origcategory, \
                                                                           norm_category, norm_postprocess_dict, norm_params[inputcolumn])

        if 'inplace_option' in postprocess_dict['process_dict'][norm_category] \
        and postprocess_dict['process_dict'][norm_category]['inplace_option'] is False:
          for textcolumn in textcolumns:
            del mdf_test[textcolumn]

      if dtype in {'conditionalinteger', 'mlhs'}:

        if 'dtype_convert' not in postprocess_dict['process_dict'][norm_category] \
        or 'dtype_convert' in postprocess_dict['process_dict'][norm_category] \
        and postprocess_dict['process_dict'][norm_category]['dtype_convert'] is not False:

          for final_returned_column in final_returned_columns:

            max_encoding_for_dtype_convert = max_encoding_for_dtype_convert_dict[final_returned_column]

            if max_encoding_for_dtype_convert <= 255:
              mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint8)
            elif max_encoding_for_dtype_convert <= 65535:
              mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint16)
            else:
              mdf_test[final_returned_column] = mdf_test[final_returned_column].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        if column in postprocess_dict['column_dict']:
          inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
        else:
          #origcolumn scenario when not applied as downstream transform
          inputtextcolumns = [column]

        for inputtextcolumn in inputtextcolumns:
          del mdf_test[inputtextcolumn]
    
    return mdf_test

  def _postprocess_cnsl(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess correpsonding to _process_cnsl
    #in addition to assignparam testnoise parameter
    #noise injection may be performed based on postmunge traindata parameter
    #otherwise this is a passthrough transform
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      longest_len = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['longest_len']
      pad_character = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['pad_character']
      upstream_ordinal = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['upstream_ordinal']
      messy_data_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['messy_data_headers']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      inputtextcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
        
      #___
      
      textcolumn = column + '_' + suffix
      
      mdf_test[textcolumn] = ''
      
      #now consolidate with padding
      #we'll do two versions with and without integer conversion to remove float trailing 0's  
      for inputtextcolumn in inputtextcolumns:

        if upstream_ordinal is True:
          mdf_test[textcolumn] += mdf_test[inputtextcolumn].astype(int).astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)

        else:
          mdf_test[textcolumn] += mdf_test[inputtextcolumn].astype(str).str.pad(longest_len, side='left', fillchar=pad_character).str.slice(start=0, stop=longest_len)
      
    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        if column in postprocess_dict['column_dict']:
          inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
        else:
          #origcolumn scenario when not applied as downstream transform
          inputtextcolumns = [column]
          
        for inputtextcolumn in inputtextcolumns:
          del mdf_test[inputtextcolumn]
    
    return mdf_test

  def _postprocess_splt(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_splt(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #here in postprocess we only create columns foir those overlaps that were identified 
    #fromthe train set
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_splt']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict
      
      else:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_' + suffix + '_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_spl2(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_spl2(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and replaces entries with their redecued overlap
    #replaces entries without overlap to 0 (unique to spl5)
    #for example, if a categorical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created in which the entry 'north' replaced cells with north in their entries
    #(here for north and northeast)
    #and cells with west would be set to 0
    #returns as column titled origcolumn_spl2
    #missing values are ignored by default
    #this alternative to splt may be benficial for instance if one wanted to follow with an ordl encoding
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      consolidate_nonoverlaps = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['consolidate_nonoverlaps']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

#       newcolumns = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['spl2_newcolumns']

      #now for mdf_test we'll only consider those overlaps already identified from train set
  
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  break
                
      #then we'll populate the spl2 replacement dict

      spl2_test_overlap_dict = {}

      test_overlap_key_list = list(test_overlap_dict)

      test_overlap_key_list.sort()
      test_overlap_key_list.sort(key = len, reverse=True)

      for overlap_key in test_overlap_key_list:

        for entry in test_overlap_dict[overlap_key]:

          if entry not in spl2_test_overlap_dict:

            spl2_test_overlap_dict.update({entry : overlap_key})
            
      #here's where we identify values to set to 0 for spl5
      spl5_test_zero_dict = {}
      if consolidate_nonoverlaps is True:

        if test_same_as_train is True:
          unique_list_test = list(mdf_test[column].unique())
          unique_list_test = list(map(str, unique_list_test))

        for entry in unique_list_test:
          if entry not in spl2_test_overlap_dict:
            spl5_test_zero_dict.update({entry : 0})

#       newcolumns = []

#       for dict_key in overlap_dict:

      newcolumn = column + '_' + suffix

#         mdf_test[newcolumn] = mdf_test[column].isin(test_overlap_dict[dict_key])
#         mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

#       mdf_train[newcolumn] = mdf_train[column].copy()
      mdf_test[newcolumn] = mdf_test[column].copy()

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl2_test_overlap_dict)
      mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl5_test_zero_dict)

#       newcolumns.append(newcolumn)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_sp19(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_splt(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #here in postprocess we only create columns foir those overlaps that were identified 
    #fromthe train set
    
    #sp15 is comparable to splt but multiple concurrent activations allowed
    #so requires a different MLinfilltype in processdict
    
    #sp19 is comparable to sp15 but with a returned binary encoding aggregation
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sp19']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      _1010_binary_encoding_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
      _1010_binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
      _1010_activations_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
      categorylist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_sp15_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
        
      
      #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
      sp19_column = column + suffix
      
      mdf_test[sp19_column] = 'activations_'
      
      for entry in newcolumns:
        mdf_test[sp19_column] = mdf_test[sp19_column] + mdf_test[entry].astype(str)
        
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(_1010_binary_encoding_dict.keys())
      # labels_train.sort()
      labels_test = list(mdf_test[sp19_column].unique())
      labels_test.sort()
      
      #labels_test is a list of strings, insert missing data marker
      labels_test = labels_test + [np.nan]
        
      #replace the cateogries in train set via ordinal trasnformation
      mdf_test[sp19_column] = mdf_test[sp19_column].astype('object').replace(_1010_binary_encoding_dict)
      
      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))
      
      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
      mdf_test[sp19_column] = mdf_test[sp19_column].astype('object').replace(testplug_dict)
      
      #now we'll apply the 1010 transformation to the test set
      mdf_test[sp19_column] = mdf_test[sp19_column].astype('object').replace(_1010_binary_encoding_dict)
      
      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []
      
      for i in range(_1010_binary_column_count):

        _1010_columnlist.append(column + '_' + suffix + '_' + str(i))

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_test[_1010_column] = mdf_test[sp19_column].str.slice(i,i+1).astype(np.int8)

        i+=1
        
      #now delete the support column
      del mdf_test[sp19_column]
      for entry in newcolumns:
        del mdf_test[entry]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_sbst(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_sbst(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbst']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        unique_list_test = sorted(unique_list_test, key=len, reverse=True)

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_' + suffix + '_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_sbs3(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_sbst(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    
    #sbs3 is comparable to sbst but with a returned binary encoding aggregation
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbs3']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      _1010_binary_encoding_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
      _1010_binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
      _1010_activations_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
      categorylist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        unique_list_test = sorted(unique_list_test, key=len, reverse=True)

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_sbst_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, newcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
        
      
      #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
      
      sbs3_column = column + suffix
      
      mdf_test[sbs3_column] = 'activations_'
      
      for entry in newcolumns:
        mdf_test[sbs3_column] = mdf_test[sbs3_column] + mdf_test[entry].astype(str)
        
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(_1010_binary_encoding_dict.keys())
      # labels_train.sort()
      labels_test = list(mdf_test[sbs3_column].unique())
      # labels_test.sort()
      
      #labels_test is a list of strings, insert missing data marker
      labels_test = labels_test + [np.nan]
        
      #replace the cateogries in train set via ordinal trasnformation
      mdf_test[sbs3_column] = mdf_test[sbs3_column].astype('object').replace(_1010_binary_encoding_dict)
      
      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))
      
      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
      mdf_test[sbs3_column] = mdf_test[sbs3_column].astype('object').replace(testplug_dict)
      
      #now we'll apply the 1010 transformation to the test set
      mdf_test[sbs3_column] = mdf_test[sbs3_column].astype('object').replace(_1010_binary_encoding_dict)
      
      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []
      
      for i in range(_1010_binary_column_count):

        _1010_columnlist.append(column + '_' + suffix + '_' + str(i))

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_test[_1010_column] = mdf_test[sbs3_column].str.slice(i,i+1).astype(np.int8)

        i+=1
        
      #now delete the support column
      del mdf_test[sbs3_column]
      for entry in newcolumns:
        del mdf_test[entry]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_hash(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #applies the "hashing trick" to encode categoric sets
    #returning a set of columns with integers corresponding to words from set vocabulary
    #this is intended for sets with very high cardinality
    #note that the same integer may be returned in different columns 
    #for same word found in different entries
    #works by segregating entries into a list of words based on space seperator
    #stripping any special characters
    #and hashing each word with hashlib md5 hashing algorithm
    #which is converted to integer and taken remainder from a division by vocab_size
    #where vocab_size is based on heuristic
    #the heuristic derives vocab_size based on number of unique entries found in train set times the multipler
    #where if that result is greater than the cap then the heuristic reverts to the cap as vocab_size
    #where for hash the number of unique entries is calculated after extracting words from entries
    #note that if vocab_size is not large enough some of words may be returned with encoding overlap
    #returns set of columns with suffix appenders '_hash_#' where # is integer
    #unless only returning one column then suffix appender is just '_hash'
    #note that entries with fewer words than max word count are padded out with 0
    #also accepts parameter for excluded_characters, space
    #uppercase conversion if desired is performed externally by the UPCS transform
    #if space passed as '' then word extraction doesn't take place
    #user can manually specify a vocab_size with vocab-size parameter
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
    
      vocab_size = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['vocab_size_hash']
      max_length = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['max_length']
      excluded_characters = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['excluded_characters']
      space = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['space']
      salt = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['salt']
      hash_alg = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['hash_alg']
      max_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['max_column_count']
      hashcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['hashcolumns']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if hash_alg == 'md5':
        from hashlib import md5

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #convert column to string, note this means that missing data converted to 'nan'
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)
      
      #now scrub special characters
      for scrub_punctuation in excluded_characters:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)

      #define some support functions
      def _assemble_wordlist(string, space = ' ', max_column_count = max_column_count):
        """
        converts a string to list of words by splitting words from space characters
        assumes any desired special characters have already been stripped
        """

        wordlist = []
        j = 0
        
        if max_column_count is False:
          for i in range(len(string)+1):
            if i < len(string):
              if string[i] == space:
                if i > 0:
                  if string[j] != space:
                    wordlist.append(string[j:i])
                j = i+1

            else:
              if j < len(string):
                if string[j] != space:
                  wordlist.append(string[j:i])
        
        #else if we have a cap on number of returned columns
        else:
          wordlist_length = 0
          for i in range(len(string)+1):
            if i < len(string):
              if string[i] == space:
                if i > 0:
                  if string[j] != space:
                    wordlist.append(string[j:i])
                    wordlist_length += 1
                    if wordlist_length == max_column_count - 1:
                      j = i+1
                      break
                j = i+1

            else:
              if j < len(string):
                if string[j] != space:
                  wordlist.append(string[j:i])
                  wordlist_length += 1
                  j = i+1
                  if wordlist_length == max_column_count - 1:
                    break

          if wordlist_length == max_column_count - 1:
            if j < len(string):
              wordlist.append(string[j:len(string)])

        return wordlist

      def _md5_hash(wordlist):
        """
        applies an md5 hashing to the list of words
        this conversion to ingtegers is known as "the hashing trick"
        md5 is partly inspired by tensorflow keras_preprocessing hashing_trick function
        requires importing from hashlib import md5
        here n is the range of integers for vocabulary
        0 is reserved for use to pad lists of shorter length
        """
        if hash_alg == 'md5':
          return [int(md5((salt + word).encode()).hexdigest(), 16) % (vocab_size-1) + 1 for word in wordlist]
        else:
          return [hash(salt + word) % (vocab_size-1) + 1 for word in wordlist]

      def _pad_hash(hash_list):
        """
        ensures hashing lists are all same length by padding shorter length lists with 0
        """
        padcount = max_length - len(hash_list)
        if padcount >= 0:
          pad = []
          for i in range(padcount):
            pad = pad + [0]
          hash_list = hash_list + pad
        else:
          #for test data we'll trim if max_length greater than max_length from train data
          hash_list = hash_list[:max_length]

        return hash_list

      #now convert entries to lists of words
      #e.g. this converts "Two words" to ['Two', 'words']
      #if you don't want to split words can pass space = ''
      if space != '':
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_assemble_wordlist)
      else:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(lambda x: [x])

      #now apply hashing to convert to integers based on vocab_size
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_md5_hash)

      #the other entries are padded out with 0 to reach same length, if a train entry has longer length it is trimmed
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_pad_hash)

      if max_length > 1:

        hashcolumns = []
        for i in range(max_length):

          hash_column = column + '_' + suffix + '_' + str(i)

          hashcolumns += [hash_column]

          #now populate the column with i'th entry from hashed list
          mdf_test[hash_column] = mdf_test[suffixcolumn].transform(lambda x: x[i])

        #remove support column
        del mdf_test[suffixcolumn]

      else:
        hashcolumns = [suffixcolumn]

        #now populate the column with i'th entry from hashed list
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(lambda x: x[0])
        
      #returned data type is conditional on the size of encoding space
      for hashcolumn in hashcolumns:

        max_encoding = vocab_size - 1
        if max_encoding <= 255:
          mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint8)
        elif max_encoding <= 65535:
          mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint16)
        else:
          mdf_test[hashcolumn] = mdf_test[hashcolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_hs10(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #applies the "hashing trick" to encode categoric sets
    #returning a set of columns binary encoded corresponding to integers returned from hash
    #this is intended for sets with very high cardinality
    #note that the same activation set may be returned for different entries
    #works by hashing each entry with hashlib md5 hashing algorithm
    #which is converted to integer and taken remainder from a division by vocab_size
    #where vocab_size is passed parameter intended to align with vocabulary size defaulting to 128
    #note that if vocab_size is not large enough some of words may be returned with encoding overlap
    #returns set of columns with suffix appenders '_hs10_#' where # is integer
    #uppercase conversion if desired is performed externally by the UPCS transform
    #applies a heuristic to
    #set a vocab_size based on number unique entries times heuristic_multiplier parameter which defaults to 2
    #also accepts heuristic_cap parameter where if unique * heuristic_muyltipler > heuristic_cap
    #then vocab_size = heuristic_cap
    #or user can manually specify a vocab_size instead of relying on heuristic
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      vocab_size = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['vocab_size']
      binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['col_count']
      salt = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['salt']
      excluded_characters = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['excluded_characters']
      hash_alg = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['hash_alg']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if hash_alg == 'md5':
        from hashlib import md5
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
        
      #convert column to string, note this means that missing data converted to 'nan'
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(str)

      #now scrub special characters
      for scrub_punctuation in excluded_characters:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.replace(scrub_punctuation, '', regex=False)
      
      def _md5_hash(entry):
        """
        applies hashing to the list of words
        this conversion to ingtegers is known as "the hashing trick"
        requires importing from hashlib import md5 if hash_alg = "md5"
        here n is the range of integers for vocabulary
        """
        if hash_alg == 'md5':
          return int(md5((salt + entry).encode()).hexdigest(), 16) % (vocab_size)
        else:
          return hash(salt + entry) % (vocab_size)

      #now apply hashing to convert to integers based on vocab_size
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(_md5_hash)
      
      #convert integer encoding to binary
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].transform(bin)
      
      #convert format to string of digits
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str[2:]
      
      #pad out zeros
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].str.zfill(binary_column_count)
      
      hashcolumns = []
      for i in range(binary_column_count):

        hash_column = column + '_' + suffix + '_' + str(i)
        
        hashcolumns += [hash_column]
        
        #now populate the column with i'th entry from hashed list
        mdf_test[hash_column] = mdf_test[suffixcolumn].str[i].astype(np.int8)
      
      #remove support column
      del mdf_test[suffixcolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test
  
  def _postprocess_srch(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_srch(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_srch']
#       newcolumns_before_aggregation = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['newcolumns_before_aggregation']
#       search = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
#       search_preflattening = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_preflattening']
      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      case = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
        
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      for newcolumn in search_dict:

        mdf_test = \
        self.__autowhere(mdf_test, newcolumn, mdf_test[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')
        
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        if aggregated_dict_key in inverse_search_dict:
          aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]

          for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
            target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

            mdf_test = \
            self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')

            del mdf_test[target_for_aggregation_column]

      for newcolumn in newcolumns:

        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
        
      #remove temporary support column
      del mdf_test[suffixcolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_src2(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_src2(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #assumes that unique values of test set are same or subset of train set
    #for more efficient application in postmunge
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']

      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['newcolumns_before_aggregation']
      
      search = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
      
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      
      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      
      case = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']

      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']

      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
        
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #convert to uppercase string when case sensitivity not desired based on case parameter
      if case is False:
        #convert column to uppercase string except for nan infill points
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str).str.upper(), specified='replacement')
      
#       #now for mdf_test

#       unique_list_test = list(mdf_test[column].unique())

#       unique_list_test = list(map(str, unique_list_test))

#       test_overlap_dict = {}

#       for search_string in search:

#         test_overlap_dict.update({search_string : []})

#       train_keys = list(overlap_dict)

#       train_keys.sort(key = len, reverse=True)

#       for dict_key in train_keys:

#         for unique_test in unique_list_test:

#           len_key = len(dict_key)

#           if len(unique_test) >= len_key:

#             nbr_iterations4 = len(unique_test) - len_key

#             for l in range(nbr_iterations4 + 1):

#               extract4 = unique_test[l:(len_key+l)]

#               if extract4 == dict_key:

#                 test_overlap_dict[dict_key].append(unique_test)

      newcolumns = []

      for dict_key in overlap_dict:
        
        if len(overlap_dict[dict_key]) > 0:

          newcolumn = column + '_' + suffix + '_' + dict_key

  #         mdf_train[newcolumn] = mdf_train[column].copy()
          mdf_test[newcolumn] = mdf_test[suffixcolumn].copy()

  #         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

          mdf_test[newcolumn] = mdf_test[newcolumn].isin(overlap_dict[dict_key])
#           mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

          newcolumns.append(newcolumn)
          
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        if aggregated_dict_key in inverse_search_dict:
          aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]

          for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
            target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

            mdf_test = \
            self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')

            del mdf_test[target_for_aggregation_column]

            newcolumns.remove(target_for_aggregation_column)

      for newcolumn in newcolumns:

        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
        
      #remove temporary support column
      del mdf_test[suffixcolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_src3(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_src3(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #where srch is preferred for unbounded range of unique values
    
    #and src2 preferred when have bounded range of unique values for both train & test
    
    #and speculation is that src3 may be preferred when have a bounded
    #range of unique values but still want capacity to handle values in 
    #test set not found in train set
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
    
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']

      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src3']
      
      search = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
      
      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      
      case = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']

      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']

      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
        
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #convert to uppercase string when case sensitivity not desired based on case parameter
      if case is False:
        #convert column to uppercase string except for nan infill points
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str).str.upper(), specified='replacement')
      
      #now for mdf_test

      unique_list_test = list(mdf_test[suffixcolumn].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      for search_string in search:

        test_overlap_dict.update({search_string : []})

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key

            for l in range(nbr_iterations4 + 1):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

      newcolumns = []

      for dict_key in overlap_dict:
        
        if len(overlap_dict[dict_key]) > 0:

          newcolumn = column + '_' + suffix + '_' + dict_key

  #         mdf_train[newcolumn] = mdf_train[column].copy()
          mdf_test[newcolumn] = mdf_test[suffixcolumn].copy()

  #         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

          mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

          newcolumns.append(newcolumn)
          
      #remove temporary support column
      del mdf_test[suffixcolumn]
      
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        if aggregated_dict_key in inverse_search_dict:
          aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]

          for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
            target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

            mdf_test = \
            self.__autowhere(mdf_test, aggregated_dict_key_column, mdf_test[target_for_aggregation_column] == 1, 1, specified='replacement')

            del mdf_test[target_for_aggregation_column]

            newcolumns.remove(target_for_aggregation_column)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_src4(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_srch(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    
    #src4 builds on the srch by converting to an ordinal activation
    #with 0 reserved for no activations
    #note that if an entry was activated for multiple search terms
    #the order of entries in search parameter will dictate the final encoding
    #(e.g. entries at end of list are prioritized over beginning)
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      #great now we can grab normalization parameters
      search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']

      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src4']

      search = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']

      case = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']

      ordl_dict1 = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict1']

      ordl_dict2 = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict2']

      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      if mdf_test[suffixcolumn].dtype.name == 'category':
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object')
      
      if len(search_dict) == 0:
        mdf_test.loc[:, suffixcolumn] = 0
        
      else:

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

        for newcolumn in search_dict:

          mdf_test = \
          self.__autowhere(mdf_test, newcolumn, mdf_test[suffixcolumn].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, specified='replacement')

    #     for newcolumn in newcolumns:

    #       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        mdf_test.loc[:, suffixcolumn] = 0

        for newcolumn in newcolumns:
          mdf_test = \
          self.__autowhere(mdf_test, suffixcolumn, mdf_test[newcolumn] == 1, ordl_dict2[newcolumn], specified='replacement')
          del mdf_test[newcolumn]
          
        #now we consolidate activations
        #note that this only runs when aggregated_dict was populated with an embedded list of search terms
        for aggregated_dict_key in aggregated_dict:
          if aggregated_dict_key in inverse_search_dict:
            aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
            aggregated_dict_key_encoding = ordl_dict2[aggregated_dict_key_column]

            for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
              target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
              target_for_aggregation_encoding = ordl_dict2[target_for_aggregation_column]

              mdf_test = \
              self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == target_for_aggregation_encoding, aggregated_dict_key_encoding, specified='replacement')

        #we'll base the integer type on number of ordinal entries
        max_encoding = len(ordl_dict1)
        if max_encoding <= 255:
          mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint8)
        elif max_encoding <= 65535:
          mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint16)
        else:
          mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_nmr4(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #extract numeric partitions from categoric entries, test treated differently than train
    #accepts parameters
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    #test_same_as_train as True/False
    #where True copiues overlap_dict from train for test, False parses test entries not found in train
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]

    #normkey is False when process function returns empty set
    if normkey is not False:
      
      nmrc_column = normkey
      
      mean = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['mean']
      unique_list = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['unique_list']
      overlap_dict = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['overlap_dict']
      convention = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['convention']
      test_same_as_train = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['test_same_as_train']
      suffix = \
      postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      mdf_test[nmrc_column] = mdf_test[column].copy()
      
      test_unique_list = list(mdf_test[nmrc_column].unique())
      test_unique_list = list(map(str, test_unique_list))
      extra_test_unique = list(set(test_unique_list) - set(unique_list))

      test_overlap_dict = self.__autocopy(overlap_dict)
      
      if test_same_as_train is True:
        
        for test_unique in extra_test_unique:
          test_overlap_dict.update({str(test_unique) : np.nan})
        
      elif test_same_as_train is False:
        
        testmaxlength = max(len(x) for x in unique_list)

        overlap_lengths = list(range(testmaxlength, 0, -1))

    #     overlap_dict = {}

        for overlap_length in overlap_lengths:

          for unique in extra_test_unique:

            if unique not in test_overlap_dict:

              len_unique = len(unique)

              if len_unique >= overlap_length:

                if overlap_length > 1:

                  nbr_iterations = len_unique - overlap_length

                  for i in range(nbr_iterations + 1):

                    if unique not in test_overlap_dict:

                      extract = unique[i:(overlap_length+i)]

      #                 extract_already_in_overlap_dict = False
                      
                      if convention == 'numbers':
                      
                        if self.__is_number(extract):

                          test_overlap_dict.update({unique : float(extract)})
                    
                      elif convention == 'commas':
                      
                        if self.__is_number_comma(extract):

                          test_overlap_dict.update({unique : float(extract.replace(',',''))})
                          
                      elif convention == 'spaces':
                      
                        if self.__is_number_EU(extract):

                          test_overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})

                #else if overlap_length == 1    
                else:

                  nbr_iterations = len_unique - overlap_length

                  in_dict = False

                  for i in range(nbr_iterations + 1):

                    if unique not in test_overlap_dict:

                      extract = unique[i:(overlap_length+i)]

      #                 extract_already_in_overlap_dict = False
                      
                      if self.__is_number(extract):

                        in_dict = True

                        test_overlap_dict.update({unique : float(extract)})

                  if in_dict is False:

                    test_overlap_dict.update({unique : np.nan})

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, nmrc_column, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #great now that test_overlap_dict is populated
      mdf_test[nmrc_column] = mdf_test[nmrc_column].astype(str)
      mdf_test[nmrc_column] = mdf_test[nmrc_column].replace(test_overlap_dict)

      mdf_test[nmrc_column] = pd.to_numeric(mdf_test[nmrc_column], errors='coerce')
      
      #replace missing data with training set mean as default infill
      mdf_test[nmrc_column] = mdf_test[nmrc_column].fillna(mean)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _custom_test_ordl(self, df, column, normalization_dict):
    """
    #rewrite of the ordl trasnform
    #corresponding to _custom_train_ordl
    """

    #access the train set properties from normalization_dict
    str_convert = normalization_dict['str_convert']
    inverse_consolidation_translate_dict = normalization_dict['inverse_consolidation_translate_dict']
    ordinal_dict = normalization_dict['ordinal_dict']
    null_activation = normalization_dict['null_activation']
    missing_marker = normalization_dict['missing_marker']
    
    #setting to object allows mixed data types for .replace operations and removes complexity of pandas category dtype
    df[column] = df[column].astype('object')
    
    #if str_convert elected (for common encoding between e.g. 2=='2')
    if str_convert is True:
      df = \
      self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')
    
    #if a consolidated_activations parameter was performed, we'll consolidated here
    if inverse_consolidation_translate_dict != {}:
      #apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)
      
    #if the test set has entries without encodings, we'll replace with missing data marker
    extra_entries = set(df[column].unique()) - set(ordinal_dict)
    extra_entries = list({x for x in extra_entries if x==x})
    if len(extra_entries) > 0:
      plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
      df[column] = df[column].astype('object').replace(plug_dict)
    
    #now replace the entries in column with their ordinal representation
    df[column] = df[column].astype('object').replace(ordinal_dict)
    
    return df

  def _postprocess_maxb(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_maxb(mdf_test, column, postprocess_dict, columnkey)
    #simpler than dualprocess version
    #just inspects new_maxactivation
    #and consolidates larger activations
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      new_maxactivation = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['new_maxactivation']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #non integers are subject to infill
      mdf_test = \
      self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn].round(), alternative=np.nan, specified='alternative')
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #get maximum train set activation which for ord3 will be least frequent entry
      maxactivation = int(mdf_test[suffixcolumn].max())
      
      if new_maxactivation < maxactivation:
        
        mdf_test = \
        self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] < new_maxactivation, alternative=new_maxactivation, specified='alternative')
        
      #set integer type based on encoding depth
      if new_maxactivation <= 255:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint8)
      elif new_maxactivation <= 65535:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint16)
      else:
        mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_ucct(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_ucct(mdf_train, mdf_test, column, category)
    #preprocess column with categories into unique class count sets
    #normalized by total row count
    #e.g. for each class in train set, 
    #counts instances and divides by total train set row count
    #(so values will fall in range 0-1)
    #test sets recive comparable encoding
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      #grab normalization parameters from postprocess_dict
      ordinal_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      ordinal_nan_value = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_nan_value']

      suffixcolumn = column + '_' + suffix
      
      #create new column for trasnformation
      mdf_test[suffixcolumn] = mdf_test[column].copy()
      
      #convert column to category
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      mdf_test = \
      self.__autowhere(mdf_test, suffixcolumn, mdf_test[suffixcolumn] == mdf_test[suffixcolumn], mdf_test[suffixcolumn].astype(str), specified='replacement')
      
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      #train categories are in the ordinal_dict we p[ulled from normalization_dict
      labels_train = set(ordinal_dict.keys())
  #     labels_train.sort()
      labels_test = set(mdf_test[suffixcolumn].unique())
      labels_test = {x for x in labels_test if x==x}
      
      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(labels_test-labels_train)
      
      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, [np.nan] * len(testspecificcategories)))
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object').replace(testplug_dict)
      
      #now we'll apply the ordinal transformation to the test set
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype('object').replace(ordinal_dict)
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(ordinal_nan_value)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _custom_test_1010(self, df, column, normalization_dict):
    """
    #rewrite of the 1010 trasnform
    #corresponding to _custom_train_1010
    """

    #access the train set properties from normalization_dict
    str_convert = normalization_dict['str_convert']
    inverse_consolidation_translate_dict = normalization_dict['inverse_consolidation_translate_dict']
    binary_encoding_dict = normalization_dict['binary_encoding_dict']
    _1010_columnlist = normalization_dict['_1010_columnlist']
    null_activation = normalization_dict['null_activation']
    missing_marker = normalization_dict['missing_marker']

    if 'inverse_binary_encoding_dict' in normalization_dict:
      inverse_binary_encoding_dict = normalization_dict['inverse_binary_encoding_dict']
    else:
      #backward compatibility preceding 8.26
      if null_activation is not False:
        inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items()}
      elif null_activation is False:
        inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items() if key==key}
    
    #setting to object allows mixed data types for .replace operations and removes complexity of pandas category dtype
    df[column] = df[column].astype('object')
    
    #if str_convert elected (for common encoding between e.g. 2=='2')
    if str_convert is True:
      df = \
      self.__autowhere(df, column, df[column] == df[column], df[column].astype(str), specified='replacement')
      
    #if a consolidated_activations parameter was performed, we'll consolidated here
    if inverse_consolidation_translate_dict != {}:
      #apply a replace to convert consolidated items to their targeted activations
      df[column] = df[column].astype('object').replace(inverse_consolidation_translate_dict)
      
    # #if the test set has entries without encodings, we'll replace with missing data marker
    # extra_entries = set(df[column].unique()) - set(binary_encoding_dict)
    # extra_entries = list({x for x in extra_entries if x==x})
    # if len(extra_entries) > 0:
    #   plug_dict = dict(zip(extra_entries, [missing_marker] * len(extra_entries)))
    #   df[column] = df[column].astype('object').replace(plug_dict)
    
    #now replace the entries in column with their binarization representation
    #note this representation is a string of 0's and 1's that will next be seperated into seperate columns
    #there is a pandas dtype drift edge case bug possible here when all entries are str(number) and missing marker is nan
    #we mitigate by assigning majority str(number) to category nmbr under automation
    df[column] = df[column].astype('object').replace(binary_encoding_dict)

    #this circumvents pandas dtype drift issue
    for entry in df[column].unique():
      if entry not in set(inverse_binary_encoding_dict):
        if entry == entry:
          df.loc[df[column] == entry, column] = binary_encoding_dict[missing_marker]
        else:
          df.loc[df[column].isna(), column] = binary_encoding_dict[missing_marker]

    #now let's store the encoding
    i=0
    for _1010_column in _1010_columnlist:
      
      if len(_1010_columnlist) > 1:
        #there is a pandas dtype drift edge case bug possible here when all entries are str(number) and missing marker is nan
        #we mitigate by assigning majority str(number) to category nmbr under automation
        df[_1010_column] = df[column].str.slice(i,i+1).astype(np.int8)
        i+=1
      else:
        #there is a pandas dtype drift edge case bug possible here when all entries are str(number) and missing marker is nan
        #we mitigate by assigning majority str(number) to category nmbr under automation
        df[_1010_column] = df[column].astype(np.int8)
      
    #now delete the support column
    del df[column]
    
    return df

  def _postprocess_tmsc(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #time data segregated by time scale
    #with sin or cos applied to address periodicity
    #such as may be useful to return both by seperate transformation categories
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #note that some scales can be returned combined by passing 
    #monthday/dayhourminute/hourminutesecond/minutesecond
    #accepts parameter 'suffix' for returned column header suffix
    #accets parameter 'function' to distinguish between sin/cos
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      time_column = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[time_column] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : time_column}, inplace = True)

      if time_column not in postprocess_dict['column_dict']:

        del mdf_test[time_column]

      else:

        scale = \
        postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['scale']
        function = \
        postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['function']

        #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
        mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce', format='mixed')

        #access time scale from one of year/month/day/hour/minute/second
        #monthday/dayhourminute/hourminutesecond/minutesecond
        if scale == 'year':
          mdf_test[time_column] = mdf_test[time_column].dt.year

          #we'll scale periodicity by decade
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 10

        elif scale == 'month':
          mdf_test[time_column] = mdf_test[time_column].dt.month

          #we'll scale periodicity by year
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 12

        elif scale == 'day':
          mdf_test[time_column] = mdf_test[time_column].dt.day

          #we'll scale periodicity by week
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 7

        elif scale == 'hour':
          mdf_test[time_column] = mdf_test[time_column].dt.hour

          #we'll scale periodicity by day
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 24

        elif scale == 'minute':
          mdf_test[time_column] = mdf_test[time_column].dt.minute

          #we'll scale periodicity by hour
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60

        elif scale == 'second':
          mdf_test[time_column] = mdf_test[time_column].dt.second

          #we'll scale periodicity by minute
          mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60

        elif scale == 'monthday':
          tempcolumn1 = 1
          tempcolumn2 = 2

          #temp1 is for number of days in month, temp2 is to handle leap year support

          mdf_test[tempcolumn1] = mdf_test[time_column].copy()
          mdf_test[tempcolumn2] = mdf_test[time_column].copy()

          mdf_test[tempcolumn1] = mdf_test[tempcolumn1].dt.month
          mdf_test[tempcolumn2] = mdf_test[tempcolumn2].dt.is_leap_year

          mdf_test = \
          self.__autowhere(mdf_test, tempcolumn2, mdf_test[tempcolumn2], 29, 28)

          mdf_test = \
          self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, specified='replacement')

          mdf_test = \
          self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([4,6,9,11]), 30, specified='replacement')

          mdf_test = \
          self.__autowhere(mdf_test, tempcolumn1, mdf_test[tempcolumn1].isin([2]), mdf_test[tempcolumn2], specified='replacement')

          #combine month and day, scale for trigonomic transform, periodicity by year
          mdf_test[time_column] = (mdf_test[time_column].dt.month + mdf_test[time_column].dt.day / \
          mdf_test[tempcolumn1]) * 2 * np.pi / 12

          #delete the support columns
          del mdf_test[tempcolumn1]
          del mdf_test[tempcolumn2]

        elif scale == 'dayhourminute':
          #we'll scale periodicity by week
          mdf_test[time_column] = (mdf_test[time_column].dt.day + mdf_test[time_column].dt.hour / 24 + mdf_test[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7

        elif scale == 'hourminutesecond':
          #we'll scale periodicity by day
          mdf_test[time_column] = (mdf_test[time_column].dt.hour + mdf_test[time_column].dt.minute / 60 + mdf_test[time_column].dt.second / 60 / 60) * 2 * np.pi / 24

        elif scale == 'minutesecond':
          #we'll scale periodicity by hour
          mdf_test[time_column] = (mdf_test[time_column].dt.minute + mdf_test[time_column].dt.second / 60) * 2 * np.pi / 60

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, time_column, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

        #apply trigometric transform

        if function == 'sin':
          mdf_test[time_column] = np.sin(mdf_test[time_column])

        if function == 'cos':
          mdf_test[time_column] = np.cos(mdf_test[time_column])

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
      
    return mdf_test

  def _postprocess_time(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #z-score normalized time data segregated by a particular time scale
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #accepts parameter 'suffix' for returned column header suffix
    #accepts parameter 'normalization' to distinguish between zscore/minmax/unscaled
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      scale = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scale']
      normalization = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['normalization']
      scaler = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scaler']
      divisor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
    
      time_column = column + '_' + suffix

      if inplace is not True:
        #copy source column into new column
        mdf_test[time_column] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : time_column}, inplace = True)

      #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
      mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce', format='mixed')

      #access time scale from one of year/month/day/hour/minute/second
      if scale == 'year':
        mdf_test[time_column] = mdf_test[time_column].dt.year
      elif scale == 'month':
        mdf_test[time_column] = mdf_test[time_column].dt.month
      elif scale == 'day':
        mdf_test[time_column] = mdf_test[time_column].dt.day
      elif scale == 'hour':
        mdf_test[time_column] = mdf_test[time_column].dt.hour
      elif scale == 'minute':
        mdf_test[time_column] = mdf_test[time_column].dt.minute
      elif scale == 'second':
        mdf_test[time_column] = mdf_test[time_column].dt.second

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, time_column, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #apply normalization
      if normalization != 'unscaled':
        mdf_test[time_column] = (mdf_test[time_column] - scaler) / divisor

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
      
    return mdf_test

  def _postprocess_qttf(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    Corresponds to _process_qttf
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean_impute = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean_impute']
      qttf = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['qttf']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix
      
      #copy source column into new column
      mdf_test[suffixcolumn] = mdf_test[column].copy()
        
      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
        
      #apply transform to test set
      if qttf is not False:
        from sklearn.preprocessing import QuantileTransformer
        mdf_test[suffixcolumn] = pd.DataFrame(qttf.transform(pd.DataFrame(mdf_test[suffixcolumn])), index=mdf_test.index)
      else:
        mdf_test.loc[:, suffixcolumn] = 0
      
    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_bxcx(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    Applies box-cox method within postmunge function.
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      bxcx_lmbda = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bxcx_lmbda']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      suffixcolumn = column + '_' + suffix
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)
        
      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
    
      #convert non-positive values to nan
      mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
      
      test_bxcx_lmbda = None
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #edge case to avoid stats.boxcox error
      if mdf_test[suffixcolumn].nunique() == 1:
        test_bxcx_lmbda = False
        
      if bxcx_lmbda is not False:

        if test_bxcx_lmbda is not False:
          mdf_test[suffixcolumn] = stats.boxcox(mdf_test[suffixcolumn], lmbda = bxcx_lmbda)
        else:
          mdf_test.loc[:, suffixcolumn] = 0

      elif bxcx_lmbda is False:

        mdf_test.loc[:, suffixcolumn] = 0
        
      #this is to address an error when bxcx transform produces overflow
      #I'm not sure of cause, showed up in the housing set)
      max_test = mdf_test[suffixcolumn].max()
      
      if max_test > (2 ** 31 - 1):
        mdf_test.loc[:, suffixcolumn] = 0
        # if postprocess_dict['printstatus'] != 'silent':
        self.__autoprint('warning', "overflow condition found in boxcox transform to test set, column set to 0: ", suffixcolumn)
        self.__autoprint('warning', '')
      
    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_log0(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base 10)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      meanlog = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meanlog']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #replace all non-positive with nan for the log operation
      mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
      
      #log transform column
      #note that this replaces negative values with nan which we will infill with meanlog
      mdf_test[suffixcolumn] = np.log10(mdf_test[suffixcolumn])

      #get mean of training data
      meanlog = meanlog  

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

  #     #replace missing data with 0
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_logn(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply natural logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base e)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      meanlog = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meanlog']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #replace all non-positive with nan for the log operation
      mdf_test.loc[mdf_test[suffixcolumn] <= 0, (suffixcolumn)] = np.nan
      
      #log transform column
      #note that this replaces negative values with nan which we will infill with meanlog
      mdf_test[suffixcolumn] = np.log(mdf_test[suffixcolumn])

      #get mean of training data
      meanlog = meanlog  

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

  #     #replace missing data with 0
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_sqrt(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply square root transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a square root transform
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      meansqrt = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meansqrt']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #replace all non-positive with nan for the log operation
      mdf_test.loc[mdf_test[suffixcolumn] < 0, (suffixcolumn)] = np.nan
      
      #log transform column
      #note that this replaces negative values with nan which we will infill with meanlog
      mdf_test[suffixcolumn] = np.sqrt(mdf_test[suffixcolumn])

      #get mean of training data
      meansqrt = meansqrt  

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

  #     #replace missing data with 0
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].fillna(0)

  #     #change data type for memory savings
  #     mdf_test[suffixcolumn] = mdf_test[suffixcolumn].astype(np.float32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_addd(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_addd(.)
    #function to apply addition transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'add' for amount of addition, otherwise defaults to adding 1
    #applies an addition transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      add = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['add']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform addition
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] + add
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_sbtr(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_sbtr(.)
    #function to apply subtraction transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'subtract' for amount of subtraction, otherwise defaults to subtracting 1
    #applies a subtraction transform
    #replaces non-numeric entries with set mean after subtraction
    #returns same dataframes with new column of name suffixcolumn
    '''
    
    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      subtract = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['subtract']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform subtraction
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] - subtract
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_mltp(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mltp(.)
    #function to apply multiplication transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'multiply' for amount of addition, otherwise defaults to multiplying 2
    #applies an multiplication transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      multiply = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiply']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform addition
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] * multiply
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
    
  def _postprocess_divd(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_divd(.)
    #function to apply division transform
    #accepts parameter 'divide' for amount of division, otherwise defaults to dividing by 2
    #applies an division transform
    #replaces non-numeric entries with set mean after division
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      divide = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divide']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform addition
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] / divide
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_rais(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_rais(.)
    #function to apply raise to a power transform
    #accepts parameter 'raiser' for amount of power, otherwise defaults to square (raise by 2)
    #applies an raise transform
    #replaces non-numeric entries with set mean after raise
    #returns same dataframes with new column of name suffixcolumn
    '''
    
    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      raiser = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['raiser']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform addition
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn] ** raiser
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
    
  def _postprocess_absl(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_absl(.)
    #function to apply absolute transform
    #does not accept paraemters
    #applies an absolute transform
    #replaces non-numeric entries with set mean after transform
    #returns same dataframes with new column of name suffixcolumn
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      suffixcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[suffixcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : suffixcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[suffixcolumn] = pd.to_numeric(mdf_test[suffixcolumn], errors='coerce')
      
      #lperform addition
      mdf_test[suffixcolumn] = mdf_test[suffixcolumn].abs()
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, suffixcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_pwrs(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating bins corresponding to powers
    #of ten in one hot encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #postiive values encoded under column 'column' + '_10^#' where # is power of 10
    #0 and negative values considered infill with no activations
    
    #if all values are infill no columns returned
    
    #accepts boolean 'negvalues' parameter, defaults False, True activates encoding for values <0
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:

      #normkey = columnkey

      powerlabelsdict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['powerlabelsdict_pwrs']
      labels_train = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['labels_train']
      negvalues = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['negvalues']
      zeroset = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['zeroset']
      cap = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

      suffixcolumn = column + '_' + suffix

      textcolumns = postprocess_dict['column_dict'][normkey]['categorylist']
      pos_and_negative_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['pos_and_negative_list']

      tempcolumn = 1
      tempcolumn_zero = suffixcolumn + '_zero'
      negtempcolumn = -1

      #store original column for later reversion
      mdf_test[tempcolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[tempcolumn] = pd.to_numeric(mdf_test[tempcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[tempcolumn] > cap, (tempcolumn)] = cap
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[tempcolumn] < floor, (tempcolumn)] = floor
      
      #convert all values <= 0 to Nan
      mdf_test = \
      self.__autowhere(mdf_test, tempcolumn, mdf_test[tempcolumn] <= 0, np.nan, specified='replacement')
      
      #now log trasnform positive values in column column 
      mdf_test[tempcolumn] = np.floor(np.log10(mdf_test[tempcolumn].astype(float)))

      test_pos_dict = {}
      posunique = mdf_test[tempcolumn].unique()
      for unique in posunique:
        if unique != unique:
          newunique = np.nan
        else:
          newunique = suffixcolumn + '_10^' + str(int(unique))
        if newunique in labels_train and newunique == newunique:
          test_pos_dict.update({unique : newunique})
        else:
          test_pos_dict.update({unique : np.nan})

      mdf_test[tempcolumn] = mdf_test[tempcolumn].replace(test_pos_dict)    
      
      if negvalues is True:
        #create copy with negative values
        mdf_test[negtempcolumn] = mdf_test[column].copy()
        
        #convert all values to either numeric or NaN
        mdf_test[negtempcolumn] = pd.to_numeric(mdf_test[negtempcolumn], errors='coerce')
        
        if cap is not False:
          #replace values in test > cap with cap
          mdf_test.loc[mdf_test[negtempcolumn] > cap, (negtempcolumn)] = cap
        if floor is not False:
          #replace values in test < floor with floor
          mdf_test.loc[mdf_test[negtempcolumn] < floor, (negtempcolumn)] = floor

        #convert all values in negtempcolumn >= 0 to Nan
        mdf_test = \
        self.__autowhere(mdf_test, negtempcolumn, mdf_test[negtempcolumn] >= 0, np.nan, specified='replacement')
        
        #take abs value of negtempcolumn
        mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
        
        #now log trasnform
        mdf_test[negtempcolumn] = np.floor(np.log10(mdf_test[negtempcolumn].astype(float)))

        test_neg_dict = {}
        negunique = mdf_test[negtempcolumn].unique()
        for unique in negunique:
          if unique != unique:
            newunique = np.nan
          else:
            #this is update for difference between pwr2 and pwrs
            if negvalues:
              newunique = suffixcolumn + '_-10^' + str(int(unique))
            else:
              newunique = np.nan
          if newunique in labels_train and newunique == newunique:
            test_neg_dict.update({unique : newunique})
          else:
            test_neg_dict.update({unique : np.nan})

        mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
        
      #now if the zero column included is a little simpler
      if zeroset is True:
        mdf_test[tempcolumn_zero] = mdf_test[column].copy()

        #convert all values to either numeric or NaN
        mdf_test[tempcolumn_zero] = pd.to_numeric(mdf_test[tempcolumn_zero], errors='coerce')
        
        if cap is not False:
          #replace values in test > cap with cap
          mdf_test.loc[mdf_test[tempcolumn_zero] > cap, (tempcolumn_zero)] = cap
        if floor is not False:
          #replace values in test < floor with floor
          mdf_test.loc[mdf_test[tempcolumn_zero] < floor, (tempcolumn_zero)] = floor

        #convert to 1 activations for zero and 0 elsewhere
        mdf_test = \
        self.__autowhere(mdf_test, tempcolumn_zero, mdf_test[tempcolumn_zero]==0, 1, 0)
        
      #combine the positive and negative columns if applicable
      if negvalues is True:
        mdf_test = \
        self.__autowhere(mdf_test, tempcolumn, mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[negtempcolumn], specified='replacement')
        
      #apply onehotencoding
      df_test_cat = self.__onehot_support(mdf_test, tempcolumn, scenario=1, activations_list=pos_and_negative_list)

      #concatinate the sparse set with the rest of our training data
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)

      #delete support columns
      del mdf_test[tempcolumn]
      if negvalues is True:
        del mdf_test[negtempcolumn]

      #change data types to 8-bit (1 byte) integers for memory savings
      for textcolumn in textcolumns:

        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_pwor(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating bins coresponding to powers
    #of ten in ordinal encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #infill has 0, other designations are based on the data
    
    #negative values allows, comparable to pwr2
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      #retrieve stuff from normalization dictionary
      train_replace_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['train_replace_dict']
      negvalues = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['negvalues']
      activations_list = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['activations_list']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      inverse_train_replace_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_train_replace_dict']
      zeroset = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['zeroset']
      cap = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      
      pworcolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[pworcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : pworcolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[pworcolumn] = pd.to_numeric(mdf_test[pworcolumn], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[pworcolumn] > cap, (pworcolumn)] = cap
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[pworcolumn] < floor, (pworcolumn)] = floor
      
      if negvalues is True:
        #copy set for negative values
        negtempcolumn = -1

        mdf_test[negtempcolumn] = mdf_test[pworcolumn].copy()
        
        #convert all values >= 0 to Nan
        mdf_test = \
        self.__autowhere(mdf_test, negtempcolumn, mdf_test[negtempcolumn] >= 0, np.nan, specified='replacement')
        
        #take abs value of negtempcolumn
        mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
        
        mdf_test[negtempcolumn] = np.floor(np.log10(mdf_test[negtempcolumn].astype(float)))

        newunique_list = list(train_replace_dict)

        test_neg_dict = {}
        negunique = mdf_test[negtempcolumn].unique()
        for unique in negunique:
          if unique != unique:
            newunique = np.nan
          else:
            #this is update for difference between pwr2 and pwrs
            if negvalues:
              newunique = column + '_-10^' + str(int(unique))
            else:
              newunique = np.nan
          if newunique in newunique_list and newunique == newunique:
            test_neg_dict.update({unique : newunique})
          else:
            test_neg_dict.update({unique : np.nan})

        mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
        
      if zeroset is True:
        #copy set for negative values
        zerotempcolumn = 0

        mdf_test[zerotempcolumn] = mdf_test[pworcolumn].copy()
        
        #convert all values != 0 to Nan
        mdf_test = \
        self.__autowhere(mdf_test, zerotempcolumn, mdf_test[zerotempcolumn] != 0, np.nan, column + '_zero')
      
      #convert all values <= 0 in column to Nan
      mdf_test = \
      self.__autowhere(mdf_test, pworcolumn, mdf_test[pworcolumn] <= 0, np.nan, specified='replacement')

      mdf_test[pworcolumn] = np.floor(np.log10(mdf_test[pworcolumn].astype(float)))

      newunique_list = list(train_replace_dict)
  
      test_pos_dict = {}
      posunique = mdf_test[pworcolumn].unique()
      for unique in posunique:
        if unique != unique:
          newunique = np.nan
        else:
          newunique = column + '_10^' + str(int(unique))
        if newunique in newunique_list and newunique == newunique:
          test_pos_dict.update({unique : newunique})
        else:
          test_pos_dict.update({unique : np.nan})
      
      mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_pos_dict)
      
      #combine the columns
      if negvalues is True:
        mdf_test = \
        self.__autowhere(mdf_test, pworcolumn, mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[negtempcolumn], specified='replacement')

      if zeroset is True:
        mdf_test = \
        self.__autowhere(mdf_test, pworcolumn, mdf_test[zerotempcolumn] == mdf_test[zerotempcolumn], mdf_test[zerotempcolumn], specified='replacement')
      
      test_unique = mdf_test[pworcolumn].unique()
    
      #Get missing entries in test set that are present in training set
      missing_cols = set( list(newunique_list) ) - set( list(test_unique) )
      
      extra_cols = set( list(test_unique) ) - set( list(newunique_list) )
        
      test_replace_dict = {}
      for testunique in test_unique:
        if testunique in newunique_list:
          test_replace_dict.update({testunique : train_replace_dict[testunique]})
        else:
          test_replace_dict.update({testunique : 0})
      
  #     pworcolumn = column + '_por2'
  #     mdf_test[pworcolumn] = mdf_test[column].copy()
      
      mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_replace_dict)

      #delete support columns
      if negvalues is True:
        del mdf_test[negtempcolumn]    
      if zeroset is True:
        del mdf_test[zerotempcolumn]

      #returned data type is conditional on the size of encoding space
      max_encoding = max(list(inverse_train_replace_dict))
      if max_encoding <= 255:
        mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint16)
      else:
        mdf_test[pworcolumn] = mdf_test[pworcolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_bins(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins will be intended for a raw set that is not normalized
    #bint will be intended for a previously normalized set
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      std = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
      bincuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
      binlabels = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binscolumns']
      normalizedinput = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['normalizedinput']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      binscolumn = column + '_' + suffix

      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bins'
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      #process bins as a categorical set
      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_bsor(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    
    #bsor is comparable to bins but returns ordinal encoded column
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      #retrieve normalization parameters from postprocess_dict

      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      std = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
      ordinal_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
      bincuts = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
      binlabels = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
      normalizedinput = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['normalizedinput']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      bincount = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      binscolumn = column + '_' + suffix

      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)
      
      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      if normalizedinput is False:
      
        #z-score normalize
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #create bins based on standard deviation increments
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      #data type is conditional based on encoding space
      max_encoding = len(ordinal_dict) - 1
      if max_encoding <= 255:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_bnwd(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in one-hot encoded set
    
    #deletes columns without activations
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      suffix = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bn_width = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width_bnwd']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      binscolumn = column + '_' + suffix
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)))

      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_bnwo(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bn_width = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      binscolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwo'
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
            labels = bins_id, precision=len(str(bn_count)))

      #returned data type is conditional on the size of encoding space
      max_encoding = bn_count - 1
      if max_encoding <= 255:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_bnep(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount_bnep']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      suffix = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      binscolumn = column + '_' + suffix
      
      #copy original column
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

#       #replace missing data with training set mean
#       mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      if bn_delta > 0 and bn_min == bn_min:

        mdf_test[binscolumn] = \
        pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
               labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

        df_test_cat = \
        self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
      
        mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
        
        del df_test_cat

        #change data type for memory savings
        for textcolumn in textcolumns:
          mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

        #delete the support column
        del mdf_test[binscolumn]
        
      else:
        
        mdf_test.loc[:, binscolumn] = 0

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_bneo(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in ordinal encoded set
    
    #default infill is adjacent cell infill
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
      inplace = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      binscolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

  #     #replace missing data with training set mean
  #     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

      if bn_delta > 0 and bn_min == bn_min:

        #create bins based on prepared increments
    #     binscolumn = column + '_bnwo'
        mdf_test[binscolumn] = \
        pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
              labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

        #apply defaultinfill based on processdict entry
        mdf_test, _1 = \
        self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

        #returned data type is conditional on the size of encoding space
        max_encoding = bn_count - 1
        if max_encoding <= 255:
          mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
        elif max_encoding <= 65535:
          mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
        else:
          mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)
        
      else:
        
        mdf_test.loc[:, binscolumn] = 0

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test
  
  def _postprocess_tlbn(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 9
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_tlbn']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      binscolumn = column + '_' + suffix
      
      #copy original column
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

#       #replace missing data with training set mean
#       mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      if bn_delta > 0 and bn_min == bn_min:

        mdf_test[binscolumn] = \
        pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
               labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

        df_test_cat = \
        self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
      
        mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
        
        del df_test_cat
                
        #initialize binscolumn once more
        mdf_test[binscolumn] = mdf_test[column].copy()        
        mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
        
        
        if len(textcolumns) > 1:

          #for i in range(bincount):
          for i in range(len(textcolumns)):

            tlbn_column = binscolumn + '_' + str(i)

            if i == 0:

              mdf_test = \
              self.__autowhere(mdf_test, 
                              tlbn_column, 
                              mdf_test[tlbn_column] == 1, 
                              (bins_cuts[i+1] - mdf_test[binscolumn]) / (bins_cuts[i+1] - bn_min), 
                              -1)

            # elif i == bincount - 1:
            elif i == len(textcolumns) - 1:

              mdf_test = \
              self.__autowhere(mdf_test, 
                              tlbn_column, 
                              mdf_test[tlbn_column] == 1, 
                              (mdf_test[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), 
                              -1)

            else:

              mdf_test = \
              self.__autowhere(mdf_test, 
                              tlbn_column, 
                              mdf_test[tlbn_column] == 1, 
                              (mdf_test[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), 
                              -1)

#         #change data type for memory savings
#         for textcolumn in textcolumns:
#           mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

        #delete the support column
        del mdf_test[binscolumn]

        #there's still an edge scenario for custom buckets where nan get's populated, 
        #so this is just a hack to clean up
        for textcolumn in textcolumns:
          mdf_test[textcolumn] = mdf_test[textcolumn].fillna(-1)
        
      else:
        
        mdf_test.loc[:, binscolumn] = 0

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_bkt1(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt1']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      binscolumn = column + '_' + suffix
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(len(bins_id))))

      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_bkt2(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:
      normkey = columnkey[0]
        
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt2']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      binscolumn = column + '_' + suffix
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(len(bins_id))))

      df_test_cat = \
      self.__onehot_support(mdf_test, binscolumn, scenario=2, activations_list = textcolumns)
    
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
      
      del df_test_cat

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test
  
  def _postprocess_bkt3(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      ordl_activations_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
      infill_activation = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
      inplace = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      binscolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      # #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwo'
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
            labels = bins_id, precision=len(str(len(bins_id))))

      mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #replace missing data with infill_activation
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)

      #returned data type is conditional on the size of encoding space
      max_encoding = len(bins_cuts) - 1
      if max_encoding <= 255:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test
  
  def _postprocess_bkt4(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      ordl_activations_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
      infill_activation = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
      inplace = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']

      binscolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      #set all values that fall outside of bounded buckets to nan for replacement with mean
      mdf_test.loc[mdf_test[binscolumn] <= buckets[0], (binscolumn)] = np.nan
      
      mdf_test.loc[mdf_test[binscolumn] > buckets[-1], (binscolumn)] = np.nan

      # #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwo'
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
            labels = bins_id, precision=len(str(len(bins_id))))
    
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, binscolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #replace missing data with infill_activation
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)

      #returned data type is conditional on the size of encoding space
      max_encoding = len(bins_cuts) - 1
      if max_encoding <= 255:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint8)
      elif max_encoding <= 65535:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint16)
      else:
        mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]
    
    return mdf_test

  def _postprocess_DPnb(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPnb(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data with z-score normalization to mean 0 and sigma 1
    #adds data sampled from normal distribution with mean 0 and sigma 0.06 by default
    #where noise only injected to a subset of data based on flip_prob defaulting to 0.03
    #the noise properties may be customized with parameters 'mu', 'sigma', 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mu = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
      sigma = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      noisedistribution = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']
      
      if 'test_mu' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_mu = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu']
        test_sigma = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma']
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        test_noisedistribution = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_noisedistribution']
      else:
        #backward compatibility preceding 7.37
        test_mu = mu
        test_sigma = sigma
        test_flip_prob = flip_prob
        test_noisedistribution = noisedistribution
        
      if 'mu_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu_dist']
        sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_dist']
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu_dist']
        test_sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        mu_dist = False
        sigma_dist = False
        flip_prob_dist = False
        test_mu_dist = False
        test_sigma_dist = False
        test_flip_prob_dist = False
        
      if 'protected_feature' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        protected_feature = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['protected_feature']
        attribute_stdev_dict = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attribute_stdev_dict']
        attributes = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attributes']
        feature_stdev = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['feature_stdev']

        #applied in case of nan type drift from pickling and encryption
        attribute_stdev_dict = self.__convert_floatnan_to_npnan(attribute_stdev_dict)
        attributes = self.__convert_floatnan_to_npnan(attributes)

      else:
        #backward compatibility preceding 7.87
        protected_feature = False
        attribute_stdev_dict = {}
        attributes = []
        feature_stdev = 0

      if 'inplace' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        inplace = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      else:
        #backward compatibility preceding 7.62
        inplace = False
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }

      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
        
      if 'sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_list']
        if isinstance(sigma_list, list) and len(sigma_list) > 0:
          sigma = sigma_list
          
      if 'test_sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_list']
        if isinstance(test_sigma_list, list) and len(test_sigma_list) > 0:
          test_sigma = test_sigma_list
      
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(sigma, list):
        sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_sigma, list):
        test_sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
        
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')
      
      #________
        
      #scenarios where parameters passed as a scipy stats distribution
      if isinstance(mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(mu_dist, sampling_resource_dict, nprandom_dict, 'train')

      if isinstance(test_mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_mu_dist, sampling_resource_dict, nprandom_dict, 'test')

      if isinstance(sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(sigma_dist, sampling_resource_dict, nprandom_dict, 'train')
        if sigma < 0:
          sigma = 0

      if isinstance(test_sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_sigma_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_sigma < 0:
          test_sigma = 0

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
          
      #___

      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      DPnm_column = column + '_' + suffix
      
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        mu = mu
        sigma = sigma
        flip_prob = flip_prob
        noisedistribution = noisedistribution
      elif traindata is False:
        mu = test_mu
        sigma = test_sigma
        flip_prob = test_flip_prob
        noisedistribution = test_noisedistribution
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0 and sigma > 0:
      
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'

        #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
        if flip_prob == 1:
          binomial_samples = np.ones(mdf_test.shape[0]).astype(int)
        else:
          sampling_id = 'binomial' + sampling_type
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
        binomial_activation_count = np.sum(binomial_samples)

        #now derive our sampled noise for injection
        sampling_id = 'distribution' + sampling_type
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
        if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
          normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
          normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
          normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

        if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
          normal_samples = abs(normal_samples)
        elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
          normal_samples = (-1) * abs(normal_samples)

        #convert binolial sampels to column in dataframe
        mdf_test[DPnm_column] = pd.DataFrame(binomial_samples, index=mdf_test.index)

        #now multiply activations by the normal samples
        #this results in combining the Bernoulli and distribution samples into single set
        #this is equivalent to multiplying the Bernoulli and Distribution samples since Bernoulli had 0's and 1's
        mdf_test.loc[mdf_test[DPnm_column] == 1, DPnm_column] = normal_samples
        
        #___
        
        if protected_feature is not False:

          for attribute in attributes:

            if feature_stdev == feature_stdev and feature_stdev > 0:
              
              if attribute == attribute:
                #this applied the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature] == attribute, DPnm_column] *= \
                attribute_stdev_dict[attribute] / feature_stdev
              elif attribute != attribute:
                #this applied the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature].isna(), DPnm_column] *= \
                attribute_stdev_dict[attribute] / feature_stdev
        
        #___

        #now inject noise
        #this inplace support is a little different from convention
        #due to DPnm_column already being used to hold support data
        #has the same effect
        if inplace is True:
          mdf_test[column] = mdf_test[DPnm_column] + pd.to_numeric(mdf_test[column], errors='coerce')
          del mdf_test[DPnm_column]
          mdf_test.rename(columns = {column : DPnm_column}, inplace = True)

        elif inplace is False:
          mdf_test[DPnm_column] = mdf_test[DPnm_column] + pd.to_numeric(mdf_test[column], errors='coerce')
        
      else:
        #else test data is just pass-through
        if inplace is True:
          mdf_test.rename(columns = {column : DPnm_column}, inplace = True)
        elif inplace is False:
          mdf_test[DPnm_column] = pd.to_numeric(mdf_test[column], errors='coerce').copy()

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_DPmm(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPmm(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data min-max scaled within range 0-1
    #adds data sampled from normal distribution with mean 0 and sigma 0.03 by default
    #the noise properties may be customized with parameters 'mu', 'sigma'
    #also accepts parameter 'flip_prob' for ratio of data that will be adjusted (defaults to 1.)
    #noise is scaled based on the received points to keep within range 0-1
    #(e.g. for received data point 0.1, noise is scaled so as not to fall below -0.1)
    #gaussian noise source is also capped to maintain the range -0.5 to 0.5 (rare outlier points)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mu = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
      sigma = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      noisedistribution = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']
      
      if 'test_mu' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_mu = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu']
        test_sigma = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma']
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        test_noisedistribution = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_noisedistribution']
      else:
        #backward compatibility preceding 7.37
        test_mu = mu
        test_sigma = sigma
        test_flip_prob = flip_prob
        test_noisedistribution = noisedistribution
        
      if 'mu_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu_dist']
        sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_dist']
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu_dist']
        test_sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        mu_dist = False
        sigma_dist = False
        flip_prob_dist = False
        test_mu_dist = False
        test_sigma_dist = False
        test_flip_prob_dist = False
        
      if 'protected_feature' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        protected_feature = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['protected_feature']
        attribute_stdev_dict = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attribute_stdev_dict']
        attributes = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attributes']
        feature_stdev = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['feature_stdev']

        #applied in case of nan type drift from pickling and encryption
        attribute_stdev_dict = self.__convert_floatnan_to_npnan(attribute_stdev_dict)
        attributes = self.__convert_floatnan_to_npnan(attributes)

      else:
        #backward compatibility preceding 7.87
        protected_feature = False
        attribute_stdev_dict = {}
        attributes = set()
        feature_stdev = 0
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
        
      if 'sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_list']
        if isinstance(sigma_list, list) and len(sigma_list) > 0:
          sigma = sigma_list
          
      if 'test_sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_list']
        if isinstance(test_sigma_list, list) and len(test_sigma_list) > 0:
          test_sigma = test_sigma_list
      
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(sigma, list):
        sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_sigma, list):
        test_sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
        
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution
      if isinstance(mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(mu_dist, sampling_resource_dict, nprandom_dict, 'train')

      if isinstance(test_mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_mu_dist, sampling_resource_dict, nprandom_dict, 'test')

      if isinstance(sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(sigma_dist, sampling_resource_dict, nprandom_dict, 'train')
        if sigma < 0:
          sigma = 0

      if isinstance(test_sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_sigma_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_sigma < 0:
          test_sigma = 0

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
          
      #___
      
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

      DPmm_column = column + '_' + suffix
      DPmm_column_temp1 = column + '_' + suffix + '_tmp1'
      
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        mu = mu
        sigma = sigma
        flip_prob = flip_prob
        noisedistribution = noisedistribution
      elif traindata is False:
        mu = test_mu
        sigma = test_sigma
        flip_prob = test_flip_prob
        noisedistribution = test_noisedistribution
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0 and sigma > 0:
        
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'
        
        #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
        if flip_prob == 1:
          binomial_samples = np.ones(mdf_test.shape[0]).astype(int)
        else:
          sampling_id = 'binomial' + sampling_type
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
          binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))
        binomial_activation_count = np.sum(binomial_samples)

        #now derive our sampled noise for injection
        sampling_id = 'distribution' + sampling_type
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
        if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
          normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
          normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
          normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

        if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
          normal_samples = abs(normal_samples)
        elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
          normal_samples = (-1) * abs(normal_samples)

        #pass binomial samples to column in dataframe
        mdf_test[DPmm_column] = pd.DataFrame(binomial_samples, index=mdf_test.index)

        #now multiply activations by the normal samples
        mdf_test.loc[mdf_test[DPmm_column] == 1, DPmm_column] = normal_samples
        
        #___
        
        if protected_feature is not False:

          if feature_stdev == feature_stdev and feature_stdev > 0:

            for attribute in attributes:

              if attribute == attribute:
                #this applies the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature] == attribute, DPmm_column] *= \
                attribute_stdev_dict[attribute] / feature_stdev
              elif attribute != attribute:
                #this applies the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature].isna(), DPmm_column] *= \
                attribute_stdev_dict[attribute] / feature_stdev
        
        #___
        
        #cap outliers
        mdf_test = \
        self.__autowhere(mdf_test, DPmm_column, mdf_test[DPmm_column] < -0.5, -0.5, specified='replacement')
        mdf_test = \
        self.__autowhere(mdf_test, DPmm_column, mdf_test[DPmm_column] > 0.5, 0.5, specified='replacement')

        #support column to signal sign of noise, 0 is neg, 1 is pos
        mdf_test = \
        self.__autowhere(mdf_test, DPmm_column_temp1, mdf_test[DPmm_column] >= 0., 1, specified='replacement')
        
        #now inject noise, with scaled noise to maintain range 0-1
        #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
        mdf_test = \
        self.__autowhere(mdf_test, 
                        DPmm_column, 
                        mdf_test[column] < 0.5, 
                        mdf_test[column] + \
                        (1 - mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column] * mdf_test[column] / 0.5) + \
                        (mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column]), \
                        specified='replacement')

        mdf_test = \
        self.__autowhere(mdf_test, 
                        DPmm_column, 
                        mdf_test[column] >= 0.5, 
                        mdf_test[column] + \
                        (1 - mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column]) + \
                        (mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column] * (1 - mdf_test[column]) / 0.5), \
                        specified='replacement')

        #adjacent cell infill (this is included as a precaution shouldn't have any effect since upstream normalization)
        mdf_test[DPmm_column] = mdf_test[DPmm_column].fillna(method='ffill')
        mdf_test[DPmm_column] = mdf_test[DPmm_column].fillna(method='bfill')

        #remove support column
        del mdf_test[DPmm_column_temp1]

      else:
        
        #for test data is just pass-through
        mdf_test[DPmm_column] = mdf_test[column].copy()

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_DPrt(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_DPrt 
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #followed by a noise injection similar to DPrt based based on this set's retn range
    
    #replaces missing or improperly formatted data with mean of remaining values
    #(prior to noise injection)
    
    #returns same dataframes with new column of name column + '_DPrt'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
      minimum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
      maximum = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
      scalingapproach = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scalingapproach']
      offset = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
      multiplier = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
      cap = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
      floor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
      divisor = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      
      mu = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
      sigma = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      noisedistribution = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']
      
      if 'test_mu' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_mu = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu']
        test_sigma = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma']
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        test_noisedistribution = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_noisedistribution']
      else:
        #backward compatibility preceding 7.37
        test_mu = mu
        test_sigma = sigma
        test_flip_prob = flip_prob
        test_noisedistribution = noisedistribution
        
      if 'mu_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu_dist']
        sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_dist']
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_mu_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_mu_dist']
        test_sigma_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        mu_dist = False
        sigma_dist = False
        flip_prob_dist = False
        test_mu_dist = False
        test_sigma_dist = False
        test_flip_prob_dist = False
        
      if 'protected_feature' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        protected_feature = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['protected_feature']
        attribute_stdev_dict = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attribute_stdev_dict']
        attributes = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attributes']
        feature_stdev = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['feature_stdev']

        #applied in case of nan type drift from pickling and encryption
        attribute_stdev_dict = self.__convert_floatnan_to_npnan(attribute_stdev_dict)
        attributes = self.__convert_floatnan_to_npnan(attributes)

      else:
        #backward compatibility preceding 7.87
        protected_feature = False
        attribute_stdev_dict = {}
        attributes = set()
        feature_stdev = 0
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
        
      if 'sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma_list']
        if isinstance(sigma_list, list) and len(sigma_list) > 0:
          sigma = sigma_list
          
      if 'test_sigma_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_sigma_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_sigma_list']
        if isinstance(test_sigma_list, list) and len(test_sigma_list) > 0:
          test_sigma = test_sigma_list
      
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(sigma, list):
        sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(sigma_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_sigma, list):
        test_sigma, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_sigma_list, sampling_resource_dict, nprandom_dict, 'test')
        
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution
      if isinstance(mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(mu_dist, sampling_resource_dict, nprandom_dict, 'train')

      if isinstance(test_mu_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_mu, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_mu_dist, sampling_resource_dict, nprandom_dict, 'test')

      if isinstance(sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(sigma_dist, sampling_resource_dict, nprandom_dict, 'train')
        if sigma < 0:
          sigma = 0

      if isinstance(test_sigma_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_sigma, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_sigma_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_sigma < 0:
          test_sigma = 0

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
          
      #___

      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      
      #initialize returned column and support columns
      DPrt_column = column + '_' + suffix
      DPrt_column_temp1 = column + '_' + suffix + '_tmp1'
      DPrt_column_temp2 = column + '_' + suffix + '_tmp2'
      
      #copy original column for implementation
      mdf_test[DPrt_column] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[DPrt_column] = pd.to_numeric(mdf_test[DPrt_column], errors='coerce')
      
      if cap is not False:
        #replace values in test > cap with cap
        mdf_test.loc[mdf_test[DPrt_column] > cap, (DPrt_column)] \
        = cap
      
      if floor is not False:
        #replace values in test < floor with floor
        mdf_test.loc[mdf_test[DPrt_column] < floor, (DPrt_column)] \
        = floor

      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, DPrt_column, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #avoid outlier div by zero when max = min
      maxminusmin = maximum - minimum
      if maxminusmin == 0:
        maxminusmin = 1
      
      if maximum >= 0 and minimum <= 0:
        scalingapproach = 'retn'
      elif maximum >= 0 and minimum >= 0:
        scalingapproach = 'mnmx'
      elif maximum <= 0 and minimum <= 0:
        scalingapproach = 'mxmn'
      
      #initially we'll scale comparable to min-max, and then after noise convert to retain scaling
      #note that any multiplier or offset will be applied after conversion to retain
      
      #scale to min-max

      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - minimum) / \
                                   (maxminusmin)
        
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        mu = mu
        sigma = sigma
        flip_prob = flip_prob
        noisedistribution = noisedistribution
      elif traindata is False:
        mu = test_mu
        sigma = test_sigma
        flip_prob = test_flip_prob
        noisedistribution = test_noisedistribution
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0 and sigma > 0:
        
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'
      
        #first we'll derive our Bernoulli samples which will tell us how many samples we need from normal
        if flip_prob == 1:
          binomial_samples = np.ones(mdf_test.shape[0]).astype(int)
        else:
          sampling_id = 'binomial' + sampling_type
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
          binomial_samples = nprandom.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))
        binomial_activation_count = np.sum(binomial_samples)

        #now derive our sampled noise for injection
        sampling_id = 'distribution' + sampling_type
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += binomial_activation_count
        if noisedistribution in {'normal', 'abs_normal', 'negabs_normal'}:
          normal_samples = nprandom.normal(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'laplace', 'abs_laplace', 'negabs_laplace'}:
          normal_samples = nprandom.laplace(loc=mu, scale=sigma, size=(binomial_activation_count))
        elif noisedistribution in {'uniform', 'abs_uniform', 'negabs_uniform'}:
          normal_samples = nprandom.uniform(low=(mu-sigma), high=(sigma-mu), size=(binomial_activation_count))

        if noisedistribution in {'abs_normal', 'abs_laplace', 'abs_uniform'}:
          normal_samples = abs(normal_samples)
        elif noisedistribution in {'negabs_normal', 'negabs_laplace', 'negabs_uniform'}:
          normal_samples = (-1) * abs(normal_samples)

        #pass binomial samples to column in dataframe
        mdf_test[DPrt_column_temp2] = pd.DataFrame(binomial_samples, index=mdf_test.index)

        #now multiply activations by the normal samples
        mdf_test.loc[mdf_test[DPrt_column_temp2] == 1, DPrt_column_temp2] = normal_samples
        
        #___
        
        if protected_feature is not False:

          if feature_stdev == feature_stdev and feature_stdev > 0:

            for attribute in attributes:
              
              if attribute == attribute:
                #this applies the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature] == attribute, DPrt_column_temp2] *= \
                attribute_stdev_dict[attribute] / feature_stdev
              elif attribute != attribute:
                #this applies the noise scaling
                #note that mdf_test[DPnm_column] will have entries of 0 or distribution samples
                mdf_test.loc[mdf_test[protected_feature].isna(), DPrt_column_temp2] *= \
                attribute_stdev_dict[attribute] / feature_stdev
        
        #___
        
        #cap outliers
        mdf_test = \
        self.__autowhere(mdf_test, DPrt_column_temp2, mdf_test[DPrt_column] < -0.5, -0.5, specified='replacement')

        mdf_test = \
        self.__autowhere(mdf_test, DPrt_column_temp2, mdf_test[DPrt_column] > 0.5, 0.5, specified='replacement')
        
        #support column to signal sign of noise, 0 is neg, 1 is pos
        mdf_test = \
        self.__autowhere(mdf_test, DPrt_column_temp1, mdf_test[DPrt_column_temp2] >= 0., 1, specified='replacement')
        
        #now inject noise, with scaled noise to maintain range 0-1
        #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
        mdf_test = \
        self.__autowhere(mdf_test, 
                        DPrt_column, 
                        mdf_test[DPrt_column] < 0.5,
                        mdf_test[DPrt_column] + \
                        (1 - mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2] * mdf_test[DPrt_column] / 0.5) + \
                        (mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2]), \
                        specified='replacement')

        mdf_test = \
        self.__autowhere(mdf_test, 
                        DPrt_column, 
                        mdf_test[DPrt_column] >= 0.5,
                        mdf_test[DPrt_column] + \
                        (1 - mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2]) + \
                        (mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2] * (1 - mdf_test[DPrt_column]) / 0.5), \
                        specified='replacement')
        
        #remove support columns
        del mdf_test[DPrt_column_temp1]
        del mdf_test[DPrt_column_temp2]

        #now that we've injected noise, will convert the minmax representation to 'retain' normalization
        if scalingapproach == 'retn':
          mdf_test[DPrt_column] = (mdf_test[DPrt_column] + (minimum / divisor) ) * multiplier + offset
        elif scalingapproach == 'mnmx':
          mdf_test[DPrt_column] = (mdf_test[DPrt_column]) * multiplier + offset
        elif scalingapproach == 'mxmn':
          mdf_test[DPrt_column] = (mdf_test[DPrt_column] - 1) * multiplier + offset

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_DPbn(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPbn(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is bnry encoded data (i.e. boolean integers in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #unless testnoise is activated which applies test noise injection in both automunge and postmunge
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      
      if 'test_flip_prob' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
      else:
        #backward compatibility preceding 7.37
        test_flip_prob = flip_prob
        
      #___
        
      if 'flip_prob_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        flip_prob_dist = False
        test_flip_prob_dist = False
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
        
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
        
      #___
      
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

      DPbn_column = column + '_' + suffix
      
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        flip_prob = flip_prob
      elif traindata is False:
        flip_prob = test_flip_prob
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0:
        
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'
        
        #first we'll derive our sampled noise for injection
        sampling_id = 'binomial' + sampling_type
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
        mdf_test[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)
      
        #now inject noise
        mdf_test[DPbn_column] = abs(mdf_test[column] - mdf_test[DPbn_column])
        
      else:
        
        #for test data is just pass-through
        mdf_test[DPbn_column] = mdf_test[column].copy()

      #change data types to 8-bit (1 byte) integers for memory savings
      mdf_test[DPbn_column] = mdf_test[DPbn_column].astype(np.int8)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_DPod(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPod(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is ordinal encoded data (i.e. categoric by integer in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #when flip activated selects from the set of encodings per level random draw
    #(including potenitally the current encoding for no flip)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #unless testnoise is activated which applies test noise injection in both automunge and postmunge
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      ord_encodings = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ord_encodings']
      if 'weighted' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        weighted = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['weighted']
        weights = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['weights']
      else:
        #backward compatibility preceding 7.20
        weighted = False
        weights = []
        
      if 'test_flip_prob' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        test_weighted = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_weighted']
      else:
        #backward compatibility preceding 7.37
        test_flip_prob = flip_prob
        test_weighted = weighted
        
      #___
        
      if 'flip_prob_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        flip_prob_dist = False
        test_flip_prob_dist = False
        
      #___
      
      if 'protected_feature' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        protected_feature = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['protected_feature']
        attribute_weightings_dict = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attribute_weightings_dict']
        attributes = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attributes']
        inplace = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']

        #applied in case of nan type drift from pickling and encryption
        attribute_weightings_dict = self.__convert_floatnan_to_npnan(attribute_weightings_dict)
        attributes = self.__convert_floatnan_to_npnan(attributes)

      else:
        #backward compatibility preceding 7.48
        protected_feature = False
        attribute_weightings_dict = {}
        attributes = set()
        inplace = False

      if 'passthrough' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        passthrough = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['passthrough']
      else:
        passthrough = False
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
        
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')

      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
        
      #___

      DPod_column = column + '_' + suffix
      DPod_tempcolumn1 = 1
      DPod_tempcolumn2 = 2
      
      if inplace is not True:
        #copy source column into new column
        mdf_test[DPod_column] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : DPod_column}, inplace = True)
      
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        flip_prob = flip_prob
        weighted = weighted
      elif traindata is False:
        flip_prob = test_flip_prob
        weighted = test_weighted
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0:
        
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'
        
        if flip_prob == 1:
          mdf_test[DPod_tempcolumn1] = np.ones(mdf_test.shape[0]).astype(int)
        else:
          #first we'll derive our sampled noise for injection
          sampling_id = 'binomial' + sampling_type
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
          mdf_test[DPod_tempcolumn1] = pd.DataFrame(nprandom.binomial(n=1, p=test_flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)
        binomial_activation_count = mdf_test[DPod_tempcolumn1].sum()

        sampling_id = 'choice' + sampling_type
        
        if protected_feature is False or weighted is False:

          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]

          if weighted is False:
            choice_samples = nprandom.choice(ord_encodings, size=(binomial_activation_count))
          elif weighted is True:
            choice_samples = nprandom.choice(ord_encodings, p=weights, size=(binomial_activation_count))

          #the samples were a shape based on number of binomial activations, now inject
          mdf_test.loc[mdf_test[DPod_tempcolumn1] == 1, DPod_column] = choice_samples
        
        elif protected_feature is not False:
          
          seeds_available = sampling_resource_dict[sampling_id + '_seeds']
          
          #for test data we'll also consider test attributes that weren't found in the train data
          #and apply aggregate feature weighting
          #since attribute_weightings_dict is inspected as part of sampling we'll need to add temporary entries
          temp_attribute_weightings_dict = self.__autocopy(attribute_weightings_dict)

          test_attributes = list(mdf_test[protected_feature].unique())
          extra_test_attributes = set(test_attributes) - set(attributes)

          for extra_test_attribute in extra_test_attributes:
            temp_attribute_weightings_dict.update({extra_test_attribute : weights})

          for attribute in (attributes + list(extra_test_attributes)):

            temp_sampling_resource_dict = self.__autocopy(sampling_resource_dict)

            if attribute == attribute:
              attribute_count = mdf_test.loc[(mdf_test[protected_feature]==attribute) & (mdf_test[DPod_tempcolumn1] == 1)].shape[0]
            elif attribute != attribute:
              attribute_count = mdf_test.loc[(mdf_test[protected_feature].isna()) & (mdf_test[DPod_tempcolumn1] == 1)].shape[0]

            attribute_seeds = seeds_available
            
          if attribute_count < len(seeds_available):
            
            attribute_seeds = seeds_available[:attribute_count]
            
          temp_sampling_resource_dict.update({sampling_id + '_seeds' : attribute_seeds})
          
          nprandom = self.__get_nprandom(sampling_id, temp_sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += attribute_count
          
          choice_samples = nprandom.choice(ord_encodings, 
                                           p=temp_attribute_weightings_dict[attribute], 
                                           size=(attribute_count))
          
          if attribute == attribute:
            #the samples were a shape based on number of binomial activations, now inject
            mdf_test.loc[(mdf_test[protected_feature]==attribute) & (mdf_test[DPod_tempcolumn1] == 1), DPod_column] = choice_samples
            
          elif attribute != attribute:
            #the samples were a shape based on number of binomial activations, now inject
            mdf_test.loc[(mdf_test[protected_feature].isna()) & (mdf_test[DPod_tempcolumn1] == 1), DPod_column] = choice_samples
          
          #if we have enough seeds left for the next attribute we'll expend the current used
          if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
            next_attribute = attributes[(1+attributes.index(attribute))]
            next_attribute_count = mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0]

            if len(seeds_available[attribute_count:]) < next_attribute_count:
              seeds_available = seeds_available[attribute_count:]

        del mdf_test[DPod_tempcolumn1]
        
      #___

      if passthrough is False:
        #data type is conditional based on encoding space
        max_encoding = len(ord_encodings) - 1
        if max_encoding <= 255:
          mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint8)
        elif max_encoding <= 65535:
          mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint16)
        else:
          mdf_test[DPod_column] = mdf_test[DPod_column].astype(np.uint32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_DPmc(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess correpsonding to _process_DPmc
    #in addition to assignparam testnoise parameter
    #noise injection may be performed based on postmunge traindata parameter
    #otherwise this is a passthrough transform
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      inputtextcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
      textcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      textlabelsdict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      weighted = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['weighted']
      weights = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['weights']
      #df_unique only saved for first textcolumn
      df_unique = \
      postprocess_dict['column_dict'][textcolumns[0]]['normalization_dict'][textcolumns[0]]['df_unique']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      maxencodings = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxencodings']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      
      if 'test_flip_prob' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        test_weighted = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_weighted']
      else:
        #backward compatibility preceding 7.37
        test_flip_prob = flip_prob
        test_weighted = weighted
        
      if 'swap_noise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        swap_noise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['swap_noise']
      else:
        #backward compatibility preceding 7.49
        swap_noise = False

      if 'messy_data_headers' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        messy_data_headers = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['messy_data_headers']
      else:
        #backward compatibility preceding 8.10
        messy_data_headers = False
        
      if 'protected_feature' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        protected_feature = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['protected_feature']
        attribute_weightings_dict = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attribute_weightings_dict']
        attributes = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['attributes']

        #applied in case of nan type drift from pickling and encryption
        attribute_weightings_dict = self.__convert_floatnan_to_npnan(attribute_weightings_dict)
        attributes = self.__convert_floatnan_to_npnan(attributes)

      else:
        #backward compatibility preceding 7.87
        protected_feature = False
        attribute_weightings_dict = {}
        attributes = []

      #___
        
      if 'flip_prob_dist' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
        test_flip_prob_dist = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
      else:
        #backward compatibility preceding 7.48
        flip_prob_dist = False
        test_flip_prob_dist = False
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma

      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
        
      #___

      if isinstance(messy_data_headers, list):

        none_categorical = True
        for inputtextcolumn in inputtextcolumns:
          if isinstance(mdf_test[inputtextcolumn].dtype, pd.api.types.CategoricalDtype):
            none_categorical = False

        if none_categorical is False:
          mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].astype('object')

        #since messy_data_headers is for input headers, we'll apply an adjinfill
        #since DPmc needs all valid entries
        #apply ffill to replace NArows with value from adjacent cell in preceding row
        mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(method='ffill')

        #we'll follow with a bfill just in case first row had a nan
        mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(method='bfill')

        mdf_test[inputtextcolumns] = mdf_test[inputtextcolumns].fillna(0)
      
      if inplace is not True:
        for inputtextcolumn in inputtextcolumns:
          mdf_test[textlabelsdict[inputtextcolumn]] = mdf_test[inputtextcolumn].copy()
      else:
        mdf_test.rename(columns = textlabelsdict, inplace = True)
        
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      def _noise_inject(df, textcolumns, df_unique, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, traintest, protected_feature=False):
        """
        common support function used to inject noise to either mdf_train or mdf_test when applicable
        """

        sampling_resource_dict = self.__autocopy(sampling_resource_dict)

        unique_count = df_unique.shape[0]
        unique_range = list(range(unique_count))

        #df_noise is support dataframe for noise injection to data
        df_noise = pd.DataFrame(index = df.index)
        df_noise_tempcolumn1 = 1
        df_noise_tempcolumn2 = 2

        if flip_prob == 1:
          df_noise[df_noise_tempcolumn1] = np.ones(df.shape[0]).astype(int)
        else:
          #df_noise_tempcolumn1 will return 1 for rows receiving injection and 0 elsewhere
          sampling_id = 'binomial_' + traintest
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
          sampling_resource_dict[sampling_id + '_call_count'] += 1
          sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
          df_noise[df_noise_tempcolumn1] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(df.shape[0])), index=df.index)
        binomial_activation_count = df_noise[df_noise_tempcolumn1].sum()

        #_____
        if protected_feature is not False:

          seeds_available = sampling_resource_dict['choice_' + traintest + '_seeds']

          if binomial_activation_count < len(seeds_available):
            
            #in the protectect_feature case this function is being called on an attribute partition
            attribute_seeds = seeds_available[:binomial_activation_count]

            sampling_id = 'choice_' + traintest
            sampling_resource_dict.update({sampling_id + '_seeds' : attribute_seeds})

            nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)

        elif protected_feature is False:

          sampling_id = 'choice_' + traintest
          nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)

        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += df.shape[0]
        if weighted is False:
          choice_samples = nprandom.choice(unique_range, size=(binomial_activation_count))
        elif weighted is True:
          choice_samples = nprandom.choice(unique_range, p=weights, size=(binomial_activation_count))

        #the samples were a shape based on number of binomial activations, now extract to full column
        df_noise.loc[:, df_noise_tempcolumn2] = 0
        df_noise.loc[df_noise[df_noise_tempcolumn1] == 1, df_noise_tempcolumn2] = choice_samples

        #df_unique2 will be used to populate replacement activation sets corresponding to indexes sampled in df_noise[df_noise_tempcolumn2]
        df_unique2 = df_unique.iloc[df_noise[df_noise_tempcolumn2]]
        df_unique2.index = df.index

        #now we can carry the values from df_unique2 to replace entries in df associated with noise injection
        #for cases where df_noise[df_noise_tempcolumn1] was sampled as 1
        #here the columns in df_unique2 match the columns in df[textcolumns]
        for textcolumn in textcolumns:

          #setting to dataframe for single column case
          df = \
          self.__autowhere(df, textcolumn, df_noise[df_noise_tempcolumn1] == 1, df_unique2[textcolumn], specified='replacement')

        return df, sampling_resource_dict, binomial_activation_count
      
      if traindata is True:
        flip_prob = flip_prob
        weighted = weighted
      elif traindata is False:
        flip_prob = test_flip_prob
        weighted = test_weighted

      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0:
        
        if traindata is True:
          traintest = 'train'
        if traindata is False:
          traintest = 'test'
        
        if swap_noise is False:
          if protected_feature is False: 
            #inject noise to mdf_test
            mdf_test, sampling_resource_dict, binomial_activation_count = \
            _noise_inject(mdf_test, textcolumns, df_unique, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, traintest)
          elif protected_feature is not False:
            
            binomial_activation_count = 0
            
            #first we'll apply noise without scaling for test qattributes that weren't found in train set
            #this is neccesary because we're inspecting protected_feature in the input state
            test_attributes = list(mdf_test[protected_feature].unique())
            extra_test_attributes = set(test_attributes) - set(attributes)
            
            if len(extra_test_attributes) > 0:

              returned_noise_inject_df, temp_sampling_resource_dict, binomial_activation_count_attribute = \
              _noise_inject(mdf_test.loc[mdf_test[protected_feature].isin(extra_test_attributes)].copy(), 
                            textcolumns, 
                            df_unique, 
                            flip_prob, 
                            weighted, 
                            weights, 
                            sampling_resource_dict, 
                            nprandom_dict, 
                            traintest,
                            protected_feature)

              for textcolumn in textcolumns:
                mdf_test.loc[mdf_test[protected_feature].isin(extra_test_attributes), textcolumn] = \
                returned_noise_inject_df[textcolumn]
              del returned_noise_inject_df

              binomial_activation_count += binomial_activation_count_attribute
              
              #if we have enough seeds left for the next attribute we'll expend the current used
              if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
                next_attribute = attributes[0]

                if next_attribute == next_attribute:
                  next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0] * flip_prob * 0.15)
                else:
                  next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature].isna()].shape[0] * flip_prob * 0.15)

                if len(sampling_resource_dict['choice_' + traintest + '_seeds'][binomial_activation_count_attribute:]) < next_attribute_count:                
                  #this expends the used seeds
                  sampling_resource_dict['choice_' + traintest + '_seeds'] = sampling_resource_dict['choice_' + traintest + '_seeds'][binomial_activation_count_attribute:]
                  
              #for protected_feature case we updated seeds for choice, now revert to orig but keep other logged entries
              temp_sampling_resource_dict.update({'choice_' + traintest + '_seeds' : sampling_resource_dict['choice_' + traintest + '_seeds']})
              sampling_resource_dict = temp_sampling_resource_dict
              
              #now cycle through the attributes
              for attribute in attributes:
                
                if attribute == attribute:
                  returned_noise_inject_df, temp_sampling_resource_dict, binomial_activation_count_attribute = \
                  _noise_inject(mdf_test.loc[mdf_test[protected_feature] == attribute].copy(), 
                                textcolumns, 
                                attribute_weightings_dict[attribute]['df_unique'], 
                                flip_prob, 
                                weighted, 
                                attribute_weightings_dict[attribute]['weights'], 
                                sampling_resource_dict, 
                                nprandom_dict, 
                                traintest,
                                protected_feature)

                  for textcolumn in textcolumns:
                    mdf_test.loc[mdf_test[protected_feature] == attribute, textcolumn] = returned_noise_inject_df[textcolumn]
                  del returned_noise_inject_df

                elif attribute != attribute:
                  returned_noise_inject_df, temp_sampling_resource_dict, binomial_activation_count_attribute = \
                  _noise_inject(mdf_test.loc[mdf_test[protected_feature].isna()].copy(), 
                                textcolumns, 
                                attribute_weightings_dict[attribute]['df_unique'], 
                                flip_prob, 
                                weighted, 
                                attribute_weightings_dict[attribute]['weights'], 
                                sampling_resource_dict, 
                                nprandom_dict, 
                                traintest,
                                protected_feature)

                  for textcolumn in textcolumns:
                    mdf_test.loc[mdf_test[protected_feature].isna(), textcolumn] = returned_noise_inject_df[textcolumn]
                  del returned_noise_inject_df

                binomial_activation_count += binomial_activation_count_attribute
                
                #if we have enough seeds left for the next attribute we'll expend the current used
                if attribute != attributes[-1] and not (attribute != attribute and attributes[-1] != attributes[-1]):
                  next_attribute = attributes[(1+attributes.index(attribute))]

                  if next_attribute == next_attribute:
                    next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature]==next_attribute].shape[0] * flip_prob * 0.15)
                  else:
                    next_attribute_count = int(mdf_test.loc[mdf_test[protected_feature].isna()].shape[0] * flip_prob * 0.15)

                  if len(sampling_resource_dict['choice_' + traintest + '_seeds'][binomial_activation_count_attribute:]) < next_attribute_count:                
                    #this expends the used seeds
                    sampling_resource_dict['choice_' + traintest + '_seeds'] = sampling_resource_dict['choice_' + traintest + '_seeds'][binomial_activation_count_attribute:]

                #for protected_feature case we updated seeds for choice, now revert to orig but keep other logged entries
                temp_sampling_resource_dict.update({'choice_' + traintest + '_seeds' : sampling_resource_dict['choice_' + traintest + '_seeds']})
                sampling_resource_dict = temp_sampling_resource_dict

        elif swap_noise is True:
          mdf_test, sampling_resource_dict, binomial_activation_count = \
          _noise_inject(mdf_test, textcolumns, mdf_test, flip_prob, weighted, weights, sampling_resource_dict, nprandom_dict, traintest)
        
      #now apply data type conversion, this should align with received data types, just applying in case of drift
      #in swap_noise scenario will defer to the upstream data type
      if swap_noise is False and not isinstance(messy_data_headers, list):

        for textcolumn in textcolumns:

          max_encoding_for_dtype_convert = maxencodings[textcolumn]

          if max_encoding_for_dtype_convert <= 255:
            mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint8)
          elif max_encoding_for_dtype_convert <= 65535:
            mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint16)
          else:
            mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.uint32)
      
    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        if column in postprocess_dict['column_dict']:
          inputtextcolumns = postprocess_dict['column_dict'][column]['categorylist']
        else:
          #origcolumn scenario when not applied as downstream transform
          inputtextcolumns = [column]
          
        for inputtextcolumn in inputtextcolumns:
          del mdf_test[inputtextcolumn]
    
    return mdf_test

  def _postprocess_DPsk(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPsk(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #makes no assumption about input data
    #injects a mask into sampled rows based on a binomial sampling
    #mask defaults to integer 0, can be specified by mask_value parameter
    #accepts flip_prob and test_flip_prob to set injection rate
    #flip_prob/test_flip_prob can also be passed as list of condidate values or as scipy stats distribution
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:

      mask_value = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mask_value']
      
      if 'additive' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        additive = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['additive']
      else:
        #backward compatibility preceding 7.86
        additive = False
      
      flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
      
      test_flip_prob = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob']
        
      flip_prob_dist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_dist']
      test_flip_prob_dist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_dist']
        
      #________

      #note that random_generator accessed from postmunge(.) parameter and may not be same as applied in automunge(.)
      if 'random_generator' in params:
        random_generator = params['random_generator']
      else:
        random_generator = np.random.PCG64
        
      nprandom_dict = {'custom' : random_generator,
                       'default': np.random.PCG64}
      
      #note that sampling_resource_dict populated externally 
      #based on automunge(.) sampling_dict and entropy_seeds parameters 
      #and not passed to postmunge
      #postmunge(.) has corresponding parameters to support
      if 'sampling_resource_dict' in params:
        sampling_resource_dict = params['sampling_resource_dict']
      else:
        #sampling_resource_dict initialized here for backward compatibility
        sampling_resource_dict = {'seeding_type' : 'supplemental_seeds', 
                                  'binomial_train' : 'custom',
                                  'binomial_train_seeds' : [],
                                  'binomial_train_call_count' : 0,
                                  'binomial_train_sample_count' : 0,
                                  'binomial_test' : 'custom',
                                  'binomial_test_seeds' : [],
                                  'binomial_test_call_count' : 0,
                                  'binomial_test_sample_count' : 0,
                                  'distribution_train' : 'custom',
                                  'distribution_train_seeds' : [],
                                  'distribution_train_call_count' : 0,
                                  'distribution_train_sample_count' : 0,
                                  'distribution_test' : 'custom',
                                  'distribution_test_seeds' : [],
                                  'distribution_test_call_count' : 0,
                                  'distribution_test_sample_count' : 0,
                                  'choice_train' : 'custom',
                                  'choice_train_seeds' : [],
                                  'choice_train_call_count' : 0,
                                  'choice_train_sample_count' : 0,
                                  'choice_test' : 'custom',
                                  'choice_test_seeds' : [],
                                  'choice_test_call_count' : 0,
                                  'choice_test_sample_count' : 0,
                                  'parameterlist_train' : 'custom',
                                  'parameterlist_train_seeds' : [],
                                  'parameterlist_train_call_count' : 0,
                                  'parameterlist_train_sample_count' : 0,
                                  'parameterlist_test' : 'custom',
                                  'parameterlist_test_seeds' : [],
                                  'parameterlist_test_call_count' : 0,
                                  'parameterlist_test_sample_count' : 0,
                                  'statsdistribution_train' : 'custom',
                                  'statsdistribution_train_seeds' : [],
                                  'statsdistribution_train_call_count' : 0,
                                  'statsdistribution_train_sample_count' : 0,
                                  'statsdistribution_test' : 'custom',
                                  'statsdistribution_test_seeds' : [],
                                  'statsdistribution_test_call_count' : 0,
                                  'statsdistribution_test_sample_count' : 0,
                                  'random_generator_accepts_seeds' : True,
                                 }
      #________
      
      #scenarios where parameters passed as a list of candidates, options available for flip_prob/sigma
      
      if 'flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob_list']
        if isinstance(flip_prob_list, list) and len(flip_prob_list) > 0:
          flip_prob = flip_prob_list
          
      if 'test_flip_prob_list' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        test_flip_prob_list = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_flip_prob_list']
        if isinstance(test_flip_prob_list, list) and len(test_flip_prob_list) > 0:
          test_flip_prob = test_flip_prob_list
      
      if isinstance(flip_prob, list):
        flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(flip_prob_list, sampling_resource_dict, nprandom_dict, 'train')
        
      if isinstance(test_flip_prob, list):
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_parameter_list(test_flip_prob_list, sampling_resource_dict, nprandom_dict, 'test')

      #________
        
      #scenarios where parameters passed as a scipy stats distribution

      if isinstance(flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is True:
        flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(flip_prob_dist, sampling_resource_dict, nprandom_dict, 'train')
        if flip_prob < 0:
          flip_prob = 0
        if flip_prob > 1:
          flip_prob = 1

      if isinstance(test_flip_prob_dist, type(stats.expon(1))) and postprocess_dict['traindata'] is False:
        test_flip_prob, sampling_resource_dict = \
        self.__sample_from_scipy_stats(test_flip_prob_dist, sampling_resource_dict, nprandom_dict, 'test')
        if test_flip_prob < 0:
          test_flip_prob = 0
        if test_flip_prob > 1:
          test_flip_prob = 1
        
      #___
      
      testnoise = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['testnoise']
      trainnoise = True
      #backward compatibility - trainnoise added in 7.30
      if 'trainnoise' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
        trainnoise = \
        postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['trainnoise']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

      DPbn_column = column + '_' + suffix
      
      #check if df_test is to be treated as train or test data
      traindata = postprocess_dict['traindata']
      
      if traindata is True:
        flip_prob = flip_prob
      elif traindata is False:
        flip_prob = test_flip_prob
      
      #this is consistent with no noise injected for traindata scenarios of 'train_no_noise' or 'test_no_noise'
      if ( (trainnoise is True and traindata is True) \
      or (testnoise is True and traindata is False) ) \
      and flip_prob > 0:
        
        if traindata is True:
          sampling_type = '_train'
        elif traindata is False:
          sampling_type = '_test'
        
        #first we'll derive our sampled noise for injection
        sampling_id = 'binomial' + sampling_type
        nprandom = self.__get_nprandom(sampling_id, sampling_resource_dict, nprandom_dict)
        sampling_resource_dict[sampling_id + '_call_count'] += 1
        sampling_resource_dict[sampling_id + '_sample_count'] += mdf_test.shape[0]
        mdf_test[DPbn_column] = pd.DataFrame(nprandom.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0])), index=mdf_test.index)
        
        if additive is False:
          #now inject noise
          mdf_test = \
          self.__autowhere(mdf_test, DPbn_column, mdf_test[DPbn_column]==1, mask_value, mdf_test[column], specified='replacementalternative')
        elif additive is True:
          #this adds the additive value only onto numeric entries to avoid bug for adding number to string
          mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] = \
          mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] * mask_value \
          + mdf_test.loc[mdf_test[column]==pd.to_numeric(mdf_test[column], errors='coerce'), column]
          #and this carries over non-numeric entries from column to the returned column DPbn_column
          mdf_test.loc[mdf_test[column]!=pd.to_numeric(mdf_test[column], errors='coerce'), DPbn_column] = \
          mdf_test.loc[mdf_test[column]!=pd.to_numeric(mdf_test[column], errors='coerce'), column]
        
      else:
        
        #for test data is just pass-through
        mdf_test[DPbn_column] = mdf_test[column].copy()

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  def _postprocess_exc2(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      
      exclcolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[exclcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
      
      #del df[column]
      
      mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
      
      #fillvalue = mdf_train[exclcolumn].mode()[0]
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, exclcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test
  
  def _postprocess_exc5(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''

    #normkey used to retrieve the normalization dictionary 
    normkey = False
    if len(columnkey) > 0:      
      normkey = columnkey[0]
          
    #normkey is False when process function returns empty set
    if normkey is not False:
      
      defaultinfill_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['defaultinfill_dict']
      inplace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inplace']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      encodingspace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['encodingspace']
      integertype = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['integertype']
      
      exclcolumn = column + '_' + suffix
      
      if inplace is not True:
        mdf_test[exclcolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
      
      #del df[column]
      
      mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
      
      #non integers are subject to infill
      mdf_test = \
      self.__autowhere(mdf_test, exclcolumn, mdf_test[exclcolumn] == mdf_test[exclcolumn].round(), alternative=np.nan, specified='alternative')
      
      #fillvalue = mdf_train[exclcolumn].mode()[0]
      
      #apply defaultinfill based on processdict entry
      mdf_test, _1 = \
      self.__apply_defaultinfill(mdf_test, exclcolumn, postprocess_dict, treecategory=False, defaultinfill_dict=defaultinfill_dict)
      
      #set data type based on integertype parameter
      if integertype == 'singlct':

        if encodingspace <= 255:
          mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint8)
        elif encodingspace <= 65535:
          mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint16)
        else:
          mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.uint32)

      if integertype == 'integer':

        mdf_test[exclcolumn] = mdf_test[exclcolumn].astype(np.int32)

    else:

      if 'inplace' in params:
        inplace = params['inplace']
      else:
        inplace = False

      if inplace is True:
        del mdf_test[column]

    return mdf_test

  #__FunctionBlock: postmunge ML infill

  def __createpostMLinfillsets(self, df_test, column, testNArows, category, ML_cmnd, \
                             postprocess_dict, columnslist = [], categorylist = []):
    '''
    #createpostMLinfillsets(df_test, column, testNArows, category, \
    #columnslist = []) function that when fed dataframe of
    #test set, column id, df of True/False corresponding to rows from original \
    #sets with missing values, a string category of 'text', 'date', 'nmbr', or \
    #'bnry', and a list of column id's for the text category if applicable. The \
    #function returns a series of dataframes which can be applied to apply a \
    #machine learning model previously trained on our train set as part of the 
    #original automunge application to predict apppropriate infill values for those\
    #points that had missing values from the original sets, returning the dataframe\
    #df_test_fillfeatures
    '''

    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']

    #if category in {'nmbr', 'nbr2', 'bxcx', 'bnry', 'text', 'bins', 'bint'}:
    if MLinfilltype in {'numeric', 'singlct', 'integer', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_ordl', 'concurrent_nmbr'}:

      #if this is a single column set or concurrent_act
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][category]['MLinfilltype'] in {'concurrent_act', 'concurrent_ordl', 'concurrent_nmbr'}:

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures.iloc[(testNArows[testNArows.columns[0]] == True).to_numpy()]
        #delete column
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        leakage_set = []
        if column in ML_cmnd['leakage_dict']:
          leakage_set = ML_cmnd['leakage_dict'][column]
          #this drop entries not in dataframe, such as based on other drops like for NArows above
          #this same derived leakage_set is used again below
          leakage_set = list(set(leakage_set) & set(df_test_fillfeatures))
          df_test_fillfeatures = df_test_fillfeatures.drop(leakage_set, axis=1)


      #else if categorylist wasn't empty
      else:

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures.iloc[(testNArows[testNArows.columns[0]] == True).to_numpy()]
        #delete column
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        
        #now delete any columns associated with leakage_sets
        leakage_set = []
        if column in ML_cmnd['leakage_dict']:
          leakage_set = ML_cmnd['leakage_dict'][column]
          #this drop entries not in dataframe (as a precaution)
          #this same derived leakage_set is used again below
          leakage_set = list(set(leakage_set) & set(df_test_fillfeatures))
          df_test_fillfeatures = df_test_fillfeatures.drop(leakage_set, axis=1)


    #elif MLinfilltype not in supported entries:
    else:

      #create empty sets for now
      df_test_fillfeatures = pd.DataFrame({'foo' : []})
    
    return df_test_fillfeatures

  def __predictpostinfill(self, column, category, model, df_test_fillfeatures, \
                        postprocess_dict, ML_cmnd, autoMLer, printstatus, categorylist = []):
    '''
    #predictpostinfill(category, model, df_test_fillfeatures, \
    #categorylist = []), function that takes as input \
    #a category string, a model trained as part of automunge on the coresponding \
    #column from the train set, the output of createpostMLinfillsets(.), a seed \
    #for randomness, and a list of columns \
    #produced by a text class preprocessor when applicable and returns \
    #predicted infills for the test feature sets as df_testinfill based on \
    #derivations using scikit-learn, with the lenth of \
    #infill consistent with the number of True values from NArows
    #a reasonable extension of this funciton would be to allow ML inference with \
    #other ML architectures such a SVM or something SGD based for instance
    '''

    #MLinfilltype distinguishes between classifier/regressor, single/multi column, ordinal/onehot/binary, etc
    #see potential values documented in assembleprocessdict function
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = ML_cmnd['autoML_type']

    #concurrent MLinfilltypes target a single column at a time, so categorylist is reset
    if MLinfilltype in {'concurrent_nmbr', 'concurrent_act', 'concurrent_ordl'}:
      categorylist = [column]
    
    #ony run the following if we successfuly trained a model in automunge
    if model is not False:
      
      #if target category is numeric:
      if MLinfilltype in {'numeric', 'integer', 'concurrent_nmbr'}:
        
        ML_application = 'regression'
    
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)

        # #this might be useful for tlbn, leaving out or now since don't want to clutter mlinfilltypes
        # #as concurrent_nmbr is intended as a resource for more than just tlbn
        # if MLinfilltype == 'concurrent_nmbr':
        #   df_testinfill = \
        #   self.__autowhere(df_testinfill, column, df_testinfill[column] < 0, -1, specified='replacement')

        if MLinfilltype == 'integer':
          df_testinfill = df_testinfill.round()
        
      #if target category is single column categoric (eg ordinal or boolean integer)
      if MLinfilltype in {'singlct', 'binary', 'concurrent_act', 'concurrent_ordl'}:
        
        if MLinfilltype in {'singlct', 'concurrent_ordl'}:
          ML_application = 'ordinalclassification'
        else:
          ML_application = 'booleanclassification'
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
        
      #if target category is multi-column categoric (one hot encoding) / (binary encoded sets handled sepreately)
      if MLinfilltype in {'multirt'}:
        
        ML_application = 'onehotclassification'
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
          
        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
      
      #if target is binary encoded
      if MLinfilltype in {'1010'}:
        
        ML_application = 'onehotclassification'

        _1010_categorylist_proxy_for_postmunge_MLinfill = \
        postprocess_dict['column_dict'][categorylist[0]]['_1010_categorylist_proxy_for_postmunge_MLinfill']
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, _1010_categorylist_proxy_for_postmunge_MLinfill)
          
          df_testinfill = \
          self.__convert_onehot_to_1010(df_testinfill)
          
        else:
          #this needs to have same number of columns
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
        
        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)         
        
      #if target is exlcuded from ML infill
      if MLinfilltype in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

        #create empty sets
        df_testinfill = pd.DataFrame({column : [0]}) 

    #else if we didn't have a trained model let's create some plug values
    else:

      df_testinfill = np.zeros(shape=(1,len(categorylist)))
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist) 
    
    return df_testinfill

  def __postMLinfillfunction(self, df_test, column, postprocess_dict, \
                            masterNArows_test, printstatus, logger):
    '''
    #new function ML infill, generalizes the MLinfill application
    #def __MLinfill (df_train, df_test, column, postprocess_dict, \
    #masterNArows_train, masterNArows_test, randomseed)
    #function that applies series of functions of createMLinfillsets, 
    #predictinfill, and insertinfill to a categorical encoded set.
    #please note that general convention is even single column sets are cast as dataframes instead of series
    #we prefer this convention for support of common operations independant of single or multi column sets
    '''

    if postprocess_dict['column_dict'][column]['infillcomplete'] is False:

      columnslist = postprocess_dict['column_dict'][column]['columnslist']
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      category = postprocess_dict['column_dict'][column]['category']
      model = postprocess_dict['column_dict'][column]['infillmodel']
      ML_cmnd = postprocess_dict['ML_cmnd']
      
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_test[column][:0]).copy()

      elif len(categorylist) > 1:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_test[categorylist][:0]).copy()
      
      #createMLinfillsets
      df_test_fillfeatures = \
      self.__createpostMLinfillsets(df_test, column, \
                         pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                         category, ML_cmnd, postprocess_dict, \
                         columnslist = columnslist, \
                         categorylist = categorylist)
      
      #run validations of all valid numeric
      postprocess_dict = \
      self.__check_ML_infill_2(False, False, 
                             False, df_test_fillfeatures, printstatus,
                             column, postprocess_dict, logger, ampm = 'pm')

      #predict infill values using defined function predictinfill(.)
      df_testinfill = \
      self.__predictpostinfill(column, category, model, df_test_fillfeatures, \
                            postprocess_dict, postprocess_dict['ML_cmnd'], postprocess_dict['autoMLer'], \
                            printstatus, categorylist = categorylist)

      #if model is not False:
      if postprocess_dict['column_dict'][column]['infillmodel'] is not False \
      and masterNArows_test[origcolumn + '_NArows'].astype(bool).any():

        #apply _stochastic_impute to test data based on ML_cmnd['stochastic_impute_categoric'] or ML_cmnd['stochastic_impute_numeric']
        df_testinfill, postprocess_dict = \
        self.__stochastic_impute(ML_cmnd, df_testinfill, column, postprocess_dict, df_train=False)

        df_test = self.__insertinfill(df_test, column, df_testinfill, category, \
                               pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                               postprocess_dict, columnslist = columnslist, \
                               categorylist = categorylist)
          
      #now change the infillcomplete marker in the text_dict for each \
      #associated text column unless in concurrent_activations MLinfilltype
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        postprocess_dict['column_dict'][column]['infillcomplete'] = True
        
      else:
        for columnname in categorylist:
          postprocess_dict['column_dict'][columnname]['infillcomplete'] = True
        
      #reset data type to ensure returned data is consistent with what was passed
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
        df_test[column] = \
        df_test[column].astype({column:df_temp_dtype[column].dtypes})

      elif len(categorylist) > 1:
        for dtype_column in categorylist:
          df_test[dtype_column] = \
          df_test[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})

    return df_test, postprocess_dict

  #__FunctionBlock: postmunge PCA

  def __postcreatePCAsets(self, df_test, postprocess_dict):
    '''
    Function that takes as input the dataframes df_train and df_test 
    Removes those columns associated with the PCAexcl (which are the original 
    columns passed to automunge which are to be exlcuded from PCA), and returns 
    those sets as PCAset_trian, PCAset_test, and the list of columns extracted as
    PCAexcl_posttransform.
    '''

    PCAexcl = postprocess_dict['PCAexcl']

    PCAexcl_posttransform = self.__column_convert_support(PCAexcl, postprocess_dict, convert_to='returned')

    #we'll also exclude any features designated in ML_cmnd['full_exclude'], since they may have non-numeric data
    full_exclude = []
    if 'full_exclude' in postprocess_dict['ML_cmnd']:
      full_exclude = postprocess_dict['ML_cmnd']['full_exclude']
      if len(full_exclude) > 0:
        #convert to returned column convention
        full_exclude = self.__column_convert_support(full_exclude, postprocess_dict, convert_to='returned')
        
    #assemble the set of columns to be dropped
    PCAexcl_posttransform = list((set(PCAexcl_posttransform) | set(full_exclude)) & set(df_test))

    #assemble the sets by dropping the columns excluded
    PCAset_test = df_test.drop(PCAexcl_posttransform, axis=1)

    return PCAset_test, PCAexcl_posttransform

  def __postPCAfunction(self, PCAset_test, postprocess_dict):
    '''
    Function that takes as input the train and test sets intended for PCA
    dimensionality reduction. Returns a trained PCA model saved in postprocess_dict
    and trasnformed sets.
    '''

    #a future extension could only import one of these model types
    #issue is the PCA_type = 'default' scenario could be multiple types
    #importing all three is equivalent to what was already in place when these were global imports

    # if 'ML_cmnd' in postprocess_dict \
    # and 'PCA_type' in postprocess_dict['ML_cmnd']['PCA_type'] \
    # and postprocess_dict['ML_cmnd']['PCA_type'] in {}:
    #   PCA_type = postprocess_dict['ML_cmnd']['PCA_type']

    from sklearn.decomposition import PCA
    from sklearn.decomposition import SparsePCA
    from sklearn.decomposition import KernelPCA

    #apply the transform
    PCAset_test = postprocess_dict['PCAmodel'].transform(PCAset_test)

    #get new number of columns
    newcolumncount = np.size(PCAset_test,1)

    #generate a list of column names for the conversion to pandas
    columnnames = postprocess_dict['returned_PCA_columns']

    #convert output to pandas
    PCAset_test = pd.DataFrame(PCAset_test, columns = columnnames)

    return PCAset_test, postprocess_dict

  #__FunctionBlock: postmunge feature selection

  def __postfeatureselect(self, df_test, testID_column, \
                        postprocess_dict, printstatus, entropy_seeds, random_generator, sampling_dict, logger):
    '''
    featureselect is a function called within automunge() that applies methods
    to evaluate predictive power of derived features towards a downstream model
    such as to trim the branches of the transform tree.
    
    The function returns a list of column names that "made the cut" so that
    automunge() can then remove extraneous branches.
    '''

    #now we'll use automunge() to prepare the subset for feature evaluation
    #note the passed arguments, these are all intentional (no MLinfill applied,
    #primary goal here is to produce a processed dataframe for df_subset
    #with corresponding labels)
    
    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Begin Feature Importance evaluation")
    self.__autoprint('info', "")

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[]
    for column in df_test.columns:
      testlabels.append(str(column))
    df_test.columns = testlabels

    labels_column_listofcolumns = postprocess_dict['labels_column_listofcolumns']
    
    labels_present = False
    if set(labels_column_listofcolumns).issubset(set(df_test)):
      labels_present = True
      
    else:
      
      labelscolumn_for_postfeatureselect_valresult = True
      logger_dict['validations'].update({'labelscolumn_for_postfeatureselect_valresult' : labelscolumn_for_postfeatureselect_valresult})
      logger = self.__logger_append(logger)
      
      FSmodel = False

      baseaccuracy = False

      FScolumn_dict = {}
      
      FS_origcolumns = []
      
      #printout display progress
      # if printstatus != 'silent':
      self.__autoprint('warning', "_______________")
      self.__autoprint('warning', "No labels_column passed, Feature Importance halted")
      self.__autoprint('warning', "")
    
    if labels_present is True:
    
      #copy postprocess_dict to customize for feature importance evaluation
      FSpostprocess_dict = self.__autocopy(postprocess_dict)
      testID_column = testID_column
      pandasoutput = 'dataframe'
      printstatus = printstatus
      TrainLabelFreqLevel = False
      featureeval = False
      FSpostprocess_dict['shuffletrain'] = True
      FSpostprocess_dict['TrainLabelFreqLevel'] = False
      FSpostprocess_dict['MLinfill'] = False
      FSpostprocess_dict['featureselection'] = False
      FSpostprocess_dict['privacy_encode'] = False
      FSpostprocess_dict['PCAn_components'] = None
      FSpostprocess_dict['PCA_applied'] = False
      FSpostprocess_dict['Binary'] = False
      FSpostprocess_dict['excl_suffix'] = True
      FSpostprocess_dict['ML_cmnd']['PCA_type'] = 'off'
      FSpostprocess_dict['assigninfill'] = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[], \
                                             'adjinfill':[], 'meaninfill':[], 'medianinfill':[]}
      randomseed = FSpostprocess_dict['postmunge_randomseed']
      process_dict = FSpostprocess_dict['process_dict']
      ML_cmnd = FSpostprocess_dict['ML_cmnd']
      FSpostprocess_dict['postprocess_assigninfill_dict']['naninfill'] = []

      #turn off assigninfill for postmunge applied for feature select
      FSpostprocess_dict['postprocess_assigninfill_dict'] = \
      {'stdrdinfill' : FSpostprocess_dict['pre_dimred_finalcolumns_train'],
      'MLinfill' : [],
      'zeroinfill' : [],
      'oneinfill' : [],
      'adjinfill' : [],
      'meaninfill' : [],
      'medianinfill' : [],
      'negzeroinfill' : [],
      'modeinfill' : [],
      'lcinfill' : [],
      'naninfill' : [],
      'unspecified' : []}

      #since we dont' save the training functions in postprocess_dict for customML case, feature selection defaults to random forest
      if ML_cmnd['autoML_type'] in {'customML'}:
        autoMLer = self.__assemble_autoMLer()
        FSpostprocess_dict['autoMLer'] = autoMLer
        ML_cmnd['autoML_type'] = 'randomforest'
        FSpostprocess_dict['ML_cmnd']['autoML_type'] = 'randomforest'

      #totalvalidation = valpercent

      #if totalvalidation == 0:
      totalvalidation = 0.2

      #following postmunge call will return validation results in original bucket
      #which we'll store in postfeatureselection_validations
      orig_validations_dict = self.__autocopy(logger_dict['validations'])
      logger_dict['validations'] = {'suffixoverlap_results':{}}

      #prepare sets for FS with postmunge
      am_train, _1, am_labels, FSpostreports_dict = \
      self.postmunge(FSpostprocess_dict, df_test, testID_column = testID_column, \
                     pandasoutput = pandasoutput, printstatus = printstatus, \
                     TrainLabelFreqLevel = TrainLabelFreqLevel, featureeval = featureeval, \
                     shuffletrain = True, \
                     entropy_seeds = entropy_seeds, random_generator = random_generator, sampling_dict=sampling_dict, \
                     logger = logger)

      #logger_dict validation results stored in logger_dict['postfeatureselection_validations']
      logger_dict['postfeatureselection_validations'] = self.__autocopy(logger_dict['validations'])
      logger_dict['validations'] = {'suffixoverlap_results':{}}
      logger_dict['validations'] = self.__autocopy(orig_validations_dict)

      #prepare validaiton sets for FS
      am_train, am_validation1 = \
      self.__df_split(am_train, totalvalidation, False, randomseed)

      am_labels, am_validationlabels1 = \
      self.__df_split(am_labels, totalvalidation, False, randomseed)

      #__

      #customML autoMLtype not supported for postmunge feature importance 
      #since don't have access to model training function

      #__
      
      #we'll remove columns from ML_cmnd['full_exclude'] or with MLinfilltype == 'totalexclude'
      #using a simpler method than applied in ML infill for this purpose since we're only training one model

      full_exclude_specified = []
      if 'full_exclude' in ML_cmnd:
        full_exclude_specified = ML_cmnd['full_exclude']
        
      totalexclude_MLinfilltype = []
      for column_dict_entry in FSpostprocess_dict['column_dict']:
        column_dict_entry_category = FSpostprocess_dict['column_dict'][column_dict_entry]['category']
        column_dict_entry_category_MLinfilltype = FSpostprocess_dict['process_dict'][column_dict_entry_category]['MLinfilltype']
        if column_dict_entry_category_MLinfilltype == 'totalexclude':
          totalexclude_MLinfilltype.append(column_dict_entry)
          
      nonnumeric_columns = full_exclude_specified + totalexclude_MLinfilltype
      #convert to returtned header format
      nonnumeric_columns = self.__column_convert_support(nonnumeric_columns, FSpostprocess_dict, convert_to='returned')

      nonnumeric_columns = list(set(nonnumeric_columns) & set(am_train))
      
      #now drop any potentially nonnumeric columns
      am_train = am_train.drop(nonnumeric_columns, axis=1)
      am_validation1 = am_validation1.drop(nonnumeric_columns, axis=1)
      
      #__

      #this is the returned process_dict
      #(remember "processdict" is what we pass to automunge() call, "process_dict" is what is 
      #assembled inside automunge, there is a difference)
      FSprocess_dict = FSpostprocess_dict['process_dict']

      if am_labels.empty is True:
        FSmodel = False

        baseaccuracy = False

        logger_dict['validations'].update({'FS_numeric_data_result': False})
        logger_dict['validations'].update({'FS_all_valid_entries_result': False})
        logger = self.__logger_append(logger)
        
        #printout display progress
        # if printstatus != 'silent':
        self.__autoprint('warning', "_______________")
        self.__autoprint('warning', "No labels returned from Postmunge, Feature Importance halted")
        self.__autoprint('warning', "")

        returned_label_set_for_postfeatureselect_valresult = True
        logger_dict['validations'].update({'returned_label_set_for_postfeatureselect_valresult' : returned_label_set_for_postfeatureselect_valresult})
        logger = self.__logger_append(logger)
    
      #if am_labels is not an empty set
      if am_labels.empty is False:

        #find origcateogry of am_labels from FSpostprocess_dict
        labelcolumnkey = list(am_labels)[0]

        if labelcolumnkey in FSpostprocess_dict['column_dict']:

          origcolumn = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
          origcategory = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcategory']

          #find labelctgy from process_dict based on this origcategory
          labelctgy = FSprocess_dict[origcategory]['labelctgy']

          am_categorylist = []

          for am_label_column in am_labels.columns:
            
            if am_label_column in FSpostprocess_dict['column_dict'] \
            and FSpostprocess_dict['column_dict'][am_label_column]['category'] == labelctgy:

              am_categorylist = FSpostprocess_dict['column_dict'][am_label_column]['categorylist']
              
              #we'll follow convention that if target label category MLinfilltype is concurrent
              #we'll arbitrarily take the first column and use that as target
              if FSpostprocess_dict['process_dict'][labelctgy]['MLinfilltype'] \
              in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:
                
                am_categorylist = [am_categorylist[0]]
                
              break
            
        #same as if labelcolumnkey not in FSpostprocess_dict['column_dict']:
        #only other scenario for labels is column originated from categoric consolidation
        elif len(FSpostprocess_dict['labels_Binary_dict']) > 0:
          
          #for categoric consolidated labels, we are going to edit the column_dict and process_dict
          #as a hack to enable __trainFSmodel support
          #noting that in so doing there is a remote edge case of process_dict edit interfering with user overwrite to processdict
          #so we have some additional accomodation there as well
          
          am_label_column = FSpostprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns'][0]
          
          am_categorylist = FSpostprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns']
          
          labelctgy = FSpostprocess_dict['labels_Binary_dict'][0]['Btype']
            
          #since process_dict only returns entries applied for transformations, there is chance a Binary transform isn't included
          #so as part of the hack we'll add the associated Binary transform for purposes of inspecting the MLinfilltype
          #FSpostprocess_dict['process_dict'] will only be inspected for MLinfilltype in __trainFSmodel and __shuffleaccuracy
          
          #in cases where labelctgy is already an entry in process_dict we'll add suffix until unique
          remote_overwrite_edge_case_marker = False
          if labelctgy in FSpostprocess_dict['process_dict']:
            remote_overwrite_edge_case_marker = True
          
          if labelctgy == 'ord3':
            
            if remote_overwrite_edge_case_marker is True:
              #this is in case user specified by chance the same string in processdict
              while labelctgy in FSpostprocess_dict['process_dict']:
                labelctgy += 'B'
            
            FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'singlct'}})
            
          if labelctgy == 'onht':
            
            if remote_overwrite_edge_case_marker is True:
              #this is in case user specified by chance the same string in processdict
              while labelctgy in FSpostprocess_dict['process_dict']:
                labelctgy += 'B'
            
            FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'multirt'}})
            
          if labelctgy == '1010':
            
            if remote_overwrite_edge_case_marker is True:
              #this is in case user specified by chance the same string in processdict
              while labelctgy in FSpostprocess_dict['process_dict']:
                labelctgy += 'B'
            
            FSpostprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : '1010'}})
          
          #this is a hack to support __trainFSmodel with categoric consolidations
          FSpostprocess_dict['column_dict'].update({am_label_column : {'categorylist' : am_categorylist,
                                                                      'columnslist' : list(am_labels), 
                                                                      'category' : labelctgy,
                                                                      'origcategory' : labelctgy}})
          
      #____________________

        if len(am_categorylist) == 0:
          # if printstatus != 'silent':
          #this is a remote edge case, printout added for troubleshooting support
          self.__autoprint('warning', "Label root category processdict entry contained a labelctgy entry not found in family tree")
          self.__autoprint('warning', "Feature Selection model training will not run without valid labelgctgy processdict entry")
          self.__autoprint('warning', '')

          labelctgy_not_found_in_familytree_pm_valresult = True
          logger_dict['validations'].update({'labelctgy_not_found_in_familytree_pm_valresult' : labelctgy_not_found_in_familytree_pm_valresult})
          logger = self.__logger_append(logger)

        elif len(am_categorylist) == 1:
          am_labels = pd.DataFrame(am_labels[am_categorylist[0]])
          am_validationlabels1 = pd.DataFrame(am_validationlabels1[am_categorylist[0]])

        else:
          am_labels = am_labels[am_categorylist]
          am_validationlabels1 = am_validationlabels1[am_categorylist]

        #if there's a bug occuring after this point it might mean the labelctgy wasn't
        #properly populated in the process_dict for the root category assigned to the labels
        #again the labelctgy entry to process_dict represents for labels returned in 
        #multiple configurations the trasnofrmation category whose returned set will be
        #used to train the feature selection model

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Training feature importance evaluation model")
        self.__autoprint('debug', "")

        #first validate that data is all valid numeric
        FS_numeric_data_result, FS_all_valid_entries_result = \
        self.__validate_allvalidnumeric(am_train, printstatus)
  
        logger_dict['validations'].update({'FS_numeric_data_result': FS_numeric_data_result})
        logger_dict['validations'].update({'FS_all_valid_entries_result': FS_all_valid_entries_result})
        logger = self.__logger_append(logger)

        #this is assocaited with adding support for XGBoost and customML
        #since they use this to log information for label conversion to fully represented range integer
        FSpostprocess_dict['customML_inference_support'] = {}

        #apply function trainFSmodel
        #FSmodel, baseaccuracy = \
        FSmodel, FSpostprocess_dict = \
        self.__trainFSmodel(am_train, am_labels, randomseed, \
                          FSprocess_dict, FSpostprocess_dict, labelctgy, ML_cmnd, \
                          printstatus, logger)

        #the inference functions inspect customML_inference_support in ML_cmnd
        FSpostprocess_dict['ML_cmnd']['customML_inference_support'] = FSpostprocess_dict['customML_inference_support']
        
        if FSmodel is False:
          
          #initialize dictionary FScolumn_dict = {}
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

          baseaccuracy = False

          logger_dict['validations'].update({'FS_numeric_data_result': False})
          logger_dict['validations'].update({'FS_all_valid_entries_result': False})
          logger = self.__logger_append(logger)
          
          #printout display progress
          # if printstatus != 'silent':
          self.__autoprint('warning', "_______________")
          self.__autoprint('warning', "No model returned from training, Feature Importance halted")
          self.__autoprint('warning', "")

          postfeatureselect_trained_model_valresult = True
          logger_dict['validations'].update({'postfeatureselect_trained_model_valresult' : postfeatureselect_trained_model_valresult})
          logger = self.__logger_append(logger)
          
        elif FSmodel is not False:

          #update v2.11 baseaccuracy should be based on validation set
          baseaccuracy = self.__shuffleaccuracy(am_validation1, am_validationlabels1, \
                                              FSmodel, randomseed, am_categorylist, \
                                              FSprocess_dict, labelctgy, FSpostprocess_dict)

          # if printstatus is True:
          self.__autoprint('debug', "Base Accuracy of feature importance model:")
          self.__autoprint('debug', baseaccuracy)
          self.__autoprint('debug', '')

          #get list of columns
          am_train_columns = list(am_train)

          #initialize dictionary FScolumn_dict = {}
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

          #assemble FScolumn_dict to support the feature evaluation
          for column in am_train_columns:

            #Binary and PCA consolidated columns don't get evaluated
            if column in FSpostprocess_dict['column_dict']:

              #pull categorylist, category, columnslist
              categorylist = FSpostprocess_dict['column_dict'][column]['categorylist']
              category = FSpostprocess_dict['column_dict'][column]['category']
              columnslist = FSpostprocess_dict['column_dict'][column]['columnslist']
              origcolumn = FSpostprocess_dict['column_dict'][column]['origcolumn']

              #create entry to FScolumn_dict
              FScolumn_dict.update({column : {'categorylist' : categorylist, \
                                              'category' : category, \
                                              'columnslist' : columnslist, \
                                              'origcolumn' : origcolumn, \
                                              'FScomplete' : False, \
                                              'shuffleaccuracy' : None, \
                                              'shuffleaccuracy2' : None, \
                                              'baseaccuracy' : baseaccuracy, \
                                              'metric' : None, \
                                              'metric2' : None}})

          #printout display progress
          # if printstatus is True:
          self.__autoprint('debug', "_______________")
          self.__autoprint('debug', "Evaluating feature importances")
          self.__autoprint('debug', "")

          #perform feature evaluation on each column
          for column in am_train_columns:

            if column not in nonnumeric_columns and column in FScolumn_dict:

  #             if FSpostprocess_dict['column_dict'][column]['category'] != 'NArw' \
  #             and FScolumn_dict[column]['FScomplete'] is False:
              if FScolumn_dict[column]['FScomplete'] is False:

                columnslist = FScolumn_dict[column]['columnslist']

                #create set with columns shuffle from columnslist
                #shuffleset = self.__createFSsets(am_train, column, categorylist, randomseed)
                #shuffleset = self.__createFSsets(am_train, column, columnslist, randomseed)
                shuffleset = self.__createFSsets(am_validation1, column, columnslist, randomseed)

                #determine resulting accuracy after shuffle
                columnaccuracy = self.__shuffleaccuracy(shuffleset, am_validationlabels1, \
                                                      FSmodel, randomseed, am_categorylist, \
                                                      FSprocess_dict, labelctgy, FSpostprocess_dict)

                #I think this will clear some memory
                del shuffleset

                #category accuracy penalty metric
                metric = baseaccuracy - columnaccuracy
                #metric2 = baseaccuracy - columnaccuracy2
                
                #save accuracy to FScolumn_dict and set FScomplete to True
                #(for each column in the categorylist)
                #for categorycolumn in FSpostprocess_dict['column_dict'][column]['categorylist']:
                for categorycolumn in FSpostprocess_dict['column_dict'][column]['columnslist']:

                  if categorycolumn not in nonnumeric_columns:

                    FScolumn_dict[categorycolumn]['FScomplete'] = True
                    FScolumn_dict[categorycolumn]['shuffleaccuracy'] = columnaccuracy
                    FScolumn_dict[categorycolumn]['metric'] = metric
                    #FScolumn_dict[categorycolumn]['shuffleaccuracy2'] = columnaccuracy2
                    #FScolumn_dict[categorycolumn]['metric2'] = metric2

              columnslist = FScolumn_dict[column]['columnslist']

              #create second set with all but one columns shuffled from columnslist
              #this will allow us to compare the relative importance between columns
              #derived from the same parent
              #shuffleset2 = self.__createFSsets2(am_train, column, columnslist, randomseed)
              shuffleset2 = self.__createFSsets2(am_validation1, column, columnslist, randomseed)

              #determine resulting accuracy after shuffle
      #           columnaccuracy2 = self.__shuffleaccuracy(shuffleset2, am_labels, FSmodel, \
      #                                                 randomseed, \
      #                                                 process_dict)
              columnaccuracy2 = self.__shuffleaccuracy(shuffleset2, am_validationlabels1, \
                                                    FSmodel, randomseed, am_categorylist, \
                                                    FSprocess_dict, labelctgy, FSpostprocess_dict)

              metric2 = baseaccuracy - columnaccuracy2

              FScolumn_dict[column]['shuffleaccuracy2'] = columnaccuracy2
              FScolumn_dict[column]['metric2'] = metric2

    #     madethecut = self.__assemblemadethecut(FScolumn_dict, featurepct, featuremetric, \
    #                                          featuremethod, am_train_columns)


        #if the only column left in madethecut from origin column is a NArw, delete from the set
        #(this is going to lean on the column ID string naming conventions)
        #couldn't get this to work, this functionality a future extension
    #     trimfrommtc = []
    #     for traincolumn in list(df_train):
    #       if (traincolumn + '_') not in [checkmtc[:(len(traincolumn)+1)] for checkmtc in madethecut]:
    #         for mtc in madethecut:
    #           #if mtc originated from traincolumn
    #           if mtc[:(len(traincolumn)+1)] == traincolumn + '_':
    #             #count the number of same instance in madethecut set
    #             madethecut_trim = [mdc_trim[:(len(traincolumn)+1)] for mdc_trim in madethecut]
    #             if madethecut_trim.count(mtc[:(len(traincolumn)+1)]) == 1 \
    #             and mtc[-5:] == '_NArw':
    #               trimfrommtc = trimfrommtc + [mtc]
    #     madethecut = list(set(madethecut).difference(set(trimfrommtc)))


        #apply function madethecut(FScolumn_dict, featurepct)
        #return madethecut
        #where featurepct is the percent of features that we intend to keep
        #(might want to make this a passed argument from automunge)

        #I think this will clear some memory

          del am_train, _1, am_labels, FSpostreports_dict, am_validation1, am_validationlabels1

          # if printstatus is True:
          self.__autoprint('debug', "_______________")
          self.__autoprint('debug', "Feature Importance results:")
          self.__autoprint('debug', "")

          #to inspect values returned in featureimportance object one could run
          # if printstatus is True:
          for keys,values in FScolumn_dict.items():
            self.__autoprint('debug', keys)
            self.__autoprint('debug', 'metric = ', values['metric'])
            self.__autoprint('debug', 'metric2 = ', values['metric2'])
            self.__autoprint('debug', '')
              
    FS_sorted = {'baseaccuracy':baseaccuracy, \
                 'metric_key':{}, \
                 'column_key':{}, \
                 'metric2_key':{}, \
                 'metric2_column_key':{}}
    
    #first we'll handle first metric based on source column
    for FS_origcolumn in FS_origcolumns:
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric'] in FS_sorted['metric_key']:
            if isinstance(FS_sorted['metric_key'][FScolumn_dict[key]['metric']], list):
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
            else:
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']] = \
              [FS_sorted['metric_key'][FScolumn_dict[key]['metric']]]
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
          else:
            FS_sorted['metric_key'].update({FScolumn_dict[key]['metric'] : [FS_origcolumn]})
          break

    FS_sorted['metric_key'] = dict(sorted(FS_sorted['metric_key'].items(), reverse=True))
    
    for key in FS_sorted['metric_key']:
      for entry in FS_sorted['metric_key'][key]:
        entry_index = FS_sorted['metric_key'][key].index(entry)
        FS_sorted['column_key'].update({FS_sorted['metric_key'][key][entry_index] : key})
      
    #now for metric2 based on derived columns relative importance, note sorted in other order
    for FS_origcolumn in FS_origcolumns:
      FS_sorted['metric2_key'].update({FS_origcolumn : {}})
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric2'] in FS_sorted['metric2_key'][FS_origcolumn]:
            if isinstance(FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']], list):
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
            else:
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']] = \
              [FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']]]
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
          else:
            FS_sorted['metric2_key'][FS_origcolumn].update({FScolumn_dict[key]['metric2'] : [key]})
    
    for key in FS_sorted['metric2_key']:
      FS_sorted['metric2_key'][key] = dict(sorted(FS_sorted['metric2_key'][key].items(), reverse=False))
    
    for key1 in FS_sorted['metric2_key']:
      FS_sorted['metric2_column_key'].update({key1 : {}})
      for key2 in FS_sorted['metric2_key'][key1]:
        for entry in FS_sorted['metric2_key'][key1][key2]:
          entry_index = FS_sorted['metric2_key'][key1][key2].index(entry)
          FS_sorted['metric2_column_key'][key1].update({FS_sorted['metric2_key'][key1][key2][entry_index] : key2})
        
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', '')
    self.__autoprint('info', "______________________")
    self.__autoprint('info', "sorted metric results:")
    self.__autoprint('info', '')
    for keys,values in FS_sorted['metric_key'].items():
      for entry in values:
        self.__autoprint('info', entry)
        self.__autoprint('info', keys)
        self.__autoprint('info', '')
    self.__autoprint('info', "______________________")
    self.__autoprint('info', "sorted metric2 results:")
    self.__autoprint('info', '')
    for key in FS_sorted['metric2_key']:
      self.__autoprint('info', "for source column: ", key)
      for keys,values in FS_sorted['metric2_key'][key].items():
        for entry in values:
          self.__autoprint('info', entry)
          self.__autoprint('info', keys)
          self.__autoprint('info', '')
      self.__autoprint('info', '')
        
    if FSmodel is False:
      
      FScolumn_dict = {}
    
    #printout display progress
    # if printstatus is True:
    self.__autoprint('debug', "")
    self.__autoprint('debug', "_______________")
    self.__autoprint('debug', "Feature Importance evaluation complete")
    self.__autoprint('debug', "")

    return FSmodel, FScolumn_dict, FS_sorted

  #__FunctionBlock: postmunge driftreport

  def __PCA_drift_stats(self, df, returned_PCA_columns):
    """
    returns a dictionary of drift stats for the list of columns in returned_PCA_columns
    as will be recorded in postprocess_dict['dimensionality_reduction_driftstats']['PCA_drift']
    which will later be transferred to postreports_dict['dimensionality_reduction_driftstats']['PCA_drift']['orig_stats']
    or postreports_dict['dimensionality_reduction_driftstats']['PCA_drift']['new_stats']


    """

    drift_dict = {}
    
    for returned_PCA_column in returned_PCA_columns:
      
      drift_dict.update({returned_PCA_column : 
                          {
                            'mean'    : df[returned_PCA_column].mean(),
                            'std'     : df[returned_PCA_column].std(),
                            'median'  : df[returned_PCA_column].median(),
                            # 'MAD'     : df[returned_PCA_column].mad(),
                            'MAD' : (df[returned_PCA_column] - df[returned_PCA_column].median(axis=0)).abs().median(), \
                            'maximum' : df[returned_PCA_column].max(),
                            'minimum' : df[returned_PCA_column].min(),
                          }
                        })
    
    return drift_dict
    
  def __Binary_drift_stats(self, df, Binary_dict):
    """
    returns a dictionary of drift stats for the columns returned from Binary dimensionality reduction
    note that since they may be multiple Binary consolidations specified, the first tier will be an integer
    
    the results will be recorded in postprocess_dict['dimensionality_reduction_driftstats']['Binary_drift']
    which will later be transferred to postreports_dict['dimensionality_reduction_driftstats']['Binary_drift']['orig_stats']
    or postreports_dict['dimensionality_reduction_driftstats']['Binary_drift']['new_stats']
    
    input columns are recorded in Binary_dict's categoric_column_tuple, omitting because with ordinal treatent recovery is not simple
    
    note that Binary_dict records normalization_dict entries from transform as well as Binary specific entries
    """

    drift_dict = {}
    
    for i in Binary_dict:

      if 'Btype' in Binary_dict[i]:
        
        drift_dict.update({i : {}})
        
        if Binary_dict[i]['Btype'] == '1010':
          
          returned_Binary_columns = Binary_dict[i]['returned_Binary_columns']
          unique_ratio = Binary_dict[i]['_1010_activations_dict']
          nunique = len(list(Binary_dict[i]['_1010_activations_dict']))
          conversion_dict = Binary_dict[i]['binary_encoding_dict']
          
          unique_ratio_clean = {}
          for key in unique_ratio:
            unique_ratio_clean.update({conversion_dict[key] : unique_ratio[key]})

        elif Binary_dict[i]['Btype'] == 'ord3':
          
          returned_Binary_columns = Binary_dict[i]['returned_Binary_columns']
          unique_ratio = Binary_dict[i]['ordl_activations_dict']
          nunique = len(list(Binary_dict[i]['ordl_activations_dict']))
          conversion_dict = Binary_dict[i]['ordinal_dict']
          
          unique_ratio_clean = {}
          for key in unique_ratio:
            unique_ratio_clean.update({conversion_dict[key] : unique_ratio[key]})
              
        elif Binary_dict[i]['Btype'] == 'onht':
          
          returned_Binary_columns = Binary_dict[i]['returned_Binary_columns']
          unique_ratio = Binary_dict[i]['onht_activations_dict']
          nunique = len(returned_Binary_columns)
          conversion_dict = Binary_dict[i]['labels_dict']
          
          unique_ratio_clean = unique_ratio
          
        drift_dict[i].update(
          {
            'returned_Binary_columns' : returned_Binary_columns,
            'unique_ratio' : unique_ratio_clean,
            'nunique' : nunique,
          }
        )
      
    return drift_dict

  def __prepare_driftreport(self, df_test, postprocess_dict, printstatus, logger):
    """
    #driftreport uses the processfamily functions as originally implemented
    #in automunge to recalculate normalization parameters based on the test
    #set passed to postmunge and print a comparison with those original 
    #normalization parameters saved in the postprocess_dict, such as may 
    #prove useful to track drift from original training data.
    #returns a store of the temporary postprocess_dict containing the newly 
    #calculated normalziation parameters and a report of the results
    #to be consistent with feature selection, drift report will return printouts in printstatus='summary' case
    """

    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Preparing Drift Report:")
    self.__autoprint('info', "")
      
    #initialize empty dictionary to store results
    drift_report = {}
    
    #temporary store for updated normalization parameters
    #we'll copy all the support stuff from original pp_d but delete the 'column_dict'
    #entries for our new derivations below
    drift_ppd = self.__autocopy(postprocess_dict)
    drift_ppd['column_dict'] = {}
    
    #for each column in df_test
    for drift_column in df_test:
      
      #returnedcolumns as used here is specific to a givin input drift_column
      if 'allderivedlist' in postprocess_dict['origcolumn'][drift_column]:
        returnedcolumns = postprocess_dict['origcolumn'][drift_column]['allderivedlist']
      else:
        #backward compatibility preceding 7.35
        returnedcolumns = postprocess_dict['origcolumn'][drift_column]['columnkeylist']
      
      # if printstatus in {True, 'summary'}:
      self.__autoprint('info', "______")
      self.__autoprint('info', "Preparing drift report for columns derived from: ", drift_column)
      self.__autoprint('info', "")
      self.__autoprint('info', "original returned columns:")
      self.__autoprint('info', returnedcolumns)
      self.__autoprint('info', "")
        
      if len(returnedcolumns) > 0:

        drift_category = \
        postprocess_dict['column_dict'][postprocess_dict['origcolumn'][drift_column]['columnkey']]['origcategory']
        
      else:
        
        drift_category = 'null'
        #this is remote edge case that comes up in comprehensive validations
        if 'null' not in drift_ppd['transform_dict']:
          drift_category = list(drift_ppd['transform_dict'])[0]
      
      #update driftreport with this column
      drift_report.update({drift_column : {'origreturnedcolumns_list':returnedcolumns, \
                                           'newreturnedcolumns_list':[], \
                                           'drift_category' : drift_category, \
                                           'orignotinnew' : {}, \
                                           'newnotinorig' : {}, \
                                           'newreturnedcolumn':{}}})
      
      drift_process_dict = \
      postprocess_dict['process_dict']
      
      drift_transform_dict = \
      postprocess_dict['transform_dict']
      
      drift_assign_param = \
      postprocess_dict['assign_param']
      
      #we're only going to copy one source column at a time, as should be 
      #more memory efficient than copying the entire set
      df_test2_temp = pd.DataFrame(df_test[drift_column].copy())

      #this is for edge case associated with missing data
      convert_to_nan_list = [np.inf, -np.inf, None, float("NaN")]
      df_test2_temp = self.__convert_to_nan(df_test2_temp, drift_column, drift_category, postprocess_dict, convert_to_nan_list)
      
      #then a second copy set, here of just a few rows, to follow convention of 
      #automunge processfamily calls
#       df_test3_temp = df_test2_temp[0:10].copy()
      df_test3_temp = df_test2_temp[0:1].copy()
      
      #this will be compared to tempset2 to derive list of derivations associated with drift_column as newreturnedcolumns
      tempset1 = set(drift_ppd['column_dict'])
    
      #now process family
      df_test2_temp, df_test3_temp, drift_ppd = \
      self.__processfamily(df_test2_temp, df_test3_temp, drift_column, \
                         drift_category, drift_transform_dict, \
                         drift_ppd, drift_assign_param, logger)

      #here's a second tempset to support newreturnedcolumns derivation
      tempset2 = set(drift_ppd['column_dict'])
      
      #this is a list of all derived column headers originating from drift_column
      newreturnedcolumns = \
      tempset2 - tempset1
      
      #this sorts newreturnedcolumns to match order of returned columns from train data
      #with any extra entries in alphabetic order at end of list
      newreturnedcolumns = \
      self.__list_sorting(returnedcolumns, newreturnedcolumns)
      
      if len(newreturnedcolumns) == 0:
        
        # if printstatus in {True, 'summary'}:
        self.__autoprint('info', "no new returned columns:")
        self.__autoprint('info', "")
        
      else:

        # if printstatus in {True, 'summary'}:
        self.__autoprint('info', "new returned columns:")
        self.__autoprint('info', newreturnedcolumns)
        self.__autoprint('info', "")
          
        #add to driftreport
        drift_report[drift_column]['newreturnedcolumns_list'] = newreturnedcolumns
      
      for origreturnedcolumn in returnedcolumns:
        if origreturnedcolumn not in newreturnedcolumns:
          if origreturnedcolumn == postprocess_dict['column_dict'][origreturnedcolumn]['categorylist'][0]:
            # if printstatus in {True, 'summary'}:
            self.__autoprint('info', "___")
            self.__autoprint('info', "original derived column not in new returned column: ", origreturnedcolumn)
            self.__autoprint('info', "")
            self.__autoprint('info', "original automunge normalization parameters:")
            self.__autoprint('info', postprocess_dict['column_dict'][origreturnedcolumn]['normalization_dict'][origreturnedcolumn])
            self.__autoprint('info', "")
            
            drift_report[drift_column]['orignotinnew'].update({origreturnedcolumn:{'orignormparam':\
            postprocess_dict['column_dict'][origreturnedcolumn]['normalization_dict'][origreturnedcolumn]}})

      for newreturnedcolumn in newreturnedcolumns:

        if newreturnedcolumn == drift_ppd['column_dict'][newreturnedcolumn]['categorylist'][0]:
        
          drift_report[drift_column]['newreturnedcolumn'].update(\
          {newreturnedcolumn:{'orignormparam':{}, 'newnormparam':{}}})
          
          # if printstatus in {True, 'summary'}:
          self.__autoprint('info', "___")
          self.__autoprint('info', "derived columns: ", postprocess_dict['column_dict'][newreturnedcolumn]['categorylist'])
          self.__autoprint('info', "")

          if newreturnedcolumn in returnedcolumns \
          and postprocess_dict['column_dict'][newreturnedcolumn]['categorylist'][0] == \
          drift_ppd['column_dict'][newreturnedcolumn]['categorylist'][0]:
            # if printstatus in {True, 'summary'}:
            self.__autoprint('info', "original automunge normalization parameters:")
            
            self.__autoprint('info', postprocess_dict['column_dict'][newreturnedcolumn]['normalization_dict'][newreturnedcolumn])
            self.__autoprint('info', "")
              
            #add to driftreport
            drift_report[drift_column]['newreturnedcolumn'][newreturnedcolumn]['orignormparam'] \
            = postprocess_dict['column_dict'][newreturnedcolumn]['normalization_dict'][newreturnedcolumn]
            
          else:
            
            if newreturnedcolumn in returnedcolumns:
              # if printstatus in {True, 'summary'}:
              self.__autoprint('info', "new derived column has first categorylist entry not matching first categorylist entry from original derivation")
              self.__autoprint('info', "For derived column: ", newreturnedcolumn)
              self.__autoprint('info', "This is treated comparably to new derived column not in original returned columns")
              self.__autoprint('info', "")
            else:
              # if printstatus in {True, 'summary'}:
              self.__autoprint('info', "new derived column not in original returned columns: ", newreturnedcolumn)
              self.__autoprint('info', "")
              
            drift_report[drift_column]['newnotinorig'].update({newreturnedcolumn:{'newnormparam':\
            drift_ppd['column_dict'][newreturnedcolumn]['normalization_dict'][newreturnedcolumn]}})
            
          # if printstatus in {True, 'summary'}:
          self.__autoprint('info', "new postmunge normalization parameters:")
          self.__autoprint('info', drift_ppd['column_dict'][newreturnedcolumn]['normalization_dict'][newreturnedcolumn])
          self.__autoprint('info', "")
            
          #add to driftreport
          drift_report[drift_column]['newreturnedcolumn'][newreturnedcolumn]['newnormparam'] \
          = drift_ppd['column_dict'][newreturnedcolumn]['normalization_dict'][newreturnedcolumn]
      
      #free up some memory
      del df_test2_temp, df_test3_temp, returnedcolumns
      
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "")
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Drift Report Complete")
    self.__autoprint('info', "")
      
    return drift_ppd, drift_report

  #__FunctionBlock: postmunge(.) definition

  def postmunge(self, postprocess_dict, df_test,
                testID_column = False, pandasoutput = 'dataframe', 
                printstatus = 'summary', inplace = False,
                dupl_rows = False, TrainLabelFreqLevel = False, 
                featureeval = False, traindata = False, noise_augment = 0,
                driftreport = False, inversion = False,
                returnedsets = True, shuffletrain = False,
                entropy_seeds = False, random_generator = False, sampling_dict = False,
                randomseed = False, encrypt_key = False, logger = {}):
    """
    #This function documented in READ ME, available online at:
    # https://github.com/Automunge/AutoMunge/blob/master/README.md

    #Note that postmunge(.) has one global variable: logger_dict
    """

    """
    Workflow blocks can be navigated with control F search:
    #__WorkflowBlock: postmunge decryption support
    #__WorkflowBlock: postmunge variable initializations and parameter validations
    #__WorkflowBlock: postmunge feature importance
    #__WorkflowBlock: postmunge misc initializations and conversions
    #__WorkflowBlock: postmunge inversion
    #__WorkflowBlock: postmunge noise_augment duplicate extraction
    #__WorkflowBlock: postmunge ppd_append additional features extraction
    #__WorkflowBlock: postmunge ID set populated
    #__WorkflowBlock: postmunge labels and other variable initializations
    #__WorkflowBlock: postmunge validate data set headers
    #__WorkflowBlock: postmunge drift report evaluated
    #__WorkflowBlock: postmunge column processing
    #__WorkflowBlock: postmunge label column processing
    #__WorkflowBlock: postmunge protected feature processing
    #__WorkflowBlock: postmunge apply infill
    #__WorkflowBlock: postmunge feature importance dimensionality reduction
    #__WorkflowBlock: postmunge PCA dimensionality reduction
    #__WorkflowBlock: postmunge Binary dimensionality reduction (categoric consolidations)
    #__WorkflowBlock: postmunge duplicate row consolidation
    #__WorkflowBlock: postmunge label rebalancing (e.g. oversampling)
    #__WorkflowBlock: postmunge row shuffling
    #__WorkflowBlock: postmunge data type management
    #__WorkflowBlock: postmunge excl suffix management
    #__WorkflowBlock: postmunge privacy encoding
    #__WorkflowBlock: postmunge ppd_append additional features processing and concatination
    #__WorkflowBlock: postmunge noise_augment prep duplicate sets
    #__WorkflowBlock: postmunge final preps and return
    """

    # #copy postprocess_dict into internal state so don't edit external object
    # #(going to leave this out for now in case has large memory overhead impact
    # #as in some scenarios postprocess_dict can be a large file)
    # postprocess_dict = self.__autocopy(postprocess_dict)
    # #The only edits made to postproces_dict in postmunge are:
    # #- to track infill status (via column_dict infillcomplete marker)
    # #- setting traindata setting based on traindata parameter
    # #- passing postmunge specific randomseed parameter through postmunge_randomseed
    # #- temporary entries based on entropy seeding parameters to entropy_seeds, random_generator, sampling_dict
    # #- test_plug_marker initialy reset to False and then reverted to original basis prior to return
    # #which are each reset after use

    #_________________________________________________________
    #__WorkflowBlock: postmunge decryption support
    #when the postprocess_dict returned from automunge(.) included encrypted entries
    #decruption may be performed here
    #note that if this is a inversion='labels' scenario and encrypt_key=False
    #then inversion will proceed without decription
    #otherwise decryption is based on the encrypt_key parameter

    #initialize logger_dict (used in printing)
    global logger_dict
    logger_dict = {
      'debug_report' : '',
      'info_report' : '',
      'warning_report' : '',
      'printstatus' : printstatus, 
      'validations' : {'suffixoverlap_results':{}},
    }

    #if postprocess_dict contained encrypted entries then we decode them here
    #if encryption was performed in automunge the encryption key was either user specified or returned in automunge(.) printouts
    decode_valresult = False
    if 'encryption' in postprocess_dict \
    and postprocess_dict['encryption'] is True \
    and encrypt_key is not False:

      postprocess_dict = self.__autocopy(postprocess_dict)

      postprocess_dict, decode_valresult = \
      self.__decrypt_postprocess_dict(postprocess_dict, encrypt_key, printstatus)

      #decode_valresult saved below

    elif 'encryption' in postprocess_dict \
    and postprocess_dict['encryption'] is True \
    and isinstance(inversion, str) \
    and inversion in {'labels', 'denselabels'} \
    and postprocess_dict['privacy_encode'] != 'private':

      #label inversion is available without encryption key unless privacy_encode = 'private'
      postprocess_dict = self.__autocopy(postprocess_dict['labelsencoding_dict'])

    #_________________________________________________________
    #__WorkflowBlock: postmunge variable initializations and parameter validations
    #we have a few temporary entries stored in postprocess_dict for postmunge
    #including postmunge_randomseed, and traindata
    #we have a few special conventions for privacy_encode which are addressed here
    #similarly special conventions for random seeding parameters entropy_seeds, random_generator
    #we also copy parameters passed as lists to internal state
    #perform some string conversions
    #and perform bulk parameter validations via __check_pm_miscparameters
    #and also validate df_test with __check_df_type

    #validate random_generator
    random_generator_pm_valresult, random_generator_accepts_seeds = \
    self.__check_random_generator(random_generator, printstatus)

    #validate sampling_dict and populate with defaults
    check_sampling_dict_pm_result, sampling_dict = \
    self.__check_sampling_dict(sampling_dict, printstatus, random_generator_accepts_seeds)

    #validate entropy_seeds and align to common type
    entropy_seeds, entropy_seeds_pm_result = \
    self.__check_entropy_seeds(entropy_seeds, printstatus)

    if 'test_plug_marker' in postprocess_dict:
      test_plug_marker_orig = postprocess_dict['test_plug_marker']
    else:
      #backward compatibility preceding 7.63
      test_plug_marker_orig = False
    postprocess_dict['test_plug_marker'] = False

    #perform any preparations on entropy_seeds and populate postprocess_dict with entropy_seed/sampling_dict/random_generator
    rowcount_train = df_test.shape[0]
    rowcount_test = df_test.shape[0]
    if traindata is True:
      traintest = 'train'
      rowcount_test = 0
    elif traindata is False:
      traintest = 'test'
      rowcount_train = 0
    else:
      #this is for the no noise scenarios, we'll set rowcount to zero resulting in no seed requirements
      traintest = 'test'
      rowcount_train = 0
      rowcount_test = 0
    postprocess_dict = \
    self.__prepare_seeds(postprocess_dict, sampling_dict, random_generator, entropy_seeds, rowcount_train, rowcount_test, traintest, postprocess_dict['test_plug_marker'])

    #initialize randomseed for default configuration of random random seed
    #when a sampling performed randomrandomseed returned as True
    #this is used in row shuffling and other random seeds that don't need to match automunge random seed
    #note that this may expend two entropy_seeds
    randomseed, randomrandomseed, entropy_seeds, _4 = \
    self.__populate_randomseed(randomseed, postprocess_dict['entropy_seeds'], 
                              random_generator, sampling_dict)

    #store a few temporary entries in postprocess_dict that will be struck or reset prior to return, including postmunge_randomseed, traindata
    postprocess_dict.update({'postmunge_randomseed' : randomseed})

    #traindata only matters when transforms apply different methods for train vs test
    #such as for noise injection to train data for differential privacy or for label smoothing transforms
    if traindata is not False:
      postprocess_dict['traindata'] = traindata
    else:
      postprocess_dict['traindata'] = False

    #a few special conventions for privacy_encode
    if postprocess_dict['privacy_encode'] is not False and printstatus is True:
      print("privacy_encode was performed in automunge(.), printstatus reset from True to False to align")
      printstatus = False
    if postprocess_dict['privacy_encode'] is not False and featureeval is not False:
      print("privacy_encode was performed in automunge(.), featureeval reset to False to align")
      featureeval = False
    if postprocess_dict['privacy_encode'] is not False and driftreport is not False:
      print("privacy_encode was performed in automunge(.), driftreport reset to False to align")
      driftreport = False

    #copy any input lists to internal state
    #(these won't be large so not taking account of inplace parameter)
    if isinstance(testID_column, list):
      testID_column = self.__autocopy(testID_column)
    if isinstance(inversion, list):
      inversion = self.__autocopy(inversion)
    if isinstance(entropy_seeds, list):
      entropy_seeds = self.__autocopy(entropy_seeds)

    #quick conversion of any passed column idenitfiers to str
    testID_column = self.__parameter_str_convert(testID_column)
    
    #check the range of parameters 
    #(generally speaking other than passed dictionaries, dataframes, or column identifiers)
    self.__check_pm_miscparameters(pandasoutput, printstatus, TrainLabelFreqLevel, \
                                  dupl_rows, featureeval, driftreport, inplace, \
                                  returnedsets, shuffletrain, inversion, traindata, \
                                  testID_column, randomseed, encrypt_key, noise_augment, logger)

    check_df_test_type_result, _1 = \
    self.__check_df_type(df_test, False, printstatus)

    logger_dict['validations'].update({'check_df_test_type_result' : check_df_test_type_result})
    logger_dict['validations'].update({'decode_valresult' : decode_valresult})
    logger_dict['validations'].update({'check_sampling_dict_pm_result' : check_sampling_dict_pm_result})
    logger_dict['validations'].update({'entropy_seeds_pm_result' : entropy_seeds_pm_result})
    logger_dict['validations'].update({'random_generator_pm_valresult' : random_generator_pm_valresult})
    logger = self.__logger_append(logger)

    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Begin Postmunge")
    self.__autoprint('info', "")

    #_________________________________________________________
    #__WorkflowBlock: postmunge feature importance
    #postmunge feature importance is based on the featureeval parameter
    #and is performed in __postfeatureselect
    #or otherwise some variables for returned reports initalization as empty
    
    #feature selection analysis performed here if elected
    if featureeval is True:

      if inversion is not False:
        # if printstatus != 'silent':
        self.__autoprint('warning', "featureselection not available when performing inversion")
        self.__autoprint('warning', '')

        postfeatureselect_with_inversion_valresult = True
        logger_dict['validations'].update({'postfeatureselect_with_inversion_valresult' : postfeatureselect_with_inversion_valresult})
        
        madethecut = postprocess_dict['madethecut']
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}
        logger_dict['validations'].update({'FS_numeric_data_result': False})
        logger_dict['validations'].update({'FS_all_valid_entries_result': False})
        logger = self.__logger_append(logger)
      
      elif postprocess_dict['labels_column'] is False:
        # if printstatus != 'silent':
        self.__autoprint('warning', "featureselection not available without labels_column in training set")
        self.__autoprint('warning', '')
        
        labels_column_for_postfeatureselect_valresult = True
        logger_dict['validations'].update({'labels_column_for_postfeatureselect_valresult' : labels_column_for_postfeatureselect_valresult})

        madethecut = postprocess_dict['madethecut']
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}
        logger_dict['validations'].update({'FS_numeric_data_result': False})
        logger_dict['validations'].update({'FS_all_valid_entries_result': False})
        logger = self.__logger_append(logger)

      else:

        FSmodel, FScolumn_dict, FS_sorted = \
        self.__postfeatureselect(df_test, testID_column, \
                               postprocess_dict, printstatus, entropy_seeds, random_generator, sampling_dict, logger)

        madethecut = postprocess_dict['madethecut']

    else:
      
      madethecut = postprocess_dict['madethecut']
      FSmodel = None
      FScolumn_dict = {}
      FS_sorted = {}
      logger_dict['validations'].update({'FS_numeric_data_result': False})
      logger_dict['validations'].update({'FS_all_valid_entries_result': False})
      logger = self.__logger_append(logger)

    check_FSmodel_result = self.__check_FSmodel(featureeval, FSmodel, printstatus)
    logger_dict['validations'].update({'FSmodel_valresult' : check_FSmodel_result})
    logger = self.__logger_append(logger)

    #_________________________________________________________
    #__WorkflowBlock: postmunge misc initializations and conversions
    #the initialized postreports_dict is the diciotnary returned from postmunge
    #(sort of the postmunge analog to automunge's postprocess_dict)
    #dataframe copying into internal state is based on inplace parameter which may benefit memory overhead when activated
    #numpy support is acomodated here, e.g. nujpy converted to dataframe and column headers recovered assuming consistent order
    #some dataframe formalizing including series accomodation and header string conversion

    #initialize postreports_dict
    postreports_dict = {'featureimportance':FScolumn_dict, \
                        'FS_sorted' : FS_sorted, \
                        'finalcolumns_test':[], \
                        'driftreport':{}, \
                        'dimensionality_reduction_driftstats':{}, \
                        'pm_miscparameters_results':{}}

    #copy input dataframes to internal state so as not to edit exterior objects
    #this step can be omitted to reduce memory overhead with inplace parameter
    if inplace is not True:
      df_test = df_test.copy()
      # postprocess_dict = self.__autocopy(postprocess_dict)

    #functionality to support passed numpy arrays
    #if passed object was a numpy array, convert to pandas dataframe
    checknp = np.array([])
    if isinstance(checknp, type(df_test)):
      df_test = pd.DataFrame(df_test)

      #this converts to original headers for cases where automunge originally received pandas
      #and user is passing numpy to postmunge
      if inversion is False and len(df_test.columns) == len(postprocess_dict['origcolumns_all']):
        df_test.columns = postprocess_dict['origcolumns_all']

      #this is one spot where having a labelscolumn postmunge(.) parameter would be nice, but not worth added complexity
      #this conversion from numpy has edge cases, best used in conjunction with exact match in order and composistion of columns between automunge and postmunge
      #in other words, any ID columns should be in consistent position and training data labels should be positioned as final column
      elif inversion is False and postprocess_dict['labels_column'] is not False:
        #this is scenario of matching length of features and ID columns without labels
        #(we'll assume if number of ID columns and number of label columns match that the included columns are ID columns just to pick a convention)
        #this makes sense since in most cases postmunge is applied without labels
        if len(df_test.columns) == len(postprocess_dict['origcolumns_all']) - len(postprocess_dict['labels_column_listofcolumns']):
          origcolumns_all_excluding_label = postprocess_dict['origcolumns_all'].copy()
          for labels_column_listofcolumns_entry in postprocess_dict['labels_column_listofcolumns']:
            origcolumns_all_excluding_label.remove(labels_column_listofcolumns_entry)
          df_test.columns = origcolumns_all_excluding_label
        #this is scenario of matching length of features without ID columns or labels
        elif len(df_test.columns) == len(postprocess_dict['origtraincolumns']):
          df_test.columns = postprocess_dict['origtraincolumns']

      #for postmunge processing of label columns without features, column headers required 
      #(numpy conversion not provided to avoid edge case when len(labels) == len(features))

    #if series convert to dataframe
    checkseries = pd.Series([1])
    if isinstance(checkseries, type(df_test)):
      checkseries_test_result = True
      df_test = pd.DataFrame(df_test)

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[]
    for column in df_test.columns:
      testlabels.append(str(column))
    df_test.columns = testlabels

    #this checks for case where df_test was received as labels without features, in which case a dummy feature set populated as all 0's
    #this scenario handled seperately for inversion case
    if inversion is False and set(df_test) == set(postprocess_dict['labels_column_listofcolumns']):
      df_test_temp = pd.DataFrame(np.zeros((df_test.shape[0],len(postprocess_dict['origtraincolumns']))), columns=postprocess_dict['origtraincolumns'])
      #this results in the dataframe df_test as labels and features with plug values of 0
      df_test = pd.concat([df_test_temp, df_test], axis=1)

    #_________________________________________________________
    #__WorkflowBlock: postmunge inversion
    #inversion performed here
    #preceded by some inversion specific column header prep in __inversion_header_support
    #with inversion channeled through __inversion_header_support
    #and then a postmunge(.) return of the recovered sets
    #note that we have a special convention support for cases where ppd_append was applied in automunge
    #which has support for test data inversion limited to inversion = 'test' scenario
    #which we'll apply the additional feature inversions prior to the other inversion scenarios

    #_______
    #here is where inversion is performed if selected
    if inversion is not False:
      
      #reset temporary entries in postprocess_dict to avoid overwrite of external
      postprocess_dict['traindata'] = False
      postprocess_dict['entropy_seeds'] = False
      postprocess_dict['random_generator'] = False
      postprocess_dict['sampling_dict'] = False
      postprocess_dict['test_plug_marker'] = test_plug_marker_orig
      #strike temporary log from postprocess_dict
      del postprocess_dict['postmunge_randomseed']
      
      new_feature_ppd_case = False
      
      #_________
      #we'll have convention that only inversion == 'test' has test features support with ppd_append 
      #meaning can't specify sub lists
      #(label inversions still available through regular channel)
      if inversion == 'test' \
      and 'new_feature_ppd' in postprocess_dict \
      and postprocess_dict['new_feature_ppd'] != {}:
        
        new_feature_ppd_case = True
        
        aggregate_recovered_df = pd.DataFrame(index=df_test.index)
        aggregate_recovered_list = []
        aggregate_inversion_info_dict = {}
        
        for i in postprocess_dict['new_feature_ppd']:
          
          new_feature_ppd_extract_headers = \
          postprocess_dict['new_feature_ppd'][i]['finalcolumns_train']
          
          df_new_feature_df = df_test[new_feature_ppd_extract_headers].copy()

          #now drop the extracted columns from df_test
          for new_feature_ppd_extract_header in new_feature_ppd_extract_headers:
            del df_test[new_feature_ppd_extract_header]
          
          #now apply the inversion to the extraction
          
          #first do any column header preps
          #noting that for ppd_append case numpy header conversion not supported
          df_new_feature_df = \
          self.__inversion_header_support(df_new_feature_df, postprocess_dict['new_feature_ppd'][i], inversion)
          
          #then invert
          df_new_feature_df, recovered_list, inversion_info_dict = \
          self.__inversion_parent(inversion, df_new_feature_df, postprocess_dict['new_feature_ppd'][i], printstatus, \
                                  pandasoutput, logger)
          
          aggregate_recovered_df = pd.concat([aggregate_recovered_df, df_new_feature_df], axis=1)
          aggregate_recovered_list += recovered_list
          aggregate_inversion_info_dict.update(inversion_info_dict)
          
        #then one more inversion to the original features
      #_________

      #this is all other inversion scenarios
      
      #this performs any header preps (such as privacy encode invert or numpy recovery)
      df_test = self.__inversion_header_support(df_test, postprocess_dict, inversion)
      
      #then the master inversion function
      df_test, recovered_list, inversion_info_dict = \
      self.__inversion_parent(inversion, df_test, postprocess_dict, printstatus, \
                            pandasoutput, logger)
      
      #then if this was a new_feature_ppd_case concat the other results
      if new_feature_ppd_case is True:
        df_test = pd.concat([df_test, aggregate_recovered_df], axis=1)
        recovered_list += aggregate_recovered_list
        aggregate_inversion_info_dict.update(inversion_info_dict)
        inversion_info_dict = aggregate_inversion_info_dict

      return df_test, recovered_list, inversion_info_dict
    #_______

    #_________________________________________________________
    #__WorkflowBlock: postmunge noise_augment duplicate extraction
    #prior to extracting ID and label sets
    #if the noise_augment parameter was passed as >0
    #a copy of df_test is collected
    #which will later be prepared in postmunge for one or more concatinations
    
    if noise_augment > 0:
      df_augment_duplicate = df_test.copy()

    #_________________________________________________________
    #__WorkflowBlock: postmunge ppd_append additional features extraction
    #when the ppd_append option was elected in automunge(.)
    #that means additional features were incorporated into a prior populated postprocess_dict
    #those features will be extracted and later processed seperately in an internal postmunge call
    
    if 'new_feature_ppd' in postprocess_dict and postprocess_dict['new_feature_ppd'] != {}:
      
      dupl_rows_ppd_append_postmunge_valresult = False
      if dupl_rows is not False:
        dupl_rows_ppd_append_postmunge_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "Warning of possible error channel, postmunge dupl_rows not recomended when ppd_append was applied in automunge.")
        self.__autoprint('warning', '')
      logger_dict['validations'].update({'dupl_rows_ppd_append_postmunge_valresult' : dupl_rows_ppd_append_postmunge_valresult})
      logger = self.__logger_append(logger)
      
      new_feature_ppd_extract_headers_dict = {}
      new_feature_ppd_extract_df_dict = {}
      
      #first tier of postprocess_dict['new_feature_ppd'] will be an integer asscoiated with each set of new features
      for i in postprocess_dict and postprocess_dict['new_feature_ppd']:
        
        new_feature_ppd_extract_headers = \
        postprocess_dict['new_feature_ppd'][i]['origtraincolumns']
        
        new_feature_ppd_extract_headers_dict.update(
          {i : new_feature_ppd_extract_headers}
        )
        
        new_feature_ppd_extract_df_dict.update(
          {i : df_test[new_feature_ppd_extract_headers]}
        )
        
        #now drop the extracted columns from df_test
        for new_feature_ppd_extract_header in new_feature_ppd_extract_headers:
          
          del df_test[new_feature_ppd_extract_header]

    #_________________________________________________________
    #__WorkflowBlock: postmunge ID set populated
    #ID list intialized 
    #based on either testID_column postmunge parameter or automunge ID columns if present in df_test
    #incuding validation of resulting list testID_column
    #ID dataframe initialized, including extraction of any non range integer index from df_test
    #and initialization of index column (same 'Automunge_index' as used in automunge)
    #ID columns extracted from df_test and populated in ID set

    #the convention for ID columns is that they are struck from features 
    #and returned consistently partitioned and shuffled in ID sets
    #when the ID parameters are passed as list of list
    #the first list has ID columns that are struck from features
    #and the second list has ID columns that are retained in features
    #which may be useful to inspect before and after configurations for encodings since are consistently shuffled and etc

    testID_struckfeatures = []
    testID_retainedfeatures = []
        
    if isinstance(testID_column, list):
      #if first entry is list specifies features to be struck
      if len(testID_column) > 0 and isinstance(testID_column[0], list):
        testID_struckfeatures = testID_column[0]
      #if second entry is list specifies features to be retained
      if len(testID_column) > 1 and isinstance(testID_column[1], list):
        testID_retainedfeatures = testID_column[1]
      #now consolidate trainID_column to include both struck and retained
      if len(testID_struckfeatures) > 0 or len(testID_retainedfeatures) > 0:
        testID_column = testID_struckfeatures.copy() + testID_retainedfeatures.copy()
      else:
        #if first or second entries weren't a list then all entries are struck
        testID_struckfeatures = testID_column.copy()
    elif testID_column is not False:
      #in cases where trainID_column is a single entry it is added to testID_struckfeatures
      testID_struckfeatures = [testID_column]

    #cast testID_column as a list
    if testID_column is False:
      testID_column = []
    elif isinstance(testID_column, str):
      testID_column = [testID_column]

    if 'trainID_column_features' in postprocess_dict:
      trainID_column_features = postprocess_dict['trainID_column_features']
      trainID_struckfeatures = postprocess_dict['trainID_struckfeatures']
      trainID_retainedfeatures = postprocess_dict['trainID_retainedfeatures']
    else:
      #backward compatibility preceding 7.80
      trainID_column_features = postprocess_dict['trainID_column_orig']
      trainID_struckfeatures = postprocess_dict['trainID_column_orig']
      trainID_retainedfeatures = []

    #we'll have convention that if testID_column=False, if any of trainID_column in df_test
    #then apply those trainID_column to test set
    #using a comparable retain / erase convention consistent with automunge specification when applicable
    trainID_columns_in_df_test_pm = False
    if trainID_column_features != [] and trainID_column_features is not False:
      for trainIDcolumn in trainID_column_features:
        if trainIDcolumn in df_test.columns:
          trainID_columns_in_df_test_pm = True
          if trainIDcolumn not in testID_column:
            testID_column.append(trainIDcolumn)
            if trainIDcolumn in trainID_struckfeatures:
              testID_struckfeatures.append(trainIDcolumn)
            elif trainIDcolumn in trainID_retainedfeatures:
              testID_retainedfeatures.append(trainIDcolumn)

    pm_testID_struck_includes_trainID_retained = False
    if len(set(testID_struckfeatures) & set(trainID_retainedfeatures)) > 0:
      pm_testID_struck_includes_trainID_retained = True
      # if printstatus != 'silent':
      self.__autoprint('warning', 'error: testID_column specification resulted in struck ID columns')
      self.__autoprint('warning', 'that were retained ID columns in the automunge train set')
      self.__autoprint('warning', 'for ID columns:')
      self.__autoprint('warning', set(testID_struckfeatures) & set(trainID_retainedfeatures))
      self.__autoprint('warning', 'to specify retained ID columns can use the [list1, list2] form documented for testID_column parameter in read me.')
      self.__autoprint('warning', '')

    #now run a quick validation that each entry in testID_column list present in df_test
    pm_testID_column_subset_of_df_test_valresult = False
    if not set(testID_column).issubset(set(df_test)):
      pm_testID_column_subset_of_df_test_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: entries to testID_column were not found in df_test")
      self.__autoprint('warning', "note that testID_column can either be passed as string for single entry")
      self.__autoprint('warning', "or testID_column can be passed as a list for multiple entries")
      self.__autoprint('warning', "Note that testID_column is primarily intended for use when ID columns in df_test")
      self.__autoprint('warning', "are different than those ID columns from trainID_column.")
      self.__autoprint('warning', "")
    
    logger_dict['validations'].update({'pm_testID_column_subset_of_df_test_valresult' : pm_testID_column_subset_of_df_test_valresult,
                                                          'trainID_columns_in_df_test_pm' : trainID_columns_in_df_test_pm,
                                                          'pm_testID_struck_includes_trainID_retained' : pm_testID_struck_includes_trainID_retained,
                                                          })
    logger = self.__logger_append(logger)

    #if df_test has a non-range index we'll include that in ID sets as 'Orig_index_###'
    if type(df_test.index) != pd.RangeIndex or \
    'nonrange_extract_marker' in postprocess_dict and postprocess_dict['nonrange_extract_marker'] is True:
      
      orig_index_names = list(df_test.index.names)
      
      #if df_train.index.names == [None]:
      if None in df_test.index.names:
        
        if 'origindexcolumn' in postprocess_dict:
          origindexcolumn = postprocess_dict['origindexcolumn']
        else:
          #backward compatibility preceding ()
          origindexcolumn = 'Orig_index'
        if origindexcolumn in orig_index_names:
          while origindexcolumn in orig_index_names:
            origindexcolumn += ','

        revised_index_names = self.__autocopy(orig_index_names)

        revised_index_names[orig_index_names.index(None)] = origindexcolumn

        df_test = df_test.rename_axis(revised_index_names)
        
      testID_column = testID_column + list(df_test.index.names)
      testID_struckfeatures = testID_struckfeatures + list(df_test.index.names)

      df_test = df_test.reset_index(drop=False)

    #here we derive a range integer index for inclusion in the test ID sets
    indexcolumn = postprocess_dict['indexcolumn']

    tempIDlist = []
    df_test_tempID = pd.DataFrame({indexcolumn:range(0,df_test.shape[0])})

    #now carve out ID sets from df_test for ID sets
    df_testID = pd.DataFrame(df_test[testID_column])
    
    df_test_tempID.index = df_testID.index
    
    df_testID = pd.concat([df_testID, df_test_tempID], axis=1)

    for IDcolumn in testID_struckfeatures:
      del df_test[IDcolumn]

    #then append the indexcolumn to testID_column list for use in later methods
    testID_column = testID_column + [indexcolumn]

    del df_test_tempID

    #_________________________________________________________
    #__WorkflowBlock: postmunge labels and other variable initializations
    #labels columns from automugne(.) are inspected and checked for presence in df_test
    #resulting in an initialized: 
    #labels_column / labels_column_listofcolumns (automunge) 
    #labelscolumn / labels_column_listofcolumns_test (postmunge)
    #the labels dataframe is extracted if lagbels are present
    #also a few variables initialized from postprocess_dict entries
  
    #labels_column with underscore is the automunge train set labels
    labels_column = False
    #labelscolumn without underscore is the postmunge test set labels or False if not present
    labelscolumn = False
    #labels_column_listofcolumns is list of label columns for automunge train set labels
    labels_column_listofcolumns = []
    #labels_column_listofcolumns_test is list of test set label columns, which may be empty set if not present
    labels_column_listofcolumns_test = []
    
    if postprocess_dict['labels_column'] is not False:
      labels_column = postprocess_dict['labels_column']
      labels_column_listofcolumns = postprocess_dict['labels_column_listofcolumns']
    
    if set(labels_column_listofcolumns).issubset(set(df_test)):
      labelscolumn = labels_column
      labels_column_listofcolumns_test = labels_column_listofcolumns

    #as used here labels_column is the label from automunge
    #labelscolumn without underscore is the postmunge version
    #where labelscolumn will be False when labels_column not present in postmunge df_test
    #otherwise labelscolumn = labels_column
    if labelscolumn is not False:
    
    #we originally had convention in both automunge and postmunge that rows with missing data in labels were deleted
    #current convention is that these labels now have default infill associated with applied transformation function
#         df_test = df_test.dropna(subset=[labels_column])

      df_testlabels = pd.DataFrame(df_test[labels_column_listofcolumns_test])
      for labels_column_listofcolumns_test_entry in labels_column_listofcolumns_test:
        del df_test[labels_column_listofcolumns_test_entry]
      
      #if we only had one (label) column to begin with we'll create a dummy test set
      if df_test.shape[1] == 0:
        df_test = df_testlabels[0:1].copy()
  
    else:
      df_testlabels = pd.DataFrame()

    #access a few entries from postprocess_dict
    powertransform = postprocess_dict['powertransform']
    binstransform = postprocess_dict['binstransform']
    NArw_marker = postprocess_dict['NArw_marker']
    floatprecision = postprocess_dict['floatprecision']

    transform_dict = postprocess_dict['transform_dict']
    process_dict = postprocess_dict['process_dict']
      
    assign_param = postprocess_dict['assign_param']

    #_________________________________________________________
    #__WorkflowBlock: postmunge validate data set headers
    #validate columns of df_test match quantity, composition, and order of columns passed to automunge

    #confirm consistency of train and test sets

    #check columns passed to postmunge(.) are consistent with train set passed to automunge(.)
    validate_traintest_columnlabelscompare = False
    if len(set(postprocess_dict['origtraincolumns']) - set(df_test)) > 0 \
    or len(set(df_test) - set(postprocess_dict['origtraincolumns'])) > 0:
      validate_traintest_columnlabelscompare = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error, inconsistent columns between train set passed to automunge(.)")
      self.__autoprint('warning', "and test set passed to postmunge(.)")
      self.__autoprint('warning', '')
      self.__autoprint('warning', "__________")
      self.__autoprint('warning', "original columns passed to automunge(.) (exluding any labels_column and/or trainID_column):")
      self.__autoprint('warning', '')
      self.__autoprint('warning', postprocess_dict['origtraincolumns'])
      self.__autoprint('warning', '')
      self.__autoprint('warning', "__________")
      self.__autoprint('warning', "current columns passed to postmunge(.) (exluding any labelscolumn and/or testID_column):")
      self.__autoprint('warning', '')
      self.__autoprint('warning', list(df_test))
      self.__autoprint('warning', '')
      if len(set(postprocess_dict['origtraincolumns']) - set(df_test)) > 0:
        self.__autoprint('warning', "__________")
        self.__autoprint('warning', "missing following columns in df_test passed to postmunge(.):")
        self.__autoprint('warning', '')
        self.__autoprint('warning', list(set(postprocess_dict['origtraincolumns']) - set(df_test)))
        self.__autoprint('warning', '')
        self.__autoprint('warning', "If this is a label column requires designation in automunge(.)")
        self.__autoprint('warning', "via the labels_column parameter.")
        self.__autoprint('warning', '')
      if len(set(df_test) - set(postprocess_dict['origtraincolumns'])) > 0:
        self.__autoprint('warning', "__________")
        self.__autoprint('warning', "extra columns passed in df_test to postmunge(.) are:")
        self.__autoprint('warning', '')
        self.__autoprint('warning', list(set(df_test) - set(postprocess_dict['origtraincolumns'])))
        self.__autoprint('warning', '')
        self.__autoprint('warning', "Note that extra columns can be carved out in postmunge(.)")
        self.__autoprint('warning', "with testID_column parameter.")
        self.__autoprint('warning', '')
        
      logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
    logger = self.__logger_append(logger)

    #check order of column headers are consistent
    columns_train = postprocess_dict['origtraincolumns']
    columns_test = list(df_test)
    validate_traintest_columnorder = False
    if columns_train != columns_test:
      validate_traintest_columnorder = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error, different order of column labels in the train and test set")
      self.__autoprint('warning', '')
      self.__autoprint('warning', "__________")
      self.__autoprint('warning', "original columns passed to automunge(.) (exluding any labels_column and/or trainID_column):")
      self.__autoprint('warning', '')
      self.__autoprint('warning', postprocess_dict['origtraincolumns'])
      self.__autoprint('warning', '')
      self.__autoprint('warning', "__________")
      self.__autoprint('warning', "current columns passed to postmunge(.) (exluding any labelscolumn and/or testID_column):")
      self.__autoprint('warning', '')
      self.__autoprint('warning', list(df_test))
      self.__autoprint('warning', '')
      logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
      logger = self.__logger_append(logger)
      return
    logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
    logger = self.__logger_append(logger)

    #_________________________________________________________
    #__WorkflowBlock: postmunge drift report evaluated
    #a driftreport assessment is performed here based on postmunge driftreport parameter
    #which may include a drift assessment of source columns conducted in __getNArows
    #a drift assessment of derived columns conducted in __prepare_driftreport
    #or both, as based on driftreport parameter
    #driftreport may either be returned without further processing of data 
    #or postmunge may proceed and return results in postreports_dict
    #note that PCA or Binary drift stats only available for driftreport = True 
    #and are recorded after application of dimensionality reduction

    #__________
    #here we'll perform drift report if elected
      
    if driftreport in {True, 'report_full', 'report_effic'}:
      
      postdrift_dict = {}

      # if printstatus in {True, 'summary'}:
      self.__autoprint('info', "_______________")
      self.__autoprint('info', "Preparing Source Column Drift Report:")
      self.__autoprint('info', "")
      
      for column in df_test:

        if column in postprocess_dict['drift_dict']:

          # if printstatus in {True, 'summary'}:
          self.__autoprint('info', "______")
          self.__autoprint('info', "Preparing source column drift report for column: ", column)
          self.__autoprint('info', "")
          self.__autoprint('info', "original drift stats:")
          self.__autoprint('info', postprocess_dict['drift_dict'][column])
          self.__autoprint('info', "")

          category = postprocess_dict['origcolumn'][column]['category']

          _1, postdrift_dict = \
          self.__getNArows(df_test, column, category, postprocess_dict, postdrift_dict, True)

          # if printstatus in {True, 'summary'}:
          self.__autoprint('info', "new drift stats:")
          self.__autoprint('info', postdrift_dict[column])
          self.__autoprint('info', "")
          
      postreports_dict.update({'sourcecolumn_drift' : {'orig_driftstats' : postprocess_dict['drift_dict'], \
                                                       'new_driftstats' : postdrift_dict}})

      postreports_dict.update({'rowcount_basis' : {'automunge_train_rowcount' : postprocess_dict['train_rowcount'], \
                                                   'postmunge_test_rowcount' : df_test.shape[0]}})

      #dimensionality_reduction_driftstats populated with PCA or Binary and only available with driftreport=True
      postreports_dict.update({'dimensionality_reduction_driftstats' : {}})

      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Source Column Drift Report Complete")
      self.__autoprint('debug', "")

    if driftreport in {True, 'report_full'}:

      #returns a new partially populated postprocess_dict containing
      #column_dict entries populated with newly calculated normalization parameters
      #for now we'll just print the results in the function, a future expansion may
      #return these to the user somehow, need to put some thought into that
      drift_ppd, drift_report = self.__prepare_driftreport(df_test, postprocess_dict, printstatus, logger)

      postreports_dict['driftreport'] = drift_report

    if driftreport in {'report_full', 'report_effic'}:
      
      return [], [], [], postreports_dict
    #end drift report section
    #__________

    #_________________________________________________________
    #__WorkflowBlock: postmunge column processing
    #this for loop through the set of input feature columns applies transformations to test data
    #with root category based on what was applied in automunge
    #includes missing data injection for assignnan and consolidation to NaN representation
    #any needed NArows for infill are aggregated based on the input column and root category and populated in masterNArows_test
    #the postprocessfamily function applies transormations based on the root category's family tree
    #the postcircleoflife function manages column replacements

    #first we check for protected_features, they will be prepared in a seperate for loop 

    if 'protected_features' in postprocess_dict:
      protected_features = postprocess_dict['protected_features']
    else:
      #backward compatibility preceding 7.87
      protected_features = []

    columns_train_copy = self.__autocopy(columns_train)
    if len(protected_features) > 0:
      for protected_feature in protected_features:
        columns_train_copy.remove(protected_feature)

    df_test, \
    postprocess_dict, \
    masterNArows_test, \
    postdrift_dict = \
    self.__apply_postprocessfamily(df_test,
                                    columns_train_copy,
                                    postprocess_dict,
                                    driftreport,
                                    transform_dict,
                                    assign_param,
                                    printstatus,
                                    inplace,
                                    logger,
                                    labels = False,
                                    masterNArows_test = pd.DataFrame(),
                                    postdrift_dict = {},
                                  )

    del columns_train_copy

    #_________________________________________________________
    #__WorkflowBlock: postmunge label column processing
    #this for loop through the set of input feature columns applies transformations to test data
    #with root category based on what was applied in automunge
    #includes missing data injection for assignnan and consolidation to NaN representation
    #does not include NArows aggregation (unless NArw included in family tree)
    #the postprocessfamily function applies transormations based on the root category's family tree
    #the postcircleoflife function manages column replacements

    df_testlabels, \
    postprocess_dict, \
    _1, _2 = \
    self.__apply_postprocessfamily(df_testlabels,
                                    labels_column_listofcolumns_test,
                                    postprocess_dict,
                                    driftreport,
                                    transform_dict,
                                    assign_param,
                                    printstatus,
                                    inplace,
                                    logger,
                                    labels = True,
                                    masterNArows_test = masterNArows_test,
                                    postdrift_dict = {},
                                  )

    #_________________________________________________________
    #__WorkflowBlock: postmunge protected feature processing
    #If any protected features were specified they are prepared here (after features and labels)
    #which serves purposes of ensuring they have a known configuration 
    #when other features that might inspect them are applied

    if len(protected_features) > 0:

      df_test, \
      postprocess_dict, \
      masterNArows_test, \
      postdrift_dict = \
      self.__apply_postprocessfamily(df_test,
                                      protected_features,
                                      postprocess_dict,
                                      driftreport,
                                      transform_dict,
                                      assign_param,
                                      printstatus,
                                      inplace,
                                      logger,
                                      labels = False,
                                      masterNArows_test = masterNArows_test,
                                      postdrift_dict = {},
                                    )

    #_________________________________________________________
    #__WorkflowBlock: postmunge apply infill
    #the infill assignments are accessed from postprocess_dict['postprocess_assigninfill_dict']
    #and infill is performed in __apply_pm_infill

    #now that we've pre-processed all of the columns, let's apply infill
    
    #printout display progress
    # if printstatus is True:
    self.__autoprint('debug', "______")
    self.__autoprint('debug', "")

    #access infill assignments derived in automunge(.) call
    postprocess_assigninfill_dict = \
    postprocess_dict['postprocess_assigninfill_dict']

    df_test, infill_validations = \
    self.__apply_pm_infill(df_test, postprocess_assigninfill_dict, \
                        postprocess_dict, printstatus, list(df_test), \
                        masterNArows_test, logger)

    logger_dict['validations'].update(infill_validations)
    logger = self.__logger_append(logger)

    #_________________________________________________________
    #__WorkflowBlock: postmunge feature importance dimensionality reduction
    #if feature importance dimensionality reduction was performed in automunge it is performed here
    #based on entries stored in postprocess_dict

    #trim branches associated with feature selection
    if postprocess_dict['featureselection'] in {'pct', 'metric'}:

      #get list of columns currently included
      currentcolumns = list(df_test)

      #get list of columns to trim
      trimcolumns = [b for b in currentcolumns if b not in madethecut]

      if len(trimcolumns) > 0:
        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Begin feature importance dimensionality reduction")
        self.__autoprint('debug', "")
        self.__autoprint('debug', "   method: ", postprocess_dict['featureselection'])
        if postprocess_dict['featureselection'] == 'pct':
          self.__autoprint('debug', "threshold: ", postprocess_dict['featurethreshold'])
        if postprocess_dict['featureselection'] == 'metric':
          self.__autoprint('debug', "threshold: ", postprocess_dict['featurethreshold'])
        self.__autoprint('debug', "")
        self.__autoprint('debug', "trimmed columns: ")
        self.__autoprint('debug', trimcolumns)
        self.__autoprint('debug', "")

        #trim columns manually
        for trimmee in trimcolumns:
          del df_test[trimmee]
        
      if len(trimcolumns) > 0:
        # if printstatus is True:
        self.__autoprint('debug', "returned columns: ")
        self.__autoprint('debug', list(df_test))
        self.__autoprint('debug', "")

    #_________________________________________________________
    #__WorkflowBlock: postmunge PCA dimensionality reduction
    #if PCA dimensionality reduction was performed in automunge it is performed here
    #based on entries stored in postprocess_dict
    #including set creating in __postcreatePCAsets
    #valid all numeric validation in __validate_allvalidnumeric
    #derivation in __postPCAfunction
    #and concatination onto df_test

    if postprocess_dict['PCA_applied'] is True:
      #grab parameters from postprocess_dict
      PCAn_components = postprocess_dict['PCAn_components']
      #prePCAcolumns = postprocess_dict['prePCAcolumns']

      if PCAn_components != None:

        PCAset_test, PCAexcl_posttransform = \
        self.__postcreatePCAsets(df_test, postprocess_dict)

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Applying PCA dimensionality reduction")
        self.__autoprint('debug', "")
        if len(postprocess_dict['PCAexcl']) > 0:
          self.__autoprint('debug', "columns excluded from PCA: ")
          self.__autoprint('debug', postprocess_dict['PCAexcl'])
          self.__autoprint('debug', "")
            
        #quick validation that PCA set has all valid numeric entries
        PCA_test_numeric_data_result, PCA_test_all_valid_entries_result = \
        self.__validate_allvalidnumeric(PCAset_test, printstatus)
  
        logger_dict['validations'].update({'PCA_test_numeric_data_result': PCA_test_numeric_data_result})
        logger_dict['validations'].update({'PCA_test_all_valid_entries_result': PCA_test_all_valid_entries_result})
        logger = self.__logger_append(logger)

        PCAset_test, postprocess_dict = \
        self.__postPCAfunction(PCAset_test, postprocess_dict)

        #PCA_retain can be activated to retain columns serving as PCA basis
        PCA_retain = False
        if 'PCA_retain' in ML_cmnd and ML_cmnd['PCA_retain'] is True:
          PCA_retain = True
              
        if PCA_retain is False:
          #reattach the excluded columns to PCA set (retaining the original index from df_train)
          df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test[PCAexcl_posttransform]], axis=1)
        if PCA_retain is True:
          #reattach the excluded columns to PCA set (retaining the original index from df_train)
          df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test], axis=1)

        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "returned PCA columns: ")
        self.__autoprint('debug', list(PCAset_test))
        self.__autoprint('debug', "")

        del PCAset_test

        if driftreport is True:

          # if printstatus is True:
          self.__autoprint('debug', "_______________")
          self.__autoprint('debug', 'Preparing PCA Column Drift Report:')
          
          #record orig PCA drift stats
          if 'dimensionality_reduction_driftstats' in postprocess_dict:
            postreports_dict['dimensionality_reduction_driftstats'].update({
                'PCA_drift' : {
                  'orig_stats' : postprocess_dict['dimensionality_reduction_driftstats']['PCA_drift']['orig_stats'],
                }
              })
          else:
            #backward compatibility preceding 7.59
            postreports_dict['dimensionality_reduction_driftstats'].update({
                'PCA_drift' : {
                  'orig_stats' : {},
                }
              })

          #record new PCA drift stats
          postreports_dict['dimensionality_reduction_driftstats']['PCA_drift'].update(
            {
              'new_stats' : self.__PCA_drift_stats(df_test, postprocess_dict['returned_PCA_columns'])
            })

          # if printstatus is True:
          for returned_PCA_column in postreports_dict['dimensionality_reduction_driftstats']['PCA_drift']['orig_stats']:
            self.__autoprint('debug', '')
            self.__autoprint('debug', "drift stats for PCA column ", returned_PCA_column)
            self.__autoprint('debug', '')
            self.__autoprint('debug', "original drift stats:")
            self.__autoprint('debug', postreports_dict['dimensionality_reduction_driftstats']['PCA_drift']['orig_stats'][returned_PCA_column])
            self.__autoprint('debug', '')
            self.__autoprint('debug', "new drift stats:")
            self.__autoprint('debug', postreports_dict['dimensionality_reduction_driftstats']['PCA_drift']['new_stats'][returned_PCA_column])
            self.__autoprint('debug', '')

      else:
        
        logger_dict['validations'].update({'PCA_test_numeric_data_result': False})
        logger_dict['validations'].update({'PCA_test_all_valid_entries_result': False})
        logger = self.__logger_append(logger)

    #_____

    #_________________________________________________________
    #__WorkflowBlock: postmunge Binary dimensionality reduction (categoric consolidations)
    #if Binary dimensionality reduction was performed in automunge it is performed here
    #based on entries stored in postprocess_dict
    #as channeled through __postBinaryConsolidate
    #similarly if the labels included a categoric consolidation it is also performed here 

    #Binary dimensionality reduction goes here
  
    #test processing
    Binary = postprocess_dict['Binary']
    
    #access meta_Binary_dict
    meta_Binary_dict = postprocess_dict['Binary_dict']
      
    df_test = self.__postBinaryConsolidate(df_test, meta_Binary_dict, Binary, printstatus)

    if driftreport is True \
    and 'dimensionality_reduction_driftstats' in postprocess_dict \
    and 'Binary_drift' in postprocess_dict['dimensionality_reduction_driftstats']:

      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', 'Preparing Binary Column Drift Report:')

      #record orig Binary drift stats
      if 'dimensionality_reduction_driftstats' in postprocess_dict:
        postreports_dict['dimensionality_reduction_driftstats'].update({
            'Binary_drift' : {
              'orig_stats' : postprocess_dict['dimensionality_reduction_driftstats']['Binary_drift']['orig_stats'],
            }
          })
      else:
        #backward compatibility preceding 7.59
        postreports_dict['dimensionality_reduction_driftstats'].update({
            'Binary_drift' : {
              'orig_stats' : {},
            }
          })

      #record new Binary drift stats
      postreports_dict['dimensionality_reduction_driftstats']['Binary_drift'].update(
        {
          'new_stats' : self.__Binary_drift_stats(df_test, meta_Binary_dict)
        })

      # if printstatus is True:
      for Binary_number in postreports_dict['dimensionality_reduction_driftstats']['Binary_drift']['orig_stats']:
        self.__autoprint('debug', '')
        self.__autoprint('debug', "drift stats for Binary consolidation ", Binary_number)
        self.__autoprint('debug', '')
        self.__autoprint('debug', "original drift stats:")
        self.__autoprint('debug', postreports_dict['dimensionality_reduction_driftstats']['Binary_drift']['orig_stats'][Binary_number])
        self.__autoprint('debug', '')
        self.__autoprint('debug', "new drift stats:")
        self.__autoprint('debug', postreports_dict['dimensionality_reduction_driftstats']['Binary_drift']['new_stats'][Binary_number])
        self.__autoprint('debug', '')
    
    #_________
    
    #label processing
    if len(labels_column_listofcolumns_test) > 0:
      
      if 'labelBinary' in postprocess_dict:
        Binary = postprocess_dict['labelBinary']
        meta_Binary_dict = postprocess_dict['labels_Binary_dict']

        df_testlabels = self.__postBinaryConsolidate(df_testlabels, meta_Binary_dict, Binary, printstatus)

    #_____

    #_________________________________________________________
    #__WorkflowBlock: postmunge duplicate row consolidation
    #duplicated row consolidations are optionally performed by the dupl_rows paraemter

    #populate row count basis here (before duplications or oversampling)
    postreports_dict.update({'rowcount_basis' : {'automunge_train_rowcount' : postprocess_dict['train_rowcount'], \
                                                  'postmunge_test_rowcount' : df_test.shape[0]}})
                                                  
    #this is operation to consolidate duplicate rows based on dupl_rows parameter
    #in other words, if multiple copies of same row present only returns one
    if dupl_rows is True:
      df_test, df_testID, df_testlabels = self.__dupl_rows_consolidate(df_test, df_testID, df_testlabels)

    #_________________________________________________________
    #__WorkflowBlock: postmunge label rebalancing (e.g. oversampling)
    #oversampling is optionally performed based on inspection of the TrainLabelFreqLevel parameter
    #applied in function __LabelFrequencyLevelizer
    #note that oversampling is consistently applied between features, labels, and ID sets

    #here is the process to levelize the frequency of label rows in train data
    #based on postmunge TrainLabelFreqLevel parameter
    if TrainLabelFreqLevel is True \
    and labelscolumn is not False:

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "_______________")
      self.__autoprint('debug', "Begin label rebalancing")
      self.__autoprint('debug', "")
      self.__autoprint('debug', "Before rebalancing row count = ")
      self.__autoprint('debug', df_testlabels.shape[0])
      self.__autoprint('debug', "")

      if testID_column is not False:

        #add trainID set to train set for consistent processing
        df_test = pd.concat([df_test, df_testID], axis=1)                        

      #apply LabelFrequencyLevelizer defined function
      df_test, df_testlabels = \
      self.__LabelFrequencyLevelizer(df_test, df_testlabels, postprocess_dict)

      #extract trainID
      if testID_column is not False:

        df_testID = pd.DataFrame(df_test[testID_column])

        if isinstance(testID_column, str):
          tempIDlist = [testID_column]
        elif isinstance(testID_column, list):
          tempIDlist = testID_column
        for IDcolumn in tempIDlist:
          del df_test[IDcolumn]
        #del df_train[trainID_column]

      #printout display progress
      # if printstatus is True:
      self.__autoprint('debug', "After rebalancing row count = ")
      self.__autoprint('debug', df_testlabels.shape[0])
      self.__autoprint('debug', "")

    #_________________________________________________________
    #__WorkflowBlock: postmunge row shuffling
    #row shuffling is performed based on shuffletrain parameter
    #or amy also be performed in a privacy_encode scenario
    #note that shuffling is consistently applied between features, labels, and ID sets

    #if shuffletrain passed to postmunge it takes place here
    #(postmunge does not default to consistent shuffle as train set, relies on parameter)
    #row shuffling uses postmunge randomseed
    if shuffletrain is True or postprocess_dict['privacy_encode'] == 'private':
      #shuffle training set and labels
      df_test = self.__df_shuffle(df_test, postprocess_dict['postmunge_randomseed'])
      df_testlabels = self.__df_shuffle(df_testlabels, postprocess_dict['postmunge_randomseed'])

      if testID_column is not False:
        df_testID = self.__df_shuffle(df_testID, postprocess_dict['postmunge_randomseed'])

    #_________________________________________________________
    #__WorkflowBlock: postmunge data type management
    #unlike integer types which are managed in transformation functions
    #float data types are collectively managed by the floatprecision parameter
    #unless deactivated for a category in dtype_convert process_dict entry
    #float transformations are applied by way of __floatprecision_transform
    #also any conversion to pandas categorical dtype if elected based on cat_type parameter performed here

    #now we'll apply the floatprecision transformation
    floatcolumns_test = list(df_test)
    floatcolumns_testlabels = list(df_testlabels)

    #floatprecision adjustment only applied to columns returned from transforms
    #with mlinfilltype in {'numeric', 'concurrent_nmbr', 'exclude'}
    #when dtype_convert not deactivated for the associated category in processdict
    floatcolumns_test_copy = floatcolumns_test.copy()
    for floatcolumn in floatcolumns_test_copy:
      if floatcolumn not in postprocess_dict['returned_PCA_columns'] and \
      floatcolumn in postprocess_dict['column_dict'] and \
      (postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['MLinfilltype'] \
      not in {'numeric', 'concurrent_nmbr', 'exclude'} \
      or 'dtype_convert' in postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']] \
      and postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['dtype_convert'] is False):
        floatcolumns_test.remove(floatcolumn)

    floatcolumns_testlabels_copy = floatcolumns_testlabels.copy()
    for floatcolumn in floatcolumns_testlabels_copy:
      if floatcolumn not in postprocess_dict['returned_PCA_columns'] and \
      floatcolumn in postprocess_dict['column_dict'] and \
      (postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['MLinfilltype'] \
      not in {'numeric', 'concurrent_nmbr', 'exclude'} \
      or 'dtype_convert' in postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']] \
      and postprocess_dict['process_dict'][postprocess_dict['column_dict'][floatcolumn]['category']]['dtype_convert'] is False):
        floatcolumns_testlabels.remove(floatcolumn)

    #now we'll apply the floatprecision transformation
    df_test = self.__floatprecision_transform(df_test, floatcolumns_test, floatprecision)
    if labelscolumn is not False:
      df_testlabels = self.__floatprecision_transform(df_testlabels, floatcolumns_testlabels, floatprecision)
      
    #now if cat_type activated convert df_test and df_testlabels to comparable types with same categoric basis
    if 'cat_type' in postprocess_dict and postprocess_dict['cat_type'] is True:
      #backward compatibility note: cat_type, dtype_df, dtype_labels_df added in 7.90
          
      #note that postprocess_dict['dtype_df'] only contains columns converted as a result of cat_type parameter
      df_test[list(postprocess_dict['dtype_df'].columns)] = df_test[list(postprocess_dict['dtype_df'].columns)].astype(postprocess_dict['dtype_df'].dtypes)
          
      #similar address for labels          
      df_testlabels[list(postprocess_dict['dtype_labels_df'].columns)] = df_testlabels[list(postprocess_dict['dtype_labels_df'].columns)].astype(postprocess_dict['dtype_labels_df'].dtypes)

    #_________________________________________________________
    #__WorkflowBlock: postmunge excl suffix management
    #the excl (full passthrough) transform has a unique suffix convention in library
    #in that a suffix is appended in transformations (for data structure management)
    #and then removed in the returned data (signifying a full passthrough)
    #here any translation is performed based on conversion dictionaries populated in automunge

    #a special case, those columns that we completely excluded from processing via excl
    #we'll scrub the suffix appender
    if postprocess_dict['excl_suffix'] is False:
      
      df_test_columns = list(df_test)
      self.__list_replace(df_test_columns, postprocess_dict['excl_suffix_inversion_dict'])
      df_test.columns = df_test_columns
      
      if labelscolumn is not False:
        
        df_testlabels_columns = list(df_testlabels)
        self.__list_replace(df_testlabels_columns, postprocess_dict['excl_suffix_inversion_dict'])
        df_testlabels.columns = df_testlabels_columns

    #_________________________________________________________
    #__WorkflowBlock: postmunge privacy encoding
    #if privacy encoding was performed in automunge it is now applied here
    #using conversion dictionaries populated in automunge
    #the encoding includes column shuffling (row shuffling may have been performed in the shuffling block)
    #and in 'private' scenario the indexes are reset (index even reset for ID sets, the original index can be recovered in ID set columns)

    #if orig_headers was selected in automunge it is also performed here
    #note that privacy_encode takes precendence over orig_headers

    #if automunge orig_headers parameter activated strip suffix appenders (privacy_encode takes precedence)
    #this primarily intended to support use case of integration into existing data pipeline
    #first if statement for backward compatibility preceding 7.83
    if 'orig_headers' in postprocess_dict:

      if postprocess_dict['orig_headers'] is True and postprocess_dict['privacy_encode'] is False:
      
        df_test = self.__orig_headers_support(df_test, postprocess_dict)
        df_testlabels = self.__orig_headers_support(df_testlabels, postprocess_dict)

    #now rename columns and shuffle order of columns for privacy encoding when activated
    #column shuffling uses automunge random seed
    if postprocess_dict['privacy_encode'] is not False:

      df_test = self.__df_shuffle(df_test, postprocess_dict['randomseed'], axis=1)
      df_test = df_test.rename(columns = postprocess_dict['privacy_headers_train_dict'])

    if postprocess_dict['privacy_encode'] == 'private':
      if labelscolumn is not False:
        df_testlabels = self.__df_shuffle(df_testlabels, postprocess_dict['randomseed'], axis=1)
        df_testlabels = df_testlabels.rename(columns = postprocess_dict['privacy_headers_labels_dict'])

    if postprocess_dict['privacy_encode'] == 'private':
      
      #this resets the dataframe index
      df_test = df_test.reset_index(drop=True)
      df_testID = df_testID.reset_index(drop=True)
      if labelscolumn is not False:
        df_testlabels = df_testlabels.reset_index(drop=True)

      # #this resets the Automunge_index column returned in ID sets
      # df_testID[postprocess_dict['indexcolumn']] = pd.DataFrame({postprocess_dict['indexcolumn']:range(0,df_testID.shape[0])})

    #_________________________________________________________
    #__WorkflowBlock: postmunge ppd_append additional features processing and concatination
    #when the automunge(.) ppd_append option was applied to add additional features onto a prior populated postprocess_dict
    #those features are extracted earlier in workflow at #__WorkflowBlock: postmunge ppd_append additional features extraction
    #here those extractions are passed to an internal postmunge(.) call
    #and then concatinated onto the right side of df_test in order of addition
    #(i.e., when multiple new features were added in multiple ppd_append's, they are registered in an order)
    #note this uses the postmunge randomseed to prepare the additional data
    #note that dupl_rows not recomended when automunge ppd_append was applied, dupl_rows_ppd_append_postmunge_valresult logged above

    if 'new_feature_ppd' in postprocess_dict and postprocess_dict['new_feature_ppd'] != {}:
      
      for i in postprocess_dict['new_feature_ppd']:
        
        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Preparing additional features from ppd_append ", i)
        self.__autoprint('debug', "")

        #following postmunge call will return validation results in original bucket
        #which we'll store in ppd_append_validations
        orig_validations_dict = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}
          
        new_feature_df, _1, _2, _3 = \
        self.postmunge(postprocess_dict['new_feature_ppd'][i], new_feature_ppd_extract_df_dict[i],
                      printstatus = printstatus,
                      dupl_rows = False,
                      traindata = traindata,
                      shuffletrain = shuffletrain,
                      entropy_seeds = entropy_seeds, 
                      random_generator = random_generator,
                      sampling_dict = sampling_dict,
                      randomseed = randomseed,
                      logger = logger)

        #logger_dict validation results stored in logger_dict['ppd_append_validations']
        logger_dict['ppd_append_validations'] = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}
        logger_dict['validations'] = self.__autocopy(orig_validations_dict)

        df_test = pd.concat([df_test, new_feature_df], axis=1)

    #_________________________________________________________
    #__WorkflowBlock: postmunge noise_augment prep duplicate sets
    #when noise_augment is passed as integer greater than 0
    #the duplicate set that was extracted in #__WorkflowBlock: postmunge noise_augment duplicate extraction
    #is prepared in postmunge and concatinated onto the test, ID, and label sets
    #the convention is the first duplicate is prepared without noise
    #unless noise_augment is float(int) type
    #in which case all duplicates are prepared with noise
    #after concatination the duplicates are collectively shuffled based on shuffletrain parameter

    if noise_augment > 0:
      
      for i in range(int(noise_augment)):
        
        #printout display progress
        # if printstatus is True:
        self.__autoprint('debug', "_______________")
        self.__autoprint('debug', "Preparing noise_augment duplicate number ", i+1)
        self.__autoprint('debug', "")
      
        #if noise_augment was passed as integer dtype 
        #then first duplicate is prepared without noise
        #(if noise_augment is a float then all sets have noise)
        
        #this differs from approach in automunge(.) 
        #in that noise augment can be applied to both train and test scenarios
        #in case user set traindata to a no_noise scenario
        #we'll retain convention that only one set has no noise (in that case the base set)
        if i == 0 and isinstance(noise_augment, int):

          if traindata is True:
            aug_traindata = 'train_no_noise'
            
          elif traindata == 'train_no_noise':
            aug_traindata = True
            
          elif traindata is False:
            aug_traindata = 'test_no_noise'
            
          elif traindata == 'test_no_noise':
            aug_traindata = False
            
        else:

          if traindata in {True, 'train_no_noise'}:
            aug_traindata = True
            
          elif traindata in {False, 'test_no_noise'}:
            aug_traindata = False
          
        #note that np.random used for noise doesn't accept a random seed, this seed is for other operations
        augrandomseed = False
        if randomrandomseed is False:
          augrandomseed = randomseed
        
        #these are postmunge specific temporary postprocess_dict entries, reset after this postmunge call
        traindata = postprocess_dict['traindata']
        postmunge_randomseed = postprocess_dict['postmunge_randomseed']

        #following postmunge call will return validation results in original bucket
        #which we'll store in post_noise_augment_validations
        orig_validations_dict = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}

        duplicate_i, duplicate_i_ID, duplicate_i_labels, \
        _1 = \
        self.postmunge(postprocess_dict, df_augment_duplicate,
                      printstatus = printstatus,
                      dupl_rows = dupl_rows,
                      traindata = aug_traindata,
                      entropy_seeds = entropy_seeds, 
                      random_generator = random_generator,
                      sampling_dict = sampling_dict,
                      pandasoutput = 'dataframe',
                      randomseed = augrandomseed,
                      logger = logger)

        #logger_dict validation results stored in logger_dict['post_noise_augment_validations']
        logger_dict['post_noise_augment_validations'] = self.__autocopy(logger_dict['validations'])
        logger_dict['validations'] = {'suffixoverlap_results':{}}
        logger_dict['validations'] = self.__autocopy(orig_validations_dict)

        #now reset the temporary postmunge postprocess_dict entries
        #(postmunge will have struck any temporary entries)
        postprocess_dict['postmunge_randomseed'] = postmunge_randomseed
        postprocess_dict['traindata'] = traindata

        #this adjusts Automunge_index to avoid duplicates
        #indexcolumn is 'Automunge_index' from automunge or may be adjusted in cases of overlap
        #indexcolumn will be same in privacy_encode scenario (ID sets not anonymized)
        duplicate_i_ID[postprocess_dict['indexcolumn']] += (df_testID[postprocess_dict['indexcolumn']].max() + 1)

        df_test = pd.concat([df_test, duplicate_i], axis=0, ignore_index=True)
        df_testID = pd.concat([df_testID, duplicate_i_ID], axis=0, ignore_index=True)

        if labelscolumn is not False:
          #single column labels returned from postmunge will be a dataframe
          df_testlabels = pd.concat([pd.DataFrame(df_testlabels), pd.DataFrame(duplicate_i_labels)], axis=0, ignore_index=True)

        del duplicate_i, duplicate_i_ID, duplicate_i_labels, _1
      
      del df_augment_duplicate

      #this is an additional shuffle to the one performed in #__WorkflowBlock: postmunge row shuffling
      #it results in the duplicates being intermingled
      #to maintain distinct sets can turn off shuffletrain 
      #can still distinguish with shuffling by df_test.shape in comparison to Automunge_index
      if shuffletrain is True:
        
        df_test = self.__df_shuffle(df_test, randomseed)
        df_testID = self.__df_shuffle(df_testID, randomseed)
        
        if labelscolumn is not False:
          df_testlabels = self.__df_shuffle(df_testlabels, randomseed)

    #_________________________________________________________
    #__WorkflowBlock: postmunge final preps and return
    #final preps include any numpy conversion based on pandasoutput parameter
    #any flattenting (like with ravel or casting as series) for single column sets
    #some anonymizations associated with privacy encode when applicable
    #striking any temporary postprocess_dict entries used for application
    #and returnedsets option to return all sets concatinated into single dataframe may also be performed when elected
    #Postmunge Complete

    #this defragments the dataframe
    if inplace is not True:
      df_test = df_test.copy()

    #here's a list of final column names saving here since the translation to \
    #numpy arrays scrubs the column names
    finalcolumns_test = list(df_test)

    postreports_dict['finalcolumns_test'] = finalcolumns_test
    
    postreports_dict.update({'sourcecolumn_drift' : {'orig_driftstats' : postprocess_dict['drift_dict'], \
                                                     'new_driftstats' : postdrift_dict}})

    #this functionality is still experimental
    #this is for returning results of inference to the label sets
    #when a final_model was trained after automunge(.) with automodel(.)

    if labelscolumn is False:
      if 'final_model' in postprocess_dict:
        # if printstatus in {True, 'summary'}:
        self.__autoprint('info', "_______________")
        self.__autoprint('info', "Running inference with model saved from automodel(.)")
        self.__autoprint('info', "This functionality is still experimental.")
        self.__autoprint('info', "Results of inference will be returned as the test_labels set.")
        self.__autoprint('info', '')

        labelscolumn = postprocess_dict['labels_column']

        df_testlabels = \
        self.autoinference(df_test, postprocess_dict, encrypt_key = False, 
                           printstatus = printstatus, randomseed = randomseed)

    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "Postmunge returned test column set: ")
    self.__autoprint('info', list(df_test))
    self.__autoprint('info', "")

    self.__autoprint('info', "_______________")
    if df_testID.empty is False:
      self.__autoprint('info', "Postmunge returned ID column set: ")
      self.__autoprint('info', list(df_testID))
      self.__autoprint('info', "")

    if labelscolumn is not False:
      self.__autoprint('info', "Postmunge returned label column set: ")
      self.__autoprint('info', list(df_testlabels))
      self.__autoprint('info', "")

    if testID_column is not False:
      if returnedsets in {'test_ID', 'test_ID_labels'}:
        df_test = pd.concat([df_test, df_testID], axis=1)
      
    else:
      df_testID = pd.DataFrame()

    if labelscolumn is not False:
      if returnedsets in {'test_labels', 'test_ID_labels'}:
        df_test = pd.concat([df_test, df_testlabels], axis=1)
      
    else:
      df_testlabels = pd.DataFrame()

    #else output numpy arrays, using dtype='object' to retain column dtype distinctions between integer and float
    if pandasoutput is False:
      #global processing to test set including conversion to numpy array
      df_test = df_test.to_numpy()

      if testID_column is not False \
      and returnedsets not in {False, 'test_ID', 'test_labels', 'test_ID_labels'}:
        df_testID = df_testID.to_numpy()
      else:
        df_testID = pd.DataFrame().to_numpy()

      if labelscolumn is not False \
      and returnedsets not in {False, 'test_ID', 'test_labels', 'test_ID_labels'}:
        df_testlabels = df_testlabels.to_numpy()

        #apply ravel to labels if appropriate - converts from eg [[1,2,3]] to [1,2,3]
        if df_testlabels.ndim == 2 and df_testlabels.shape[1] == 1:
          df_testlabels = np.ravel(df_testlabels)

      else:
        df_testlabels = pd.DataFrame().to_numpy()

    #else flatten any single column label dataframes to series
    elif pandasoutput is True:
      if len(df_testlabels.shape) > 1 and df_testlabels.shape[1] == 1:
        df_testlabels = df_testlabels[df_testlabels.columns[0]]

    #reset traindata entry in postprocess_dict to avoid overwrite of external
    postprocess_dict['traindata'] = False
    del postprocess_dict['postmunge_randomseed']

    #consolide validation results and reset temporary log in postprocess_dict
    postprocess_dict['entropy_seeds'] = False
    postprocess_dict['random_generator'] = False
    postprocess_dict['sampling_dict'] = False
    postprocess_dict['test_plug_marker'] = test_plug_marker_orig

    postreports_dict['pm_miscparameters_results'].update(logger_dict['validations'])

    #a few anonymizations for privacy_encode in returned postreports_dict
    if postprocess_dict['privacy_encode'] is not False:
      del postreports_dict['featureimportance']
      del postreports_dict['FS_sorted']
      del postreports_dict['driftreport']
      del postreports_dict['rowcount_basis']
      del postreports_dict['sourcecolumn_drift']
      if 'postfeatureselect_automungecall_validationresults' in postreports_dict['pm_miscparameters_results']:
        postreports_dict['pm_miscparameters_results']['postfeatureselect_automungecall_validationresults'] = {}

    #printout display progress
    # if printstatus in {True, 'summary'}:
    self.__autoprint('info', "_______________")
    self.__autoprint('info', "Postmunge Complete")
    self.__autoprint('info', "")

    #final consolidation of logger_dict with any externally initialized logger dictionary
    logger.update(logger_dict)
    
    if returnedsets is True:
    
      return df_test, df_testID, df_testlabels, postreports_dict
    
    else:
      
      return df_test

  #________________________________________________
  #03 - inversion related Function Blocks
  #________________________________________________

  #__FunctionBlock: data structure assembly support functions
    
  def __populate_categorytree(self, postprocess_dict):
    """
    #Populates mirror tree of transformations
    #to facilitate translation between transformation category space 
    #and suffix appender space
    
    #As an example of a populated tree for bxcx root cateogry transform 
    #which returns columns ['column_bxcx_nmbr', 'column_NArw']
    
    categorytree = \
    {'__root__' : \
    ['__root__', '', ['__root__'], {'NArw': ['sup', inputcol, categorylist, {}], \
                                    'bxcx': ['rep', inputcol, categorylist, \
                                            {'nmbr': ['rep', inputcol, categorylist, {}]}]}]}
    
    #here 'sup'/'rep' refers to distinction between supplement and replace primitives
    #and 'root' is for the source column entry
    
    #inputcol and categorylist access from column_dict from key of one of entries in categorylist
    #so we'll search column_dict for key with entries that match category 
    #and inputcolumn that matches categorylist entry of preceding entry
    #(convention is we only have downstream transforms applied to single entry categorylists)
    
    #Once we have this populated, we'll translate it to an inverse
    #with returned columns in the root and root in each final
    #such as to facilitate any translation from returned sets to source sets
    #(such as for converting predictions back to original label formatting)
    
    #I think this will work let's give it a try
    """

    #these are the columns passed to automunge(.) including labels
    source_columns = list(postprocess_dict['origcolumn'])
    
    #these are derived columns including replaced columns and labels
    produced_columns = list(postprocess_dict['column_dict'])
    
    #initialize categorytree
    categorytree = {'__root__' : {}}
    
    for origcolumn in source_columns:
      
      categorytree['__root__'].update({origcolumn : ['__root__', '', ['__root__'], {}]})
      
      root_category = postprocess_dict['origcolumn'][origcolumn]['category']
          
      parents     = postprocess_dict['transform_dict'][root_category]['parents']
      siblings    = postprocess_dict['transform_dict'][root_category]['siblings']
      auntsuncles = postprocess_dict['transform_dict'][root_category]['auntsuncles']
      cousins     = postprocess_dict['transform_dict'][root_category]['cousins']
      
      categorytree_entry = self.__populate_family(postprocess_dict, categorytree['__root__'][origcolumn][3], origcolumn, '__root__', \
                                           parents, siblings, auntsuncles, cousins)
      
      categorytree['__root__'][origcolumn][3].update(categorytree_entry)
      
    return categorytree

  def __populate_family(self, postprocess_dict, categorytree, inputcolumn, inputcategory, \
                      parents, siblings, auntsuncles, cousins):
    """
    #populates categorytree entries from seeding of a source-column and root category
    
    #we will run in order of
    #parents, auntsuncles, siblings, cousins
    
    #see also notes for populate_categorytree function
    """

    for entry in parents:
      
      if entry != None:
      
        categorylist = self.__get_categorylist(postprocess_dict, inputcolumn, entry)
        
        if entry not in categorytree:
          categorytree.update({entry : {}})

        #parents have downstream offspring
        children      = postprocess_dict['transform_dict'][entry]['children']
        niecesnephews = postprocess_dict['transform_dict'][entry]['niecesnephews']
        coworkers     = postprocess_dict['transform_dict'][entry]['coworkers']
        friends       = postprocess_dict['transform_dict'][entry]['friends']

        #convention is downstream offspring only allowed on sets returned with single column categorylist
        if len(categorylist) == 1:

          parentcolumn = categorylist[0]
          
          #parents is replace primitive
          categorytree[entry].update({parentcolumn : ['rep', inputcolumn, categorylist, {}]})

          categorytree_entry = self.__populate_family(postprocess_dict, categorytree[entry][parentcolumn][3], parentcolumn, entry, \
                                               children, niecesnephews, coworkers, friends)
          
          categorytree[entry][parentcolumn][3].update(categorytree_entry)
          
        else:
          
          for category_column in categorylist:
            
            categorytree[entry].update({category_column : ['rep', inputcolumn, categorylist, {}]})
          
    for entry in auntsuncles:
      
      if entry != None:
        
        if entry not in categorytree:
          categorytree.update({entry : {}})
      
        categorylist = self.__get_categorylist(postprocess_dict, inputcolumn, entry)
#         categorylist = get_categorylist(postprocess_dict, inputcolumn, entry)
        
        for category_column in categorylist:

          #auntsuncles is replace primitive
          categorytree[entry].update({category_column : ['rep', inputcolumn, categorylist, {}]})

          #auntsuncles have no downstream offspring
      
    for entry in siblings:
      
      if entry != None:
      
        categorylist = self.__get_categorylist(postprocess_dict, inputcolumn, entry)

        if entry not in categorytree:
          categorytree.update({entry : {}})

        #siblings have downstream offspring
        children      = postprocess_dict['transform_dict'][entry]['children']
        niecesnephews = postprocess_dict['transform_dict'][entry]['niecesnephews']
        coworkers     = postprocess_dict['transform_dict'][entry]['coworkers']
        friends       = postprocess_dict['transform_dict'][entry]['friends']

        #convention is downstream offspring only allowed on sets returned with single column categorylist
        if len(categorylist) == 1:

          parentcolumn = categorylist[0]
          
          #siblings is supplement primitive
          categorytree[entry].update({parentcolumn : ['sup', inputcolumn, categorylist, {}]})


          categorytree_entry = self.__populate_family(postprocess_dict, categorytree[entry][parentcolumn][3], parentcolumn, entry, \
                                               children, niecesnephews, coworkers, friends)
          
          categorytree[entry][parentcolumn][3].update(categorytree_entry)
          
        else:
          for category_column in categorylist:
            
            categorytree[entry].update({category_column : ['sup', inputcolumn, categorylist, {}]})
        
    for entry in cousins:
      
      if entry != None:
      
        categorylist = self.__get_categorylist(postprocess_dict, inputcolumn, entry)

        if entry not in categorytree:
          categorytree.update({entry : {}})

        #cousins have no downstream offspring
        
        for category_column in categorylist:

          #cousins is supplement primitive
          categorytree[entry].update({category_column : ['sup', inputcolumn, categorylist, {}]})

          #auntsuncles have no downstream offspring
      
    return categorytree
    
  def __get_categorylist(self, postprocess_dict, inputcolumn, category):
    """
    #access a returned categorylist
    #corresponding to the category of transformation applied to an inputcolumn
    #by searching entries to postprocess_dict['column_dict']
    """

    categorylist = []
    
    for entry in postprocess_dict['column_dict']:
      
      if postprocess_dict['column_dict'][entry]['inputcolumn'] == inputcolumn \
      and postprocess_dict['column_dict'][entry]['category'] == category:
        
        categorylist = postprocess_dict['column_dict'][entry]['categorylist']
          
        break
        
    #this is for edge case when a transform does not return columns
    if categorylist == None:
      categorylist = []
        
    return categorylist
      
  def __populate_inverse_categorytree(self, postprocess_dict, labelscase=False):
    """
    #So this is similar to the categorytree in that we are mirror the transformations
    #in a populated data structure
    #but in this inverse version the bottom tier are the conclusion branches
    #which progress back to the root
    #note that we'll allow redundant entries in first tiers
    #such as to aggregate each distinct path by common starting point in list
    
    #here we'll want additional data points for:
    #- depth of branch
    #- information retention of transform
    #- availability of inverse transform
    
    #As an example of excerpt from a populated tree for bxcx root category transform 
    #which returns columns ['column_bxcx_nmbr', 'column_NArw']
    
    inverse_categorytree['nmbr'] = \
    {'Age_bxcx_nmbr': ['sup', 'Age_bxcx', ['Age_bxcx_nmbr'], ['Age_bxcx_nmbr', 'Age_NArw'],
                      2, False, False, \
                      {'bxcx': {'Age_bxcx': ['rep', 'Age', ['Age_bxcx'], ['Age_bxcx_nmbr', 'Age_NArw'],
                                             1, False, False, \
                                             {'__root__': {'Age': ['__root__','__root__','__root__',\
                                                                   '__root__', 1 , False , False, \
                                                                    {}]}}]}}]}

    labelscase=True is used to populated an additional version specific to labels in labelsencoding_dict
    """

    #all returned columns including labels
    returned_columns = \
    postprocess_dict['pre_dimred_finalcolumns_train'] + postprocess_dict['pre_dimred_finalcolumns_labels']

    if labelscase is True:
      #if this inverse_categorytree is meant for the lablesencoding_dict we'll only populate for labels
      returned_columns = \
      postprocess_dict['pre_dimred_finalcolumns_labels']

    #convert returned_columns to returned suffix convention for excl edge case
    self.__list_replace(returned_columns, postprocess_dict['excl_suffix_conversion_dict'])
    
    #these are all derived columns including replaced columns and labels
    produced_columns = list(postprocess_dict['column_dict'])
    
    #these are the columns passed to automunge(.) including labels
    source_columns = list(postprocess_dict['origcolumn'])
    
    #initialize inverse_categorytree
    inverse_categorytree = {}
    
    for returned_column in returned_columns:
      
      category     = postprocess_dict['column_dict'][returned_column]['category']
      inputcolumn  = postprocess_dict['column_dict'][returned_column]['inputcolumn']
      origcolumn   = postprocess_dict['column_dict'][returned_column]['origcolumn']
      columnslist  = postprocess_dict['column_dict'][returned_column]['columnslist']
      categorylist = postprocess_dict['column_dict'][returned_column]['categorylist']
      
      if category not in inverse_categorytree:
      
        inverse_categorytree.update(
        {category : {}}
        )
      
      depth = 1
      
      info_retention = False
      if 'info_retention' in postprocess_dict['process_dict'][category]:
        if postprocess_dict['process_dict'][category]['info_retention'] is True:
          info_retention = True
      
      transforms_avail = False
      if 'custom_inversion' in postprocess_dict['process_dict'][category]:
        if callable(postprocess_dict['process_dict'][category]['custom_inversion']):
          transforms_avail = True
      if 'inverseprocess' in postprocess_dict['process_dict'][category]:
        if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
          transforms_avail = True
      
      for entry in categorylist:
        
        if entry in returned_columns:
          sup_or_rep = 'sup'
        else:
          sup_or_rep = 'rep'

        #note the index number of entries in this list are used to access
        #so any added points here should be tacked on end (after {})
        #and mirrored in other function
        inverse_categorytree[category].update(
        {entry : [sup_or_rep, 
                  inputcolumn, 
                  categorylist, 
                  columnslist, 
                  depth, 
                  info_retention,
                  transforms_avail,
                  {}]}
        )
        
        if inputcolumn not in source_columns:

          inverse_categorytree_entry, depth_, info_retention_, transforms_avail_ = \
          self.__populate_inverse_family(
            postprocess_dict, inverse_categorytree[category][entry][7], inputcolumn,
            returned_columns, source_columns, postprocess_dict['printstatus']
          )
          
          inverse_categorytree[category][entry][4] += depth_
          
          inverse_categorytree[category][entry][5] = \
          inverse_categorytree[category][entry][5] and info_retention_
          
          inverse_categorytree[category][entry][6] = \
          inverse_categorytree[category][entry][6] and transforms_avail_
          
          inverse_categorytree[category][entry][7].update(inverse_categorytree_entry)
        
        else:
          
          inverse_categorytree[category][entry][7].update(
          {'__root__' : {inputcolumn : ['__root__', 
                                        '__root__', 
                                        '__root__', 
                                        '__root__', 
                                        depth, 
                                        info_retention,
                                        transforms_avail,
                                        {}]}}
          )
          
    return inverse_categorytree
  
  def __populate_inverse_family(self, postprocess_dict, inverse_categorytree, column, \
                              returned_columns, source_columns, printstatus):
    """
    #populates inverse_categorytree entries from seeding of an inputcolumn
    
    #see also notes for populate_inverse_categorytree function
    """
  
    category     = postprocess_dict['column_dict'][column]['category']
    inputcolumn  = postprocess_dict['column_dict'][column]['inputcolumn']
    origcolumn   = postprocess_dict['column_dict'][column]['origcolumn']
    columnslist  = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']
    
    if category not in inverse_categorytree:
    
      inverse_categorytree.update(
      {category : {}}
      )
    
    depth = 1

    info_retention = False
    if 'info_retention' in postprocess_dict['process_dict'][category]:
      if postprocess_dict['process_dict'][category]['info_retention'] is True:
        info_retention = True

    transforms_avail = False
    if 'custom_inversion' in postprocess_dict['process_dict'][category]:
      if callable(postprocess_dict['process_dict'][category]['custom_inversion']):
        transforms_avail = True
    if 'inverseprocess' in postprocess_dict['process_dict'][category]:
      if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
        transforms_avail = True
        
    for entry in categorylist:

      if entry in returned_columns:
        sup_or_rep = 'sup'
      else:
        sup_or_rep = 'rep'
      
      #note the index number of entries in this list are used to access
      #so any added points here should be tacked on end (after {})
      #and mirrored in other function
      inverse_categorytree[category].update(
      {entry : [sup_or_rep, 
                inputcolumn, 
                categorylist, 
                columnslist, 
                depth, 
                info_retention,
                transforms_avail,
                {}]}
      )
      
      if inputcolumn not in source_columns:
        
        inverse_categorytree_entry, depth_, info_retention_, transforms_avail_ = \
        self.__populate_inverse_family(
          postprocess_dict, inverse_categorytree[category][entry][7], inputcolumn,
          returned_columns, source_columns, printstatus
        )
  
        inverse_categorytree[category][entry][4] += depth_

        inverse_categorytree[category][entry][5] = \
        inverse_categorytree[category][entry][5] and info_retention_

        inverse_categorytree[category][entry][6] = \
        inverse_categorytree[category][entry][6] and transforms_avail_

        inverse_categorytree[category][entry][7].update(inverse_categorytree_entry)

      else:

        inverse_categorytree[category][entry][7].update(
        {'__root__' : {inputcolumn : ['__root__', 
                                      '__root__', 
                                      '__root__', 
                                      '__root__', 
                                      depth, 
                                      info_retention,
                                      transforms_avail,
                                      {}]}}
        )
        
        
    depth            = inverse_categorytree[category][entry][4]
    info_retention   = inverse_categorytree[category][entry][5]
    transforms_avail = inverse_categorytree[category][entry][6]
    
    return inverse_categorytree, depth, info_retention, transforms_avail
  
  def __populate_inputcolumn_dict(self, postprocess_dict):
    """
    #we'll create another structure, this one flatted, similar to origcolumn or column_dict
    #this one as inputcolumn_dict
    
    #this will populate a strucutre with example entry
    
    #inputcolumn_dict = {inputcolumn : {category : {column : column_dict[column]}}}
    
    #where inputcolumn is a column serving as input to a specific generation (set) of trasnformtions
    #such as either entries to parents/siblings/auntsuncles/cousins
    #or for downstream generations entries to children/niecesnephews/coworkers/friends
    """

    inputcolumn_dict = {}
    
    for column in postprocess_dict['column_dict']:
      
      inputcolumn = postprocess_dict['column_dict'][column]['inputcolumn']
      
      if inputcolumn not in inputcolumn_dict:
        
        inputcolumn_dict.update({inputcolumn : {}})
        
      category = postprocess_dict['column_dict'][column]['category']
      
      if category not in inputcolumn_dict[inputcolumn]:
        
        inputcolumn_dict[inputcolumn].update({category:{}})
        
      inputcolumn_dict[inputcolumn][category].update({column : postprocess_dict['column_dict'][column]})
      
    return inputcolumn_dict

  def __populate_labelsencoding_dict_support(self, labelsencoding_dict, postprocess_dict, labels_column_entry, inputcolumn):
    """
    #support function to populate labelsencoding_dict
    #to ensure normalization_dict entries are included for columns not returned from transforms
    #such as those subject to replacement
    #which will be excluded from columnkeylist
    """

    labelscategory = postprocess_dict['origcolumn'][labels_column_entry]['category']
    
    #if inputcolumn has a column_dict entry we'll add the normalization_dict
    if inputcolumn in postprocess_dict['column_dict']:
      #this data strucutre is converted in __populate_labelsencoding_dict_support3 to column_dict entries
      #so don't need to populate the entire normalization_dict
      labelsnormalization_dict_key = list(postprocess_dict['column_dict'][inputcolumn]['normalization_dict'])[0]
      labelsencoding_dict['transforms'][labels_column_entry][labelscategory].update({labelsnormalization_dict_key:{}})
      
      #now do the same for the next inputcolumn, this will halt once the inputcolumn reaches the column passed to automunge(.)
      inputinputcolumn = postprocess_dict['column_dict'][inputcolumn]['inputcolumn']      
      labelsencoding_dict = self.__populate_labelsencoding_dict_support(labelsencoding_dict, postprocess_dict, labels_column_entry, inputinputcolumn)
    
    return labelsencoding_dict

  def __populate_labelsencoding_dict_support2(self, mirror_dict_, postprocess_dict, transform_dict, category, i):
    """
    #populates transform_dict and process_dict entries to mirror_dict or labelsencoding_dict
    #but only those associated with applied transforms
    #where mirror_dict is the returned form for postprocess_dict['transform_dict'] and postprocess_dict['process_dict']
    #which serves the purpose of omitting transform_dict and process_dict entries not inspected for derivations
    #which benefits privacy
    
    #since this function is used both for labelsencoding_dict and mirror_dict
    #we'll refer to the target dictionary as mirror_dict_ just in case of memory leakage

    #the received category is a root category applied to an input feature or input label
    #e.g. root category = postprocess_dict['origcolumn'][labels_column_entry]['category']

    #the family tree of that root category is inspected and upstream primitives are inspected with i=0
    #downstream tiers will receive category entries to primitive with offspring and i>0
    """

    if 'transform_dict' not in mirror_dict_:
      mirror_dict_.update({'transform_dict' : {}})
      
    if 'process_dict' not in mirror_dict_:
      mirror_dict_.update({'process_dict' : {}})
    
    familytree = transform_dict[category]
    
    mirror_dict_['transform_dict'].update({category : transform_dict[category]})
    mirror_dict_['process_dict'].update({category : postprocess_dict['process_dict'][category]})
    
    #i will be 0 for upstream primitives
    if i == 0:
      
      #populate process_dict for all category entries
      for primitive in ['parents', 'siblings', 'auntsuncles', 'cousins']:
        if len(familytree[primitive]) > 0:
          for category in familytree[primitive]:
            if category != None:
              mirror_dict_['process_dict'].update({category : postprocess_dict['process_dict'][category]})
      
      #populate transform_dict for categories with offspring, populated by calling function recursively
      for primitive in ['parents', 'siblings']:
        if len(familytree[primitive]) > 0:
          for category in familytree[primitive]:
            if category != None:
              i+=1
              mirror_dict_, i = self.__populate_labelsencoding_dict_support2(mirror_dict_, postprocess_dict, transform_dict, category, i)
              i=0
    
    #i will > 0 for downstream primitives
    elif i > 0:
      
      #populate process_dict for all category entries
      for primitive in ['children', 'niecesnephews', 'coworkers', 'friends']:
        if len(familytree[primitive]) > 0:
          for category in familytree[primitive]:
            if category != None:
              mirror_dict_['process_dict'].update({category : postprocess_dict['process_dict'][category]})
      
      #populate transform_dict for categories with offspring, populated by calling function recursively
      for primitive in ['children', 'niecesnephews']:
        if len(familytree[primitive]) > 0:
          for category in familytree[primitive]:
            if category != None:
              i+=1
              mirror_dict_, i = self.__populate_labelsencoding_dict_support2(mirror_dict_, postprocess_dict, transform_dict, category, i)

    return mirror_dict_, i

  def __populate_labelsencoding_dict_support3(self, labelsencoding_dict, postprocess_dict):
    """
    populates labelsencoding_dict with entries inspected for label inversion
    used when privacy_encode = True and encryption performed via encrypt_key parameter
    this results in 'column_dict' entries replacing the corresponding 'transforms' entries populated in __populate_labelsencoding_dict_support

    basically all that's happening here is we're mirroring label records from postprocess_dict
    into the labelsencoding_dict
    which will be returned as a public entry when encryption performed with encrypt_key and privacy_encode != 'private'
    for use to allow label inversion without the need for an encryption key
    """

    labelsencoding_dict['column_dict'] = {}
    labelsencoding_dict['inverse_categorytree'] = {}
    labelsencoding_dict['origcolumn'] = {}
    labelsencoding_dict['excl_suffix'] = postprocess_dict['excl_suffix']
    labelsencoding_dict['excl_suffix_conversion_dict'] = {}
    labelsencoding_dict['excl_suffix_inversion_dict'] = {}
    labelsencoding_dict['privacy_encode'] = postprocess_dict['privacy_encode']
    labelsencoding_dict['finalcolumns_train'] = []
    labelsencoding_dict['finalcolumns_labels'] = postprocess_dict['finalcolumns_labels']
    labelsencoding_dict['labels_column'] = postprocess_dict['labels_column']
    labelsencoding_dict['labels_column_listofcolumns'] = postprocess_dict['labels_column_listofcolumns']
    labelsencoding_dict['madethecut'] = []

    for inputcolumn in labelsencoding_dict['transforms']:
      for rootcategory in labelsencoding_dict['transforms'][inputcolumn]:
        for derivedcolumn in labelsencoding_dict['transforms'][inputcolumn][rootcategory]:

          labelsencoding_dict['column_dict'].update({derivedcolumn : postprocess_dict['column_dict'][derivedcolumn]})

          if labelsencoding_dict['column_dict'][derivedcolumn]['category'] == 'excl':
            labelsencoding_dict['excl_suffix_conversion_dict'].update({labelsencoding_dict['column_dict'][derivedcolumn]['inputcolumn'] : derivedcolumn})
            labelsencoding_dict['excl_suffix_inversion_dict'].update({derivedcolumn : labelsencoding_dict['column_dict'][derivedcolumn]['inputcolumn']})

    del labelsencoding_dict['transforms']
            
    for origcolumn in postprocess_dict['labels_column_listofcolumns']:
      labelsencoding_dict['origcolumn'].update({origcolumn : postprocess_dict['origcolumn'][origcolumn]})

    labelsencoding_dict['excl_columns_without_suffix'] = list(labelsencoding_dict['excl_suffix_conversion_dict'])
    
    #this data structure supports inversion path selection
    labels_inverse_categorytree = \
    self.__populate_inverse_categorytree(postprocess_dict, labelscase=True)
    
    labelsencoding_dict['inverse_categorytree'] = labels_inverse_categorytree
    
    return labelsencoding_dict

  #__FunctionBlock: inversion preprocessing support
  
  def __LS_invert(self, LabelSmoothing, df, categorylist, postprocess_dict):
    """
    #Converts smoothed labels back to one-hot encoding
    #for a particular categorylist
    """

    if LabelSmoothing > 0 and LabelSmoothing < 1:
      
      for categorylist_entry in categorylist:
        
        df = \
        self.__autowhere(df, categorylist_entry, df[categorylist_entry] == LabelSmoothing, 1, specified='replacement')
        
        df[categorylist_entry] = df[categorylist_entry].astype(np.int8)
        
    return df

  def __inversion_header_support(self, df, postprocess_dict, inversion):
    """
    #handles cases where headers might need to be adjusted prior to inversion
    #if privacy_encode is False only need to adjust headers for numpy array scenario
    #if privacy encode is True also need to convert from private headers
    #note that inversion is not supported for privacy_encode == 'private'
    #and return order of columns
    """

    if postprocess_dict['privacy_encode'] is False:
      
      if inversion == 'labels':
        if len(df.columns) == len(postprocess_dict['finalcolumns_labels']):
          if set(df.columns) != set(postprocess_dict['finalcolumns_labels']):
            #this is for data passed as numpy arrays
            df.columns = postprocess_dict['finalcolumns_labels']

      else:
        if len(df.columns) == len(postprocess_dict['finalcolumns_train']):
          if set(df.columns) != set(postprocess_dict['finalcolumns_train']):
            #this is for data passed as numpy arrays
            df.columns = postprocess_dict['finalcolumns_train']
    
    if postprocess_dict['privacy_encode'] in {True, 'private'}:

      if inversion not in {'labels', 'denselabels'}:
        
        #this is for data passed as numpy arrays
        if len(df.columns) == len(postprocess_dict['privacy_headers_train']):
          df.columns = postprocess_dict['privacy_headers_train']

        #now recover original column headers
        df = df.rename(columns = postprocess_dict['inverse_privacy_headers_train_dict'])
        
        #now recover original order of columns prior to privacy encoding
        df = df.reindex(columns=postprocess_dict['finalcolumns_train'])

    if postprocess_dict['privacy_encode'] == 'private':
      
      if inversion in {'labels', 'denselabels'}:
        
        #this is for data passed as numpy arrays
        if len(df.columns) == len(postprocess_dict['privacy_headers_labels']):
          df.columns = postprocess_dict['privacy_headers_labels']
        
        #now recover original column headers
        df = df.rename(columns = postprocess_dict['inverse_privacy_headers_labels_dict'])
        
        #now recover original order of columns prior to privacy encoding
        df = df.reindex(columns=postprocess_dict['finalcolumns_labels'])

    return df

  #__FunctionBlock: custom inversion support

  def __custom_inverseprocess_wrapper(self, df, categorylist, postprocess_dict):
    """
    A wrapper for custom inversion functions applied in postmunge
    Where custom inversions follow templates of custom_inversion_template
    
    The purpose of this wrapper is to extend the minimized versions of custom inversions
    To include other conventions of the library
    Such as access inputs from data structures and etc.
    
    The form of the _custom_inverseprocess_wrapper inputs/outputs 
    will be similar to what is applied for inverseprocess functions
    
    Receives dataframe of a test or labels set as df
    categorylist is a list of the returned columns from forward pass that are the basis of inversion
    postprocess_dict is the dictionary data structure passed between transforms
    
    Returns the resulting transformed dataframe df
    As well as inputcolumn which is the header of the recovered column (as may result from removing the suffix appender)
    
    Note that this wrapper does not perform infill
    The convention in the library is that missing entries in recovered sets are recorded as np.nan
    category serving as basis
    
    We'll also create seperately a similar wrapper for process functions applied in automunge (_custom_process_wrapper)
    And likewise a similar wrapper for postprocess functions applied in postmunge (_custom_postprocess_wrapper)
    """

    #ok so to call our inversion function we need df, returnedcolumn_list, inputcolumn, normalization_dict)
    #we have df
    #returnedcolumn_list is same as categorylist
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    normalization_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]
    treecategory = postprocess_dict['column_dict'][normkey]['category']
    
    if 'custom_inversion' in postprocess_dict['process_dict'][treecategory] \
    and postprocess_dict['process_dict'][treecategory]['custom_inversion'] != None:
      
      df = \
      postprocess_dict['process_dict'][treecategory]['custom_inversion'](df, categorylist, inputcolumn, normalization_dict)
    
    return df, inputcolumn

  #__FunctionBlock: postmunge inversion inverseprocess and custom_inversion functions

  def _inverseprocess_nmbr(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_numerical 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    std = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['std']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = ((df[normkey] - offset) * std / multiplier ) + mean
    
    return df, inputcolumn

  def _inverseprocess_year(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_year 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    if 'timemean' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey] \
    and 'timestd' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:

      mean = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['timemean']
      std = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['timestd']
      
      inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
      
      df[inputcolumn] = ((df[normkey]) * std + mean )
    
    return df, inputcolumn
  
  def _inverseprocess_mean(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mean 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = ((df[normkey] - offset) * maxminusmin / multiplier ) + mean
    
    return df, inputcolumn
  
  def _inverseprocess_MADn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_MADn 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    center = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['center']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    if center == 'mean':
      df[inputcolumn] = df[normkey] * MAD + mean
    elif center == 'max':
      df[inputcolumn] = df[normkey] * MAD + maximum
    
    return df, inputcolumn
  
  def _inverseprocess_MAD3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_MAD3 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    datamax = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['datamax']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * MAD + datamax
    
    return df, inputcolumn
  
  def _inverseprocess_mnmx(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mnmx 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * maxminusmin + minimum
    
    return df, inputcolumn

  def _inverseprocess_mnm3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mnmx 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemin']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemax']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * maxminusmin + minimum
    
    return df, inputcolumn

  def _inverseprocess_mxab(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mxab 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    maxabs = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxabs']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * maxabs
    
    return df, inputcolumn
  
  def _inverseprocess_retn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_retn 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    divisor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']

    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    if maximum >= 0 and minimum <= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier
      
    elif maximum >= 0 and minimum >= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier + minimum
      
    elif maximum <= 0 and minimum <= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier + maximum
    
    return df, inputcolumn

  def _inverseprocess_shft(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_shft 
    #assumes any relevant parameters were saved in normalization_dict
    #applies zzzinfill infill
    """

    normkey = categorylist[0]
    
    periods = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['periods']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey].shift(periods = -periods)
    
    df[inputcolumn] = df[inputcolumn].fillna(np.nan)
        
    return df, inputcolumn

  def _inverseprocess_qttf(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to _process_qttf
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    qttf = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['qttf']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #convert all values to either numeric or NaN
    df[inputcolumn] = pd.to_numeric(df[normkey], errors='coerce')
    
    #apply inverse_transform
    if qttf is not False:
      df[inputcolumn] = pd.DataFrame(qttf.inverse_transform(pd.DataFrame(df[inputcolumn])), index=df.index)

    #to align with inversion convention
    df[inputcolumn] = df[inputcolumn].fillna(np.nan)
    
    return df, inputcolumn
  
  def _inverseprocess_log0(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_log0 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 10 ** df[normkey]
    
    return df, inputcolumn
  
  def _inverseprocess_logn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_logn 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = np.e ** df[normkey]

    #this is to recover sign convention for lgnr
    #this breaks convention that inversion only based on columns in categorylist
    #based on lgnr family tree and process_dict's as of 5.85
    if inputcolumn in postprocess_dict['column_dict']:

      input_toinputcolumn = \
      postprocess_dict['column_dict'][inputcolumn]['inputcolumn']
      
      sign_column = input_toinputcolumn + '___mltp_bkt3'
      
      if sign_column in df.columns:
        
        df = \
        self.__autowhere(df, inputcolumn, df[sign_column]==1, (-1) * df[inputcolumn], specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_addd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_addd 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    add = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['add']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] - add
    
    return df, inputcolumn
  
  def _inverseprocess_sbtr(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbtr 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    subtract = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['subtract']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] + subtract
    
    return df, inputcolumn
  
  def _inverseprocess_mltp(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mltp 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    multiply = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiply']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] / multiply
    
    return df, inputcolumn
  
  def _inverseprocess_divd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_divd 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    divide = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divide']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * divide
    
    return df, inputcolumn
  
  def _inverseprocess_rais(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_rais 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    raiser = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['raiser']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] ** (1 / raiser)
    
    return df, inputcolumn
  
  def _inverseprocess_absl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_absl 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete info retention, no transformation performed
    #this funciton only populated to support partial info recovery
    #in case a full info_retention path is not available
    """

    normkey = categorylist[0]
    
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey]
    
    return df, inputcolumn
  
  def _inverseprocess_sqrt(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sqrt 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] ** 2
    
    return df, inputcolumn

  def _inverseprocess_bnst(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnst
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str

    #when upstreaminteger is False this does not invert
    """

    normkey = categorylist[0]
    
    upstreaminteger = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['upstreaminteger']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    inputtextcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
    
    for i, inputtextcolumn in enumerate(inputtextcolumns):
      
      if upstreaminteger is True:
        df[inputtextcolumn] = df[normkey].str.slice(i,i+1).astype(int)
      
      else:
        #inversion requires a fixed character width
        df[inputtextcolumn] = df[normkey]
        
    #this returns the first inputcolumn in upstream categorylist
    inputcolumn = inputtextcolumns[0]
    
    return df, inputcolumn
  
  def _inverseprocess_UPCS(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_UPCS 
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey].copy()
    
    return df, inputcolumn
  
  def _inverseprocess_excl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_excl 
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']

    #to accomodate excl_suffix=False scenario
    #(the scenario where the test data returned from processing had excl suffix removed)
    #this will replace column header in df to convert from header without suffix to header with suffix 
    if postprocess_dict['excl_suffix'] is False:
      normkey_without_suffix = postprocess_dict['excl_suffix_inversion_dict'][normkey]
      df.rename(columns={normkey_without_suffix : normkey}, inplace=True)

    df[inputcolumn] = df[normkey]
    
    return df, inputcolumn

  def _inverseprocess_pwr2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_pwr2 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #note that pwr2 differs from pwrs in that negative numbers are allowed
    
    #this only achieves partial information recovery as power level returned to single column
    
    #in interest of expediency, building this method to extract power
    #level from column suffix appender
    #a potential improvement would be to add an additional entry to pwrs normalization_dict
    #matching column header to power, eg {'column_10^-1' : -1}
    #saving that for a future update
    #(same functionality, but would better match convention of library for use of column headers)
    """

    normkey = categorylist[0]
    
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df.loc[:, inputcolumn] = 0
    
    #for pwr2 trasnform suffix is either '_10^#' for positive values or '_-10^#' for negative
    for column in categorylist:
      
      if column[len(inputcolumn) + 2 + len(suffix)] == '1':
      
        power = int(column.replace(inputcolumn + '_' + suffix + '_10^', ''))
      
        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, 10 ** power, specified='replacement')
        
      if column[len(inputcolumn) + 2 + len(suffix)] == '-':
        
        power = int(column.replace(inputcolumn + '_' + suffix + '_-10^', ''))
        
        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, -(10 ** power), specified='replacement')
        
      if column[len(inputcolumn) + 2 + len(suffix)] == 'z':
        
        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, 0, specified='replacement')
        
    return df, inputcolumn
  
  def _inverseprocess_pwor(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_por2 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    #this has keys of pwr2 equivalent column headers and values of ordinal values found in column
    train_replace_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['train_replace_dict']
    
    inverse_train_replace_dict = {value:key for key,value in train_replace_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df.loc[:, inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
        
    #for pwr2 trasnform suffix is either '_10^#' for positive values or '_-10^#' for negative
    for column in train_replace_dict:
      
      #this dictionary is including a nan key entry for infill points, we'll leave these points as 0
      if column == column:

        if column[len(inputcolumn)+1] == '1':

          power = int(column.replace(inputcolumn + '_10^', ''))

          df = \
          self.__autowhere(df, inputcolumn, df[normkey] == train_replace_dict[column], 10 ** power, specified='replacement')

        if column[len(inputcolumn)+1] == '-':

          power = int(column.replace(inputcolumn + '_-10^', ''))

          df = \
          self.__autowhere(df, inputcolumn, df[normkey] == train_replace_dict[column], -(10 ** power), specified='replacement')

      #note if zerosets selected they are already zero at this point
      else:
        df = \
        self.__autowhere(df, inputcolumn, df[normkey] == train_replace_dict[column], np.nan, specified='replacement')

    return df, inputcolumn

  def _inverseprocess_bins(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bins 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete information recovery
    """

    normkey = categorylist[0]
    
    binsmean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    binsstd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
    bincuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
    binlabels = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binscolumns']
    bincount = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll follow convention that values placed in mid point of buckets
    #eg for bucket with boundaries mean through mean+stsndard devation
    #we'll return value mean + 1/2 standard deviation
    #and for tails we'll arbitraricly follow consistent deltas between buckets
    #(so this loses a lot of tail distribution information in the inversion)
    returned_values_dict = {}
    
    i = - (bincount - 2) / 2 - 0.5
    
    for bucket in binlabels:
      
      #relies on suffix appender conventions
      column = inputcolumn + '_' + suffix + '_' + bucket
      value = (i * binsstd + binsmean)
      
      returned_values_dict.update({column : value})
      
      i += 1
      
    df[inputcolumn] = np.nan
    
    for column in categorylist:
        
      df = \
      self.__autowhere(df, inputcolumn, df[column] == 1, returned_values_dict[column], specified='replacement')
      
    return df, inputcolumn

  def _inverseprocess_bsor(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bsor 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete information recovery
    """

    normkey = categorylist[0]
    
    #this has keys of pwr2 equivalent column headers and values of ordinal values found in column
    binsmean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    binsstd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    bincount = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
    binlabels = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll follow convention that values placed in mid point of buckets
    #eg for bucket with boundaries mean through mean+stsndard devation
    #we'll return value mean + 1/2 standard deviation
    #and for tails we'll arbitraricly follow consistent deltas between buckets
    #(so this loses a lot of tail distribution information in the inversion)
    returned_values_dict = {}

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    i = - (bincount - 2) / 2 - 0.5
    
    df[inputcolumn] = np.nan
    
    for bucket in binlabels:
      
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == bucket, (i * binsstd + binsmean), specified='replacement')

      i += 1

    return df, inputcolumn
  
  def _inverseprocess_bnwd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnwd 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_width_bnwd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width_bnwd']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    

    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df.loc[:, inputcolumn] = 0
    
    for i in range(len(bins_id)):
      
      _id =  bins_id[i]
      
      column = inputcolumn + suffix + '_' + _id
      
      if column in df.columns:
      
        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, i * bn_width_bnwd + bn_min, specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bnwo(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnwo 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_width = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey].copy()

    df[inputcolumn] = df[inputcolumn].astype(int, errors='ignore')
    
    df[inputcolumn] = df[inputcolumn] * bn_width + bn_min
    
    return df, inputcolumn
  
  def _inverseprocess_bnep(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnep 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df.loc[:, inputcolumn] = 0

    #bins_id is False when original data in train set was all non-numeric
    if bins_id is not False:
      
      for i in range(len(textcolumns)):
        
        column = textcolumns[i]
        _id = int(bins_id[i])
        
        if i == 0:
          value = bins_cuts[i+1]
          
        elif i == len(textcolumns)-1:
          value = bins_cuts[i]
          
        else:
          value = (bins_cuts[i] + bins_cuts[i+1]) / 2

        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, value, specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bneo(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bneo 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df.loc[:, inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')

    #bins_id is False when original train set was all non-numeric
    if bins_id is not False:
      
      for i in range(len(bins_id)):
        
        _id = int(bins_id[i])
        
        if i == 0:
          value = bins_cuts[i+1]
          
        elif i == len(bins_id)-1:
          value = bins_cuts[i]
          
        else:
          value = (bins_cuts[i] + bins_cuts[i+1]) / 2

        df = \
        self.__autowhere(df, inputcolumn, df[normkey] == _id, value, specified='replacement')
    
    return df, inputcolumn

  def _inverseprocess_tlbn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_tlbn
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bincount = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_max = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = np.nan

    #bins_cuts is False when original train set was all non-numeric
    if bins_cuts is not False:
      
      for i, textcolumn in enumerate(textcolumns):
        
        if i == 0:
          
          df = \
          self.__autowhere(df, 
                          inputcolumn, 
                          df[textcolumn] >= 0, 
                          df[textcolumn] * (-1) * (bins_cuts[i+1] - bn_min) + bins_cuts[i+1], 
                          specified='replacement')
          
        # elif i == bincount - 1:
        elif i == len(textcolumns) - 1:
          
          df = \
          self.__autowhere(df, 
                          inputcolumn, 
                          df[textcolumn] >= 0, 
                          df[textcolumn] * (bn_max - bins_cuts[i]) + bins_cuts[i], 
                          specified='replacement')
          
        else:
          
          df = \
          self.__autowhere(df, 
                          inputcolumn, 
                          df[textcolumn] >= 0, 
                          df[textcolumn] * (bins_cuts[i+1] - bins_cuts[i]) + bins_cuts[i], 
                          specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bkt1(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt1 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    buckets_bkt1 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt1']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df.loc[:, inputcolumn] = 0
    
    bucket_ids = []
    for textcolumn in textcolumns:
      bucket_id = textcolumn.replace(inputcolumn + '_' + suffix + '_', '')
      bucket_ids.append(int(bucket_id))

    for i in bucket_ids:
      
      textcolumn = inputcolumn + '_' + suffix + '_' + str(i)
      
      if i == 0:
        
        value = buckets_bkt1[i]
        
      elif i == bins_id[-1]:
        
        value = buckets_bkt1[-1]
        
      else:
        
        value = (buckets_bkt1[i-1] + buckets_bkt1[i]) / 2
        
      df = \
      self.__autowhere(df, inputcolumn, df[textcolumn] == 1, value, specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bkt2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt2 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    buckets_bkt2 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt2']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df.loc[:, inputcolumn] = 0
    
    bucket_ids = []
    for textcolumn in textcolumns:
      bucket_id = textcolumn.replace(inputcolumn + '_' + suffix + '_', '')
      bucket_ids.append(int(bucket_id))
      
    for i in bucket_ids:
      
      value = (bins_cuts[i] + bins_cuts[i+1]) / 2
      
      df = \
      self.__autowhere(df, inputcolumn, df[inputcolumn + '_' + suffix + '_' + str(i)]==1, value, specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bkt3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt3 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    buckets = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    ordl_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
    infill_activation = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    df.loc[:, inputcolumn] = 0

    #infill recovery
    df = \
    self.__autowhere(df, inputcolumn, df[normkey] == infill_activation, np.nan, specified='replacement')
    
    for i in bins_id:
      
      if i == 0:
        value = bins_cuts[i+1]
        
      elif i == len(bins_id)-1:
        value = bins_cuts[i]
        
      else:
        value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == i, value, specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_bkt4(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt4 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """

    normkey = categorylist[0]
    
    buckets = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    infill_activation = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df.loc[:, inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')

    #infill recovery
    df = \
    self.__autowhere(df, inputcolumn, df[normkey] == infill_activation, np.nan, specified='replacement')
    
    for i in bins_id:
      
      value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == i, value, specified='replacement')
    
    return df, inputcolumn

  def _custom_inversion_onht(self, df, returnedcolumn_list, inputcolumn, normalization_dict):
    """
    #rewrite of the onht inversion
    #corresponding to _custom_train_onht
    """

    #First let's access the values we'll need from the normalization_dict
    labels_dict = normalization_dict['labels_dict']
    missing_marker = normalization_dict['missing_marker']
    
    inverse_labels_dict = {value:key for key,value in labels_dict.items()}
    
    df[inputcolumn] = missing_marker
    
    for categorylist_entry in inverse_labels_dict:
      
      df = \
      self.__autowhere(df, inputcolumn, df[categorylist_entry]==1, inverse_labels_dict[categorylist_entry], specified='replacement')

    return df

  def _inverseprocess_smth(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_smth
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """

    normkey = categorylist[0]
    
    textlabelsdict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
    activation = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['activation']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    
    #inverse_labels_dict maps {returnedcolumn : inputcolumn}
    inverse_labels_dict = {value:key for key,value in textlabelsdict.items()}
    
    #note this is just one of the columns in the upstream multicolumn set
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #only apply label smoothing inversion if this was a traindata set with smoothing applied
    smoothing_applied = False
    for returnedcolumn in inverse_labels_dict:
      unique_set = set(pd.unique(df[returnedcolumn]))
      if (unique_set != {0,1} \
      and unique_set != {0} \
      and unique_set != {1}):
        smoothing_applied = True
        
    #initialize input columns
    df[list(textlabelsdict)] = df[list(inverse_labels_dict)].copy()
        
    if smoothing_applied is True:
      
      df = self.__LS_invert(activation, df, textcolumns, postprocess_dict)
    
    #this returns an arbitrary one of the input columns which is fine
    return df, inputcolumn

  def _custom_inversion_GPS1(self, df, returnedcolumn_list, inputcolumn, normalization_dict):
    """
    #corresponding to _custom_train_GPS1
    #recovers form of comma seperated entries
    #with entries left blank other than latt and long coordinates
    #for the default configuration, this populates as ',,DDMM.MMMMMMM,C,DDMM.MMMMMMM,C,,,,,,,,,'
    #this will invert to format of DDMM.... or DDDMM.... when recovered degree magnitude neccesitates
    """

    #First let's access the values we'll need from the normalization_dict
    GPS_convention = normalization_dict['GPS_convention']
    comma_addresses = normalization_dict['comma_addresses']
    comma_count = normalization_dict['comma_count']
    latt_column = normalization_dict['latt_column']
    long_column = normalization_dict['long_column']
    
    supportcolumn1 = 'A1'
    supportcolumn2 = 'B2'
    #since we don't check for suffix overlap edge case in inversion this is just a little hack
    if supportcolumn1 in list(df):
      while supportcolumn1 in list(df):
        supportcolumn1 = supportcolumn1 + 'z'
    if supportcolumn2 in list(df):
      while supportcolumn2 in list(df):
        supportcolumn2 = supportcolumn2 + 'z'

    #so we'll convert supportcolumn1 to latt minutes, 
    df[supportcolumn1] = df[latt_column].abs() % 60
    
    #supportcolumn2 to latt degrees
    df[supportcolumn2] = (df[latt_column].abs() - df[supportcolumn1]) / 60
    
    #then combine everything into df[latt_column] as str CDDMM.MMMMMM where C is compass direction
    
    #first initialize latt_column with compass direction
    condition = df[latt_column].astype(float) < 0
    df = self.__autowhere(df, latt_column, condition, 'S', 'N')
    
    #now add the degrees characters
    #The following alternate frame was due to a backward compatibility halt that I couldn't identify by inspection so using a coersion
    # df[latt_column] = df[latt_column].astype(str) + df[supportcolumn2].astype(int).astype(str).str.zfill(2)
    df.loc[df[supportcolumn2]==df[supportcolumn2], latt_column] = df.loc[df[supportcolumn2]==df[supportcolumn2], latt_column].astype(str) + df.loc[df[supportcolumn2]==df[supportcolumn2], supportcolumn2].astype(int).astype(str).str.zfill(2)
    
    #now add the first two minutes characters
    #The following alternate frame was due to a backward compatibility halt that I couldn't identify by inspection so using a coersion
    # df[latt_column] = df[latt_column] + df[supportcolumn1].astype(int).astype(str).str.zfill(2)
    df.loc[df[supportcolumn1]==df[supportcolumn1], latt_column] = df.loc[df[supportcolumn1]==df[supportcolumn1], latt_column] + df.loc[df[supportcolumn1]==df[supportcolumn1], supportcolumn1].astype(int).astype(str).str.zfill(2)
    
    #now add the decimal
    df[latt_column] = df[latt_column] + '.'
    
    #now add the remaining decimal characters
    df[latt_column] = df[latt_column] + (df[supportcolumn1] % 1).astype(float).astype(str).str.slice_replace(start=0, stop=2, repl='').str.ljust(6, '0')
    
    #now trim extra characters
    df[latt_column] = df[latt_column].str.slice(0,13)
    
    #df[latt_column] is now in the form CDDMM.MMMMMM
    
    #now we'll use supportcolumn1 and supportcolumn2 to do the same for long_column
    
    #so we'll convert supportcolumn1 to latt minutes, 
    df[supportcolumn1] = df[long_column].abs() % 60
    
    #supportcolumn2 to latt degrees
    df[supportcolumn2] = (df[long_column].abs() - df[supportcolumn1]) / 60
    
    #then combine everything into df[latt_column] as str CDDMM.MMMMMM where C is compass direction
    
    #first initialize latt_column with compass direction
    condition = df[long_column].astype(float) < 0
    df = self.__autowhere(df, long_column, condition, 'W', 'E')
    
    #The following alternate frame was due to a backward compatibility halt that I couldn't identify by inspection so using a coersion
    #now add the degrees characters
    # df[long_column] = df[long_column].astype(str) + df[supportcolumn2].astype(int).astype(str).str.zfill(2)
    df.loc[df[supportcolumn2]==df[supportcolumn2], long_column] = df.loc[df[supportcolumn2]==df[supportcolumn2], long_column].astype(str) + df.loc[df[supportcolumn2]==df[supportcolumn2], supportcolumn2].astype(int).astype(str).str.zfill(2)
    
    #The following alternate frame was due to a backward compatibility halt that I couldn't identify by inspection so using a coersion
    #now add the first two minutes characters
    # df[long_column] = df[long_column] + df[supportcolumn1].astype(int).astype(str).str.zfill(2)
    df.loc[df[supportcolumn1]==df[supportcolumn1], long_column] = df.loc[df[supportcolumn1]==df[supportcolumn1], long_column] + df.loc[df[supportcolumn1]==df[supportcolumn1], supportcolumn1].astype(int).astype(str).str.zfill(2)
    
    #now add the decimal
    df[long_column] = df[long_column] + '.'
    
    #now add the remaining decimal characters
    df[long_column] = df[long_column] + (df[supportcolumn1] % 1).astype(float).astype(str).str.slice_replace(start=0, stop=2, repl='').str.ljust(6, '0')
    
    #now trim extra characters
    df[long_column] = df[long_column].str.slice(0,13)

    #now populate our inversion
    
    df[inputcolumn] = ',' * comma_addresses[0]
    
    df[inputcolumn] = df[inputcolumn] + df[latt_column].str.slice(1, 13)
    
    df[inputcolumn] = df[inputcolumn] + ',' * (comma_addresses[1] - comma_addresses[0])
    
    df[inputcolumn] = df[inputcolumn] + df[latt_column].str.slice(0, 1)
    
    df[inputcolumn] = df[inputcolumn] + ',' * (comma_addresses[2] - comma_addresses[1])
    
    df[inputcolumn] = df[inputcolumn] + df[long_column].str.slice(1, 13)
    
    df[inputcolumn] = df[inputcolumn] + ',' * (comma_addresses[3] - comma_addresses[2])
    
    df[inputcolumn] = df[inputcolumn] + df[long_column].str.slice(0, 1)
    
    remaining_commas = ',' * (comma_count - comma_addresses[3])
    
    df[inputcolumn] = df[inputcolumn] + remaining_commas
    
    #for the default configuration, this populates as ',,DDMM.MMMMMM,C,DDMM.MMMMMM,C,,,,,,,,,'
    
    del df[supportcolumn1]
    del df[supportcolumn2]

    return df

  def _inverseprocess_mlti(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mlti
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """

    normkey = categorylist[0]
    
    textlabelsdict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    inputtextcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']
    norm_column_dict_list = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_column_dict_list']
    norm_category = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_category']
    norm_params = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['norm_params']
    
    #since postprocess_dict won't have normalziation_dict saved in same place, we'll populate a mirror norm_postprocess_dict
    norm_postprocess_dict = {'process_dict' : postprocess_dict['process_dict'],
                             'column_dict' : {},
                             'traindata' : postprocess_dict['traindata'],
                             'printstatus' : postprocess_dict['printstatus']}

    for norm_column_dict in norm_column_dict_list:
      norm_postprocess_dict['column_dict'].update(norm_column_dict)
      
    origcolumn = postprocess_dict['column_dict'][categorylist[0]]['origcolumn']
    
    for categorylist_entry in categorylist:
      
      norm_categorylist = [categorylist_entry]
      
      if 'custom_inversion' in postprocess_dict['process_dict'][norm_category] \
      and callable(postprocess_dict['process_dict'][norm_category]['custom_inversion']):

        df, textcolumn = \
        self.__custom_inverseprocess_wrapper(df, norm_categorylist, norm_postprocess_dict)
      
      elif 'inverseprocess' in postprocess_dict['process_dict'][norm_category] \
      and callable(postprocess_dict['process_dict'][norm_category]['inverseprocess']):

        df, textcolumn = \
        postprocess_dict['process_dict'][norm_category]['inverseprocess'](df, norm_categorylist, norm_postprocess_dict)

      #edge case, if inversion not supported in the norm_category just apply a passthrough inversion
      else:

        df[textcolumns] = df[categorylist]
      
    #this gives us recovered columns in the form of textcolumns, now rename to inputtextcolumns
    
    #inverse_textlabelsdict maps {textcolumn : inputtextcolumn}
    inverse_textlabelsdict = {value:key for key,value in textlabelsdict.items()}
    
    df.rename(columns = inverse_textlabelsdict, inplace = True)
      
    #this returns an arbitrary one of the input columns which is fine
    inputcolumn = list(textlabelsdict)[0]
    
    return df, inputcolumn

  def _inverseprocess_cnsl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_cnsl
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    #translations are for column renaming purposes
    """

    normkey = categorylist[0]

    longest_len = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['longest_len']
    pad_character = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['pad_character']
    upstream_ordinal = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['upstream_ordinal']
    inputtextcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inputtextcolumns']

    for inputtextcolumn in inputtextcolumns:
      
      df[inputtextcolumn] = df[normkey].str.slice(start=0, stop=longest_len).str.replace(pad_character, '').copy()
      df[normkey] = df[normkey].str.slice(start=longest_len)

      if upstream_ordinal is True:
        df[inputtextcolumn] = df[inputtextcolumn].astype(int)
      
    #this returns an arbitrary one of the input columns which is fine
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    return df, inputcolumn

  def _custom_inversion_ordl(self, df, returnedcolumn_list, inputcolumn, normalization_dict):
    """
    #rewrite of the ordl inversion
    #corresponding to _custom_train_ordl
    """

    #First let's access the values we'll need from the normalization_dict
    ordinal_dict = normalization_dict['ordinal_dict']
    null_activation = normalization_dict['null_activation']
    
    #this includes the null_activation = 'Binary' case
    if null_activation is not False:
      inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
    elif null_activation is False:
      inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items() if key==key}
    
    returnedcolumn = returnedcolumn_list[0]
    
    #now perform the inversion
    df[inputcolumn] = df[returnedcolumn].astype('object').replace(inverse_ordinal_dict)

    return df

  def _inverseprocess_strg(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_strg 
    #converts strings back to integers
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = \
    df[normkey].astype(int, errors='ignore')
    
    return df, inputcolumn
  
  def _inverseprocess_bnry(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnry 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """

    normkey = categorylist[0]
    
    onevalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][1]
    zerovalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][0]

    if 'invert' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
      invert = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['invert']
    else:
      #backward compatibility preceding 8.18
      invert = False
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    if invert is False:
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == 1, onevalue, 0)
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == 0, zerovalue, specified='replacement')
    elif invert is True:
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == 1, zerovalue, 0)
      df = \
      self.__autowhere(df, inputcolumn, df[normkey] == 0, onevalue, specified='replacement')
      
    return df, inputcolumn

  def _custom_inversion_1010(self, df, returnedcolumn_list, inputcolumn, normalization_dict):
    """
    #rewrite of the 1010 trasnform
    #corresponding to _custom_train_1010
    """

    #First let's access the values we'll need from the normalization_dict
    null_activation = normalization_dict['null_activation']

    if 'inverse_binary_encoding_dict' in normalization_dict:
      inverse_binary_encoding_dict = normalization_dict['inverse_binary_encoding_dict']
    else:
      #backward compatibility preceding 8.26
      binary_encoding_dict = normalization_dict['binary_encoding_dict']
      if null_activation is not False:
        inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items()}
      elif null_activation is False:
        inverse_binary_encoding_dict = {value:key for key,value in binary_encoding_dict.items() if key==key}
    
    #first we'll aggregate all of the returned column activations into a single column string representation
    #note that returnedcolumn_list will be in order of increasing integers as was originally populated
    for returnedcolumn in returnedcolumn_list:
      
      if returnedcolumn == returnedcolumn_list[0]:
        df[inputcolumn] = df[returnedcolumn].astype(int).astype(str)
        
      else:
        df[inputcolumn] = df[inputcolumn] + df[returnedcolumn].astype(int).astype(str)
        
    if df[inputcolumn].dtype.name != 'object':
      df[inputcolumn] = df[inputcolumn].astype('object')
    
    #now perform the inversion
    df[inputcolumn] = df[inputcolumn].astype('object').replace(inverse_binary_encoding_dict)

    return df

  def _inverseprocess_splt(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sp1t 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_splt']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    df[inputcolumn] = np.nan
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_' + suffix + '_', '')

        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, overlap, specified='replacement')
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + '_' + suffix + '_', '')
        
        df = \
        self.__autowhere(df, inputcolumn, df[newcolumn] == 1, overlap, specified='replacement')

    return df, inputcolumn
  
  def _inverseprocess_spl2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_spl2 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    #also missing entries that didn't have any overlaps identified (returned as 0)
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    
    #returning zeros from inversion is counter to the convention used in other transforms
    #so returning as nan
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    consolidate_nonoverlaps = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['consolidate_nonoverlaps']
    
    df[inputcolumn] = df[normkey]
    
    if consolidate_nonoverlaps is True:

      df = \
      self.__autowhere(df, inputcolumn, df[inputcolumn] == '0', np.nan, specified='replacement')
  
    return df, inputcolumn

  def _inverseprocess_sp19(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sp19 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sp19']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    _1010_binary_encoding_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    _1010_binary_column_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
    _1010_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
    categorylist = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    for categorylist_entry in categorylist:
      
      if categorylist_entry == categorylist[0]:
        
        df[inputcolumn] = df[categorylist_entry].astype(int).astype(str)
        
      else:
        
        df[inputcolumn] = df[inputcolumn] + df[categorylist_entry].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)
    
    df[inputcolumn] = df[inputcolumn].str.replace('activations_', '', regex=False)
    
    #now let's extract the encoding
    i=0
    for newcolumn in newcolumns:
      
      df[newcolumn] = df[inputcolumn].str.slice(i,i+1).astype(np.int8)
      
      i+=1
    
    df[inputcolumn] = np.nan
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_sp15_', '')

        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, overlap, specified='replacement')
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + '_sp15_', '')
        
        df = \
        self.__autowhere(df, inputcolumn, df[newcolumn] == 1, overlap, specified='replacement')

    for newcolumn in newcolumns:
      
      del df[newcolumn]

    return df, inputcolumn

  def _inverseprocess_sbst(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbst 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbst']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

    df[inputcolumn] = np.nan
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_' + suffix + '_', '')

        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, overlap, specified='replacement')
    
    else:
      
      i = 0
      for column in preint_newcolumns:
        
        newcolumn = newcolumns[i]
        
        overlap = column.replace(inputcolumn + '_' + suffix + '_', '')
        
        df = \
        self.__autowhere(df, inputcolumn, df[newcolumn] == 1, overlap, specified='replacement')
        
        i += 1
    
    return df, inputcolumn

  def _inverseprocess_sbs3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbs3 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbs3']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    _1010_binary_encoding_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    _1010_binary_column_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
    _1010_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
    categorylist = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    for categorylist_entry in categorylist:
      
      if categorylist_entry == categorylist[0]:
        
        df[inputcolumn] = df[categorylist_entry].astype(int).astype(str)
        
      else:
        
        df[inputcolumn] = df[inputcolumn] + df[categorylist_entry].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)
    
    df[inputcolumn] = df[inputcolumn].str.replace('activations_', '', regex=False)
    
    #now let's extract the encoding
    i=0
    for newcolumn in newcolumns:
      
      df[newcolumn] = df[inputcolumn].str.slice(i,i+1).astype(np.int8)
      
      i+=1
    
    df[inputcolumn] = np.nan
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_sbst_', '')

        df = \
        self.__autowhere(df, inputcolumn, df[column] == 1, overlap, specified='replacement')
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + '_sbst_', '')
        
        df = \
        self.__autowhere(df, inputcolumn, df[newcolumn] == 1, overlap, specified='replacement')

    for newcolumn in newcolumns:
      
      del df[newcolumn]

    return df, inputcolumn
  
  def _inverseprocess_srch(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_srch 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without srch term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    search_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']
    
    df[inputcolumn] = np.nan
    
    #to match convention of prioritizing search parameter entries at end of list
    inverse_search = list(search_dict)
    inverse_search.reverse()
    
    for column in inverse_search:
      
      search = search_dict[column]
      
      if column in df.columns:

        df = \
        self.__autowhere(df, 
                        inputcolumn, 
                        ((df[column] == 1).astype(int) + (df[inputcolumn].isna()).astype(int)) == 2, 
                        search, 
                        specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_src2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src2 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['src2_newcolumns_src2']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    df[inputcolumn] = np.nan
    
    #to match convention of prioritizing search parameter entries at end of list
    newcolumns.reverse()
    
    for column in newcolumns:
      
      searchterm = column.replace(inputcolumn + '_' + suffix + '_', '')
      
      if column in df.columns:
      
        df = \
        self.__autowhere(df, 
                        inputcolumn, 
                        ((df[column] == 1).astype(int) + (df[inputcolumn].isna()).astype(int)) == 2, 
                        searchterm, 
                        specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_src3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src3 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src3']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    df[inputcolumn] = np.nan
    
    #to match convention of prioritizing search parameter entries at end of list
    newcolumns.reverse()
    
    for column in newcolumns:
      
      searchterm = column.replace(inputcolumn + '_' + suffix + '_', '')
      
      if column in df.columns:
      
        df = \
        self.__autowhere(df, 
                        inputcolumn, 
                        ((df[column] == 1).astype(int) + (df[inputcolumn].isna()).astype(int)) == 2, 
                        searchterm, 
                        specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_src4(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src4 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    ordl_dict1 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict1']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    df[inputcolumn] = np.nan
    
    #to match convention of prioritizing search parameter entries at end of list
    keys = list(ordl_dict1)
    keys.reverse()

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    for key in keys:
      
      searchterm = ordl_dict1[key].replace(inputcolumn + '_' + suffix + '_', '')
      
      df = \
      self.__autowhere(df, 
                      inputcolumn, 
                      ((df[normkey] == key).astype(int) + (df[inputcolumn].isna()).astype(int)) == 2, 
                      searchterm, 
                      specified='replacement')
    
    return df, inputcolumn
  
  def _inverseprocess_nmrc(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_nmrc 
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """

    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    
    df[inputcolumn] = np.nan
    
    for key in overlap_dict:
      
      extract = overlap_dict[key]
      
      df = \
      self.__autowhere(df, 
                      inputcolumn, 
                      ((df[normkey] == extract).astype(int) + (df[inputcolumn].isna()).astype(int)) == 2, 
                      key, 
                      specified='replacement')
    
    return df, inputcolumn

  def _inverseprocess_DPmc(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_DPmc
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    #translations are for column renaming purposes
    """

    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    textlabelsdict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict']
    
    #apply a passthrough inversion
    df[textcolumns] = df[categorylist]
      
    #this gives us recovered columns in the form of textcolumns, now rename to inputtextcolumns
    
    #inverse_textlabelsdict maps {textcolumn : inputtextcolumn}
    inverse_textlabelsdict = {value:key for key,value in textlabelsdict.items()}
    
    df.rename(columns = inverse_textlabelsdict, inplace = True)
      
    #this returns an arbitrary one of the input columns which is fine
    inputcolumn = list(textlabelsdict)[0]
    
    return df, inputcolumn

  def _inverseprocess_qbt1(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_qbt1 
    #returns floats
    #infill will be 0
    """

    normkey = categorylist[0]
    
    sign_columns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sign_columns']
    integer_columns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['integer_columns']
    fractional_columns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['fractional_columns']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    integer_bits = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['integer_bits']
    fractional_bits = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['fractional_bits']
    sign_bit = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sign_bit']
    if 'angle_bits' in postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]:
      angle_bits = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['angle_bits']
    else:
      #backward compatibility preceding 7.47
      angle_bits = False

    if angle_bits is True:
      for column in categorylist:
        #this translates entries of 0/pi to 0/1
        df[column] /= np.pi
        df[column] = df[column].astype(int)
    
    #sign_columns / integer_columns / fractional_columns
    #suffix / integer_bits / fractional_bits / sign_bit
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #initialize inputcolumn
    df.loc[:, inputcolumn] = 0
    
    #populate fractionals
    if fractional_bits > 0:
      for i, fractional in enumerate(range(fractional_bits)):
    
        fractional += 1
        fractional *= -1
        fractional = 2**fractional
        
        df[inputcolumn] += df[fractional_columns[i]] * fractional
        
    #populate integers
    if integer_bits > 0:
      for i, integer in enumerate(range(integer_bits-1, -1, -1)):
        
        df[inputcolumn] += df[integer_columns[i]] * 2**integer
        
    #apply sign conversion
    if sign_bit is True:
      
      df[inputcolumn] = df[inputcolumn] * ( (-1) ** df[sign_columns[0]])
      
    return df, inputcolumn

  def _custom_inversion_trig(self, df, returnedcolumn_list, inputcolumn, normalization_dict):
    """
    inversion fucntion in the custom_train convention
    corresponding to custom_train_trig
    built on top of numpy inverse trigometric operations np.sin, np.cos, np.tan, np.arcsin, np.arccos, np.arctan
    """

    #First let's access the values we'll need from the normalization_dict
    operation = normalization_dict['operation']

    #Now initialize the inputcolumn
    df.loc[:, inputcolumn] = 0

    #access the returned column from returnedcolumn_list which will have one entry
    returnedcolumn = returnedcolumn_list[0]
    
    if operation == 'sin':
      df[inputcolumn] = np.arcsin(df[returnedcolumn])
      
    if operation == 'cos':
      df[inputcolumn] = np.arccos(df[returnedcolumn])
      
    if operation == 'tan':
      df[inputcolumn] = np.arctan(df[returnedcolumn])
      
    if operation == 'arcsin':
      df[inputcolumn] = np.sin(df[returnedcolumn])
      
    if operation == 'arccos':
      df[inputcolumn] = np.cos(df[returnedcolumn])
      
    if operation == 'arctan':
      df[inputcolumn] = np.tan(df[returnedcolumn])

    return df

  #__FunctionBlock: master inversion functions
  
  def __df_inversion(self, categorylist_entry, df_test, postprocess_dict, inverse_categorytree, printstatus):
    """
    #support function for df_inversion_meta
    #this is where the inverseprocess functions are applied
    
    #where custom_inversion take precendence over inverseprocess
    """

    origcolumn = postprocess_dict['column_dict'][categorylist_entry]['origcolumn']
    category = postprocess_dict['column_dict'][categorylist_entry]['category']
    categorylist = postprocess_dict['column_dict'][categorylist_entry]['categorylist']

    #this defragments the dataframe
    df_test = df_test.copy()
    
    if 'custom_inversion' in postprocess_dict['process_dict'][category]:
      
      if callable(postprocess_dict['process_dict'][category]['custom_inversion']):
        
        df_test, inputcolumn = \
        self.__custom_inverseprocess_wrapper(df_test, categorylist, postprocess_dict)
    
    elif 'inverseprocess' in postprocess_dict['process_dict'][category]:
      
      if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
        
        df_test, inputcolumn = \
        postprocess_dict['process_dict'][category]['inverseprocess'](df_test, categorylist, postprocess_dict)
        #df, categorylist, postprocess_dict

    #we'll know to halt our inversion path when have successfully recovered the original target input column
    if inputcolumn != origcolumn:
      
      df_test, inputcolumn = \
      self.__df_inversion(inputcolumn, df_test, postprocess_dict, inverse_categorytree, printstatus)

    
    return df_test, inputcolumn
    
  def __df_inversion_meta(self, df_test, source_columns, postprocess_dict, printstatus, manual_path = False):
    """
    #Performs inversion of transformation sets
    #Relies on optional processdict entries of info_retention and inverseprocess
    #Uses the inverse_categorytree populated during automunge
    #For all entries associated with a single source column, 
    #Selects the returned categorylist with the lowest depth with info retention through the branch
    #If info retention not available, instead takes the shortest depth with inverse transformations available
    #If full set of inverse transfomations not available, does not perfgorm inversion for that source column
    #For each inversion path selected performs the sets of transfomrations in inverse order to the original path's population steps
    #All returned sets are added to the dataframe, and at conclusion of the function
    #any columns not matching a source column are removed form the set
    #also returns a list of the recovered columns
    #relies of column headers of received df_test matching the column headers of original columns returned from automunge
    #note that this function may be applied consistently to test or label sets
    """

    inverse_categorytree = postprocess_dict['inverse_categorytree']
    
    #we'll store the inversion paths info in this dictionary
    inversion_info_dict = {}
    
    #this will be a list of columns successfully recovered
    recovered_list = []
    
    for source_column in source_columns:
      
      # if printstatus is True:
      self.__autoprint('debug', "Evaluating inversion paths for columns derived from: ", source_column)
      
      returned_columns = postprocess_dict['origcolumn'][source_column]['columnkeylist'].copy()

      #account for excl suffix if applicable
      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(returned_columns, postprocess_dict['excl_suffix_conversion_dict'])

      returned_columns_clean = []

      #we'll just take one of the columns from each categorylist here if it is part of a multicolumn set
      if len(returned_columns) > 0:
        for returned_column in returned_columns:
          returned_column_categorylist = postprocess_dict['column_dict'][returned_column]['categorylist']
          if returned_column_categorylist[0] not in returned_columns_clean:
            returned_columns_clean.append(returned_column_categorylist[0])
      else:
        returned_columns_clean = []
      
      #initialize for ranking paths of transformation inversions
      path_depth_eval         = {}
      inforetention_eval      = {}
      transformavailable_eval = {}
      
      for returned_column in returned_columns_clean:
        
        #category as used here is the root category applied to the source column
        category = postprocess_dict['column_dict'][returned_column]['category']
        
        path_depth_eval.update({returned_column : inverse_categorytree[category][returned_column][4]})
        inforetention_eval.update({returned_column : inverse_categorytree[category][returned_column][5]})
        transformavailable_eval.update({returned_column : inverse_categorytree[category][returned_column][6]})
        
      #now invert the path_depth_eval dictionary for sorting
      #inverse_path_depth_eval = {value:key for key,value in path_depth_eval.items()}
      inverse_path_depth_eval = {}
      for key, value in path_depth_eval.items():
        if value not in inverse_path_depth_eval:
          inverse_path_depth_eval.update({value : [key]})
        else:
          inverse_path_depth_eval[value].append(key)
          
      #and sort it to find shortest depth
      inverse_path_depth_eval = dict(sorted(inverse_path_depth_eval.items()))
      
      #note that this sorted by depth method is based on a heuristic
      #that the fewest number of transforms will be the most efficient
      
      #for dense labels we aren't applying heuristic to select path
      #instead this will be run for each returned column in labels
      if manual_path is not False:

        path = manual_path
        transformavailable = transformavailable_eval[path]
        info_retention_marker = inforetention_eval[path]

        if transformavailable:
          best_path = path

        else:
          best_path = False
          info_retention_marker = False
          # if printstatus is True:
          self.__autoprint('debug', "No transformation path available from ", path)
          self.__autoprint('debug', '')
    
      else:
  
        #now let's select our returned column for the path
        best_path = False
        info_retention_marker = False

        for depth in inverse_path_depth_eval:

          for path in inverse_path_depth_eval[depth]:

            inforetention = inforetention_eval[path]
            transformavailable = transformavailable_eval[path]

            if inforetention and transformavailable:

              best_path = path
              info_retention_marker = True

              break

            elif transformavailable and best_path is False:

              best_path = path

          if info_retention_marker is True:

            break
          
      fullcategorylistforinversion = True
      if best_path is not False:
        #check that best path has all categorylist entries present
        #accounting for excl suffix if applicable
        df_test_list = list(df_test)
        self.__list_replace(df_test_list, postprocess_dict['excl_suffix_conversion_dict'])
        if not set(postprocess_dict['column_dict'][best_path]['categorylist']).issubset(set(df_test_list)):
          fullcategorylistforinversion = False
          # if printstatus is True:
          self.__autoprint('debug', "Inversion path selected based on returned column ", best_path)
          self.__autoprint('debug', "Inversion not available due to incomplete set of categorylist entries.")
          self.__autoprint('debug', "Please note that if entries are missing due to a Binary dimensionality reduction,")
          self.__autoprint('debug', "The column may still be recovered by applying a full test set inversion (inversion='test').")

          best_path = False
          
      # if printstatus is True:
        
      if best_path is not False:
        
        if info_retention_marker is True:
        
          self.__autoprint('debug', "Inversion path selected based on returned column ", best_path)
          self.__autoprint('debug', "With full recovery.")
          
        else:
          
          self.__autoprint('debug', "Inversion path selected based on returned column ", best_path)
          self.__autoprint('debug', "With partial recovery.")
        
      else:
        
        self.__autoprint('debug', "No inversion path available for source column: ", source_column)
        self.__autoprint('debug', '')
          
      #great we've selected our path for this source column's inversion
      inversion_info_dict.update({source_column : {'best_path' : best_path, \
                                                   'fullcategorylistforinversion' : fullcategorylistforinversion,
                                                   'info_retention' : info_retention_marker}})

      #great we've selected our path for this source column's inversion
      #we'll return that path in inversion_info_dict as False if no path, as string if path identified, 
      #or as True if privacy_encode is active and path identified
      reported_best_path = best_path
      if postprocess_dict['privacy_encode'] is True and reported_best_path is not False:
        reported_best_path = True

      inversion_info_dict.update({source_column : {'best_path' : reported_best_path, \
                                                   'fullcategorylistforinversion' : fullcategorylistforinversion,
                                                   'info_retention' : info_retention_marker}})
      
      #now let's apply our inversion transforms
      #if best_path is not False and callable(postprocess_dict['process_dict'][postprocess_dict['column_dict'][best_path]['category']]['inverseprocess']):
      if best_path is not False:
        
        columns_before_inversion = set(df_test)
        
        df_test, _1 = self.__df_inversion(best_path, df_test, postprocess_dict, inverse_categorytree, printstatus)

        #this defragments the dataframe
        df_test = df_test.copy()
        
        columns_after_inversion = set(df_test)
        
        #this gives, for the target source columns, returned columns, source columns, and intermediate columns
        full_returned_columns = columns_after_inversion - (columns_before_inversion - set(returned_columns))
        
      else:
        
        full_returned_columns = returned_columns
    
      for column in full_returned_columns:

        if column in source_columns \
        or 'protected_features' in postprocess_dict \
        and column in postprocess_dict['protected_features']:
          
          recovered_list += [column]

        else:

          #we're only retaining successfully recovered source columns in the returned df
          #this deletion is performed sequentially for columns returned from given source column for memory management
          if column in df_test.columns:
            del df_test[column]
          
      # if printstatus is True:        
      if best_path is not False:
        
        self.__autoprint('debug', "Recovered source column: ", source_column)
        self.__autoprint('debug', '')
        
    #special case for excl suffix
    for column in source_columns:
      if column not in recovered_list \
      and column in postprocess_dict['excl_columns_without_suffix'] \
      and (column in postprocess_dict['finalcolumns_train'] or column in postprocess_dict['finalcolumns_labels']):
        recovered_list.append(column)

    inversion_info_dict = {'pm_miscparameters_results' : logger_dict['validations'],
                           'inversion_paths' : inversion_info_dict}
    
    return df_test, recovered_list, inversion_info_dict

  def __inversion_parent(self, inversion, df_test, postprocess_dict, printstatus, \
                       pandasoutput, logger):

    if isinstance(inversion, str):
      inversion_orig = inversion
    elif isinstance(inversion, list):
      inversion_orig = self.__autocopy(inversion)

    inversion_PCA_valresult = False
    if inversion == 'test' and postprocess_dict['PCAmodel'] is not None and postprocess_dict['PCA_retain'] is not True:
      inversion_PCA_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "error: full test set inversion not currently supported with PCA.")
      self.__autoprint('warning', "user can pass partial list of columns to inversion parameter instead")
      self.__autoprint('warning', '')
      inversion = False
    logger_dict['validations'].update({'inversion_PCA_valresult' : inversion_PCA_valresult})
    logger = self.__logger_append(logger)

    inversion_setlist_privacyencode_valresult = False
    if postprocess_dict['privacy_encode'] is True and isinstance(inversion, (list, set)):
      inversion_setlist_privacyencode_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "inversion list or set specification not supported in conjunction with privacy_encode")
      self.__autoprint('warning', "inversion halted.")

      logger_dict['validations'].update({'inversion_setlist_privacyencode_valresult' : inversion_setlist_privacyencode_valresult})
      logger = self.__logger_append(logger)

      return
    logger_dict['validations'].update({'inversion_setlist_privacyencode_valresult' : inversion_setlist_privacyencode_valresult})
    logger = self.__logger_append(logger)

    #accomodate excl suffix convention by adding suffix back on
    if postprocess_dict['excl_suffix'] is False:
      df_test_columns = list(df_test)
      self.__list_replace(df_test_columns, postprocess_dict['excl_suffix_conversion_dict'])
      df_test.columns = df_test_columns

    #if PCA was applied with PCA_retain in inversion = 'test' scenario, we'll strike the PCA columns
    if inversion == 'test' and postprocess_dict['PCAmodel'] is not None and postprocess_dict['PCA_retain'] is True:
      for returned_PCA_column in postprocess_dict['returned_PCA_columns']:
        del df_test[returned_PCA_column]

    if isinstance(inversion, list):
      #convert list entries to string
      inversion = [str(entry) for entry in inversion]
      #accomodate excl suffix convention by adding suffix back on
      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(inversion, postprocess_dict['excl_suffix_conversion_dict'])
    
    #now consider whether Binary dimensionality reduction was performed
    
    if not isinstance(inversion, (list, set)) and inversion not in  {'labels', 'denselabels'}:
      meta_Binary_dict = postprocess_dict['Binary_dict']
    else:
      meta_Binary_dict = postprocess_dict['labels_Binary_dict']
    
    retain_Binary_present = False
    for key in meta_Binary_dict:
      if meta_Binary_dict[key]['Binary_specification'] in {'retain', 'ordinalretain', 'onehotretain'} \
      and len(list(meta_Binary_dict[key]['column_dict'])) > 0:
        retain_Binary_present = True
        
    replace_Binary_present = False
    for key in meta_Binary_dict:
      if meta_Binary_dict[key]['Binary_specification'] in {True, 'ordinal', 'onehot'} \
      and len(list(meta_Binary_dict[key]['column_dict'])) > 0:
        replace_Binary_present = True
        
    if inversion == 'test':
      
      if retain_Binary_present is True or replace_Binary_present is True:
        
        #for all cases where a Binary was performed, inversion = 'test' is translated to a list of columns
        inversion = list(df_test)

    #this is relevant for when feature importance dimensionality reduction was performed
    if inversion == 'test':
      if set(df_test).issubset(set(postprocess_dict['pre_dimred_finalcolumns_train'])):
        inversion = list(df_test)
        
    if inversion == 'test':

      finalcolumns_labels = list(df_test)

      #accomodate excl suffix convention by adding suffix back on
      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(finalcolumns_labels, postprocess_dict['excl_suffix_conversion_dict'])

      #confirm consistency of train an test sets
      validate_traintest_columnlabelscompare = False
      validate_traintest_columnorder = False

      #check number of columns is consistent
      if len(finalcolumns_labels)!= df_test.shape[1]:
        validate_traintest_columnlabelscompare = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error, different number of returned columns in train and test sets")

        logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
        logger = self.__logger_append(logger)

        return

      #check order of column headers are consistent
      columns_test = list(df_test)
      if set(finalcolumns_labels) == set(columns_test):
        if finalcolumns_labels != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return

      #this is for excl edge case again in case we had any updates to finalcolumns_labels above
      elif set(postprocess_dict['finalcolumns_train']) == set(columns_test):
        if postprocess_dict['finalcolumns_train'] != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return

      logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare,
                                        'validate_traintest_columnorder' : validate_traintest_columnorder})
      logger = self.__logger_append(logger)

      #assign labels to column headers if they weren't passed
      if finalcolumns_labels != columns_test:
        df_test.columns = finalcolumns_labels

      # if printstatus is True:
      self.__autoprint('debug', "Performing inversion recovery of original columns for test set.")
      self.__autoprint('debug', '')

      df_test, recovered_list, inversion_info_dict = \
      self.__df_inversion_meta(df_test, postprocess_dict['origtraincolumns'], postprocess_dict, printstatus)
      
      final_recovered_list = []
      recovered_set = set(recovered_list)

      #this puts recoverd columns in order as passed to automunge(.) in df_train
      for entry in postprocess_dict['origtraincolumns']:
        if entry in recovered_set:
          final_recovered_list.append(entry)

      #this matches order of columns as well as drops columns that weren't recovered
      df_test = df_test.reindex(columns=final_recovered_list)

      # if printstatus is True:
      self.__autoprint('info', "Inversion succeeded in recovering original form for columns:")
      self.__autoprint('info', final_recovered_list)
      self.__autoprint('info', '')

      inversion_info_dict['pm_miscparameters_results'].update(logger_dict['validations'])

      if pandasoutput is False:

        df_test = df_test.to_numpy()

      #final consolidation of logger_dict with any externally initialized logger diciotnary
      logger.update(logger_dict)

      return df_test, final_recovered_list, inversion_info_dict

    if inversion == 'labels':

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header
      finalcolumns_labels = postprocess_dict['finalcolumns_labels']

      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(finalcolumns_labels, postprocess_dict['excl_suffix_conversion_dict'])

      #confirm consistency of label sets
      validate_traintest_columnlabelscompare = False
      validate_traintest_columnorder = False

      #check number of columns is consistent
      if len(finalcolumns_labels)!= df_test.shape[1]:
        validate_traintest_columnlabelscompare = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error, different number of returned label columns in train and test sets")

        logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
        logger = self.__logger_append(logger)

        return

      #check order of column headers are consistent
      columns_test = list(df_test)
      if set(finalcolumns_labels) == set(columns_test):
        if finalcolumns_labels != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return
      #this is for excl edge case again in case we had any updates to finalcolumns_labels above
      elif set(postprocess_dict['finalcolumns_labels']) == set(columns_test):
        if postprocess_dict['finalcolumns_labels'] != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return

      logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare,
                                        'validate_traintest_columnorder' : validate_traintest_columnorder})
      logger = self.__logger_append(logger)

      #assign labels to column headers if they weren't passed
      if finalcolumns_labels != columns_test:
        df_test.columns = finalcolumns_labels
        
      #if there was a categoric consolidation address here
      if replace_Binary_present is True:

        # if printstatus is True:
        self.__autoprint('debug', "Recovering columns from Binary dimensionality reduction")
        self.__autoprint('debug', '')
          
      #for all cases where a Binary was performed, inversion = 'test' is translated to a list of columns
      inversion = finalcolumns_labels
          
      if retain_Binary_present is True or replace_Binary_present is True:

        df_test, inversion = \
        self.__masterBinaryinvert(df_test, inversion, inversion_orig, meta_Binary_dict, postprocess_dict, printstatus, logger)

      # if printstatus is True:
      self.__autoprint('debug', "Performing inversion recovery of original columns for label set.")
      self.__autoprint('debug', '')
        
      #_df_inversion_meta takes source columns as input
      inversion = \
      self.__column_convert_support(inversion, postprocess_dict, convert_to='input')

      df_test, recovered_list, inversion_info_dict = \
      self.__df_inversion_meta(df_test, inversion, postprocess_dict, printstatus)
      
      final_recovered_list = []
      recovered_set = set(recovered_list)

      #this puts recoverd columns in order as passed to automunge(.) in df_train
      for entry in postprocess_dict['labels_column_listofcolumns']:
        if entry in recovered_set:
          final_recovered_list.append(entry)

      #this matches order of columns as well as drops columns that weren't recovered
      df_test = df_test.reindex(columns=final_recovered_list)
      
      # if printstatus is True:
      self.__autoprint('info', "Inversion succeeded in recovering original form for columns:")
      self.__autoprint('info', final_recovered_list)
      self.__autoprint('info', '')

      inversion_info_dict['pm_miscparameters_results'].update(logger_dict['validations'])

      if pandasoutput is False:

        df_test = df_test.to_numpy()

      #final consolidation of logger_dict with any externally initialized logger diciotnary
      logger.update(logger_dict)

      return df_test, final_recovered_list, inversion_info_dict

    #denselabels is for case where downstream model simultaneously predicts multiple labels
    #such as when a label column presented in multiple configurations
    #in which case we'll return an inversion for each returned column for comparison
    #note that this implementation is a little inelligant, the printouts don't exaclty match, it works though
    if inversion == 'denselabels':

      #note denselabels only supported for single label case
      validate_denselabels_singlelabel = False
      if isinstance(postprocess_dict['labels_column'], list) and len(postprocess_dict['labels_column']) > 1:
        validate_denselabels_singlelabel = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error, inversion 'denselabels' option only supported for single label case, does not support consolidations.")

        logger_dict['validations'].update({'validate_denselabels_singlelabel' : validate_denselabels_singlelabel})
        logger = self.__logger_append(logger)

        return
      logger_dict['validations'].update({'validate_denselabels_singlelabel' : validate_denselabels_singlelabel})
      logger = self.__logger_append(logger)

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header
      finalcolumns_labels = postprocess_dict['finalcolumns_labels']

      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(finalcolumns_labels, postprocess_dict['excl_suffix_conversion_dict'])

      #confirm consistency of label sets
      validate_traintest_columnlabelscompare = False
      validate_traintest_columnorder = False

      #check number of columns is consistent
      if len(finalcolumns_labels)!= df_test.shape[1]:
        validate_traintest_columnlabelscompare = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error, different number of returned label columns in train and test sets")

        logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare})
        logger = self.__logger_append(logger)

        return

      #check order of column headers are consistent
      columns_test = list(df_test)
      if set(finalcolumns_labels) == set(columns_test):
        if finalcolumns_labels != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return
      #this is for excl edge case again in case we had any updates to finalcolumns_labels above
      elif set(postprocess_dict['finalcolumns_labels']) == set(columns_test):
        if postprocess_dict['finalcolumns_labels'] != columns_test:
          validate_traintest_columnorder = True
          # if printstatus != 'silent':
          self.__autoprint('warning', "error, different order of column labels in the train and test set")

          logger_dict['validations'].update({'validate_traintest_columnorder' : validate_traintest_columnorder})
          logger = self.__logger_append(logger)

          return

      logger_dict['validations'].update({'validate_traintest_columnlabelscompare' : validate_traintest_columnlabelscompare,
                                        'validate_traintest_columnorder' : validate_traintest_columnorder})
      logger = self.__logger_append(logger)

      #assign labels to column headers if they weren't passed
      if finalcolumns_labels != columns_test:
        df_test.columns = finalcolumns_labels

      # if printstatus is True:
      self.__autoprint('debug', "Performing inversion recovery of original columns for label set.")
      self.__autoprint('debug', '')
      
      dense_recovered_list = []
      recovered_categorylist = []
      
      j=0
      for denselabel_column in df_test:
        
        if denselabel_column not in recovered_categorylist:

          recovered_categorylist += postprocess_dict['column_dict'][denselabel_column]['categorylist']

          df_test_invertinput = df_test.copy()

          #df_test_denseinvert is a support dataframe with recovered column
          df_test_denseinvert, recovered_list, inversion_info_dict = \
          self.__df_inversion_meta(df_test_invertinput, [postprocess_dict['labels_column']], postprocess_dict, printstatus, \
                                 manual_path = denselabel_column)

          #since each inversion will return a different version of source column
          #we'll rename the returned source column as (source column) + '_' + (denselabel_column)
          if postprocess_dict['labels_column'] in df_test_denseinvert:

            if postprocess_dict['privacy_encode'] == 'private':
              denselabel_column = str(postprocess_dict['privacy_headers_labels_dict'][denselabel_column])

            #edge case for column overlap, just add '_z' character as suffix until no longer an overlap
            for i in range(111):
              if postprocess_dict['labels_column'] + '_' + denselabel_column in df_test:
                denselabel_column += '_z'
              else:
                break

            df_test_denseinvert.rename(columns = {postprocess_dict['labels_column'] : \
                                      postprocess_dict['labels_column'] + '_' + denselabel_column}, \
                                      inplace = True)

            dense_recovered_list += [postprocess_dict['labels_column'] + '_' + denselabel_column]

            df_test_denseinvert = pd.DataFrame(df_test_denseinvert)
            if j==0:
              df_test_denseinvert_final = pd.DataFrame(df_test_denseinvert.copy())
            else:
              df_test_denseinvert_final = pd.concat([df_test_denseinvert_final, df_test_denseinvert], axis=1)
            j+=1

      # if printstatus is True:
      self.__autoprint('info', "Inversion succeeded in recovering original form for columns:")
      self.__autoprint('info', dense_recovered_list)
      self.__autoprint('info', '')

      inversion_info_dict['pm_miscparameters_results'].update(logger_dict['validations'])

      if pandasoutput is False:

        df_test_denseinvert_final = df_test_denseinvert_final.to_numpy()

      #final consolidation of logger_dict with any externally initialized logger diciotnary
      logger.update(logger_dict)

      return df_test_denseinvert_final, dense_recovered_list, inversion_info_dict

    if isinstance(inversion, list):
      
      if replace_Binary_present is True:

        # if printstatus is True:
        self.__autoprint('debug', "Recovering columns from Binary dimensionality reduction")
        self.__autoprint('debug', '')
      
      df_test, inversion = \
      self.__masterBinaryinvert(df_test, inversion, inversion_orig, meta_Binary_dict, postprocess_dict, printstatus, logger)

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header

      #using pre_dimred_finalcolumns_train to accomodate post Binary inversion application
      # finalcolumns_train = postprocess_dict['finalcolumns_train']
      finalcolumns_train = postprocess_dict['pre_dimred_finalcolumns_train']
      
      source_columns = postprocess_dict['origtraincolumns']

      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(inversion, postprocess_dict['excl_suffix_conversion_dict'])
        self.__list_replace(finalcolumns_train, postprocess_dict['excl_suffix_conversion_dict'])

#         #for inversion need source columns
#         inversion = [postprocess_dict['column_dict'][entry]['origcolumn'] if entry in finalcolumns_train else entry for entry in inversion]

      #_df_inversion_meta takes source columns as input
      inversion = \
      self.__column_convert_support(inversion, postprocess_dict, convert_to='input')

      inversion_listentrycompare_valresult = False

      #check these are all valid source columns
      for entry in inversion:

        if entry not in source_columns:
          inversion_listentrycompare_valresult = True
          # if printstatus != 'silent':
          #(note this will trigger a printout if inversion passed as list targeting entry to label set, can be ignored)
          self.__autoprint('warning', "error: entry passed to inversion parameter list not matching a source or derived column")
          self.__autoprint('warning', "for entry: ", entry)

      logger_dict['validations'].update({'inversion_listentrycompare_valresult' : inversion_listentrycompare_valresult})
      logger = self.__logger_append(logger)

      df_test, recovered_list, inversion_info_dict = \
      self.__df_inversion_meta(df_test, inversion, postprocess_dict, printstatus)
      
      final_recovered_list = []
      recovered_set = set(recovered_list)

      #this puts recoverd columns in order as passed to automunge(.) in df_train
      for entry in postprocess_dict['origtraincolumns']:
        if entry in recovered_set:
          final_recovered_list.append(entry)

      #this matches order of columns as well as drops columns that weren't recovered
      df_test = df_test.reindex(columns=final_recovered_list)

      # if printstatus is True:
      self.__autoprint('info', "Inversion succeeded in recovering original form for columns:")
      self.__autoprint('info', final_recovered_list)
      self.__autoprint('info', '')

      inversion_info_dict['pm_miscparameters_results'].update(logger_dict['validations'])

      if pandasoutput is False:

        df_test = df_test.to_numpy()

      #final consolidation of logger_dict with any externally initialized logger diciotnary
      logger.update(logger_dict)

      return df_test, final_recovered_list, inversion_info_dict

    #the set case is for when user wants to specify a specific target for inversion path
    #and convention is set case accepts a single entry
    if isinstance(inversion, set):
      
      #convention is set case accepts single entry
      #so we'll just confirm
      inversion_setsingleentry_valresult = False
      if len(inversion) != 1:
        inversion_setsingleentry_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: inversion was passed as a set with more than one entry")
        self.__autoprint('warning', "the inversion set case is for specifying a single target inversion path")
        self.__autoprint('warning', "and thus accepts a set of one string entry representing a returned column header")
          
      logger_dict['validations'].update({'inversion_setsingleentry_valresult' : inversion_setsingleentry_valresult})
      logger = self.__logger_append(logger)

      #temporarily recast as a list
      inversion = list(inversion)
        
      #using pre_dimred_finalcolumns_train to accomodate post Binary inversion application
      # finalcolumns_train = postprocess_dict['finalcolumns_train']
      finalcolumns_train = postprocess_dict['pre_dimred_finalcolumns_train']
      
      #add excl suffix to passthrough columns if applicable
      if postprocess_dict['excl_suffix'] is False:
        self.__list_replace(inversion, postprocess_dict['excl_suffix_conversion_dict'])
        self.__list_replace(finalcolumns_train, postprocess_dict['excl_suffix_conversion_dict'])
        
      #now convert inversion to string entry (of returned column header)
      inversion = inversion[0]
      
      inversion_listentrycompare_valresult = False
      #confirm entry is a returned column header
      if inversion not in finalcolumns_train:
        inversion_listentrycompare_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "error: inversion was passed as a set with entry not matching one of the returned columns")
        self.__autoprint('warning', "the inversion set case is for specifying a single target inversion path")
        self.__autoprint('warning', "and thus accepts a set of one string entry representing a returned column header")
          
      logger_dict['validations'].update({'inversion_listentrycompare_valresult' : inversion_listentrycompare_valresult})
      logger = self.__logger_append(logger)

      origcolumn = postprocess_dict['column_dict'][inversion]['origcolumn']
      
      df_test, recovered_list, inversion_info_dict = \
      self.__df_inversion_meta(df_test, [origcolumn], postprocess_dict, printstatus, manual_path = inversion)
      
      final_recovered_list = []
      recovered_set = set(recovered_list)

      #this puts recoverd columns in order as passed to automunge(.) in df_train
      for entry in postprocess_dict['origtraincolumns']:
        if entry in recovered_set:
          final_recovered_list.append(entry)

      #this matches order of columns as well as drops columns that weren't recovered
      df_test = df_test.reindex(columns=final_recovered_list)

      # if printstatus is True:
      self.__autoprint('info', "Inversion succeeded in recovering original form for columns:")
      self.__autoprint('info', final_recovered_list)
      self.__autoprint('info', '')

      inversion_info_dict['pm_miscparameters_results'].update(logger_dict['validations'])

      if pandasoutput is False:

        df_test = df_test.to_numpy()

      #final consolidation of logger_dict with any externally initialized logger diciotnary
      logger.update(logger_dict)

      return df_test, final_recovered_list, inversion_info_dict

  #________________________________________________
  #04 - experimental model training Function Blocks
  #________________________________________________

  #__FunctionBlock: model training Function Blocks

  def __check_model_parameters(self, train, labels, postprocess_dict, 
                               ML_cmnd, encrypt_key,
                               printstatus, randomseed):
    """
    valdates parameters passed to automodel
    
    right now just checking types
    there are some further validations possible such as for the data structures
    could spot check a few entries present etc
    """
    
    model_validation_results = {}
    
    #check train
    train_valresult = False
    if not isinstance(train, type(pd.DataFrame())):
      train_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for train parameter.")
      self.__autoprint('warning', "Acceptable values are of type pandas dataframe")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'train_valresult' : train_valresult})
    
    #check labels
    labels_valresult = False
    if not isinstance(labels, type(pd.DataFrame())):
      labels_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for labels parameter.")
      self.__autoprint('warning', "Acceptable values are of type pandas dataframe")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'labels_valresult' : labels_valresult})
    
    #check postprocess_dict
    postprocess_dict_valresult = False
    if not isinstance(postprocess_dict, dict):
      postprocess_dict_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for postprocess_dict parameter.")
      self.__autoprint('warning', "Acceptable values are of type dict")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'postprocess_dict_valresult' : postprocess_dict_valresult})
    
    #check ML_cmnd
    ML_cmnd_valresult = False
    if not isinstance(ML_cmnd, dict) and ML_cmnd is not False:
      ML_cmnd_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for ML_cmnd parameter.")
      self.__autoprint('warning', "Acceptable values are of type dict or boolean False")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'ML_cmnd_valresult' : ML_cmnd_valresult})
    
    #check encrypt_key
    encrypt_key_valresult = False
    if not isinstance(encrypt_key, (int, bytes, bool)) \
    or isinstance(encrypt_key, int) and encrypt_key not in {False, 16, 24, 32} \
    or isinstance(encrypt_key, bytes) and len(encrypt_key) not in {16, 24, 32}:
      encrypt_key_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for encrypt_key parameter.")
      self.__autoprint('warning', "Acceptable values are one of {False, 16, 24, 32, or a bytes object}")
      self.__autoprint('warning', "When passed as a bytes object the length can be one of {16, 24, 32}")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'encrypt_key_valresult' : encrypt_key_valresult})
    
    #check printstatus
    printstatus_valresult = False
    if printstatus not in {True, False, 'summary', 'silent'} or \
    (printstatus in {True, False} and not isinstance(printstatus, bool)):
      printstatus_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for printstatus parameter.")
      self.__autoprint('warning', "Acceptable values are one of {True, False, 'summary', 'silent'}")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'printstatus_valresult' : printstatus_valresult})
    
    #check randomseed
    randomseed_valresult = False
    if not isinstance(randomseed, (int)) or randomseed is True:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**32-1 or False")
      self.__autoprint('warning', '')
    elif randomseed < 0 or randomseed > 2147483647:
      randomseed_valresult = True
      # if printstatus != 'silent':
      self.__autoprint('warning', "Error: invalid entry passed for randomseed parameter.")
      self.__autoprint('warning', "Acceptable values are integers within 0:2**31-1 or False")
      self.__autoprint('warning', '')
      
    model_validation_results.update({'randomseed_valresult' : randomseed_valresult})
    
    return model_validation_results

  def automodel(self, train, labels, postprocess_dict, 
                ML_cmnd = False, encrypt_key = False, 
                printstatus = True, randomseed = False):
    """
    ***first let's get this up and running then will revisit for encrypt_key considerations

    ***Please note this function is considered experimental / not fully audited
    
    uses output of automunge(.) (trian/labels) to train a model for inference
    returns postprocess_dict with additional entry 
    of a trained model for postmunge inference
    so that postmunge can apply inference internally to return predictions in label set
    
    ML_cmnd defaults to matching the basis in postprocess_dict
    however to use different model options or to use customML
    ML_cmnd should be reinitalized and passed to automodel
    In other words, ML_cmnd if specified will not inspect postprocess_dict['ML_cmnd']
    
    if postprocess_dict is recieved as encrypted
    the encrypt_key parameter accepts the key to decrypt if needed
    
    noise_augment can be applied with support of sampling parameters
    noise_augment / entropy_seeds / sampling_dict / random_generator
    
    printstatus and randomseed used as expected
    
    returns postprocess_dict with new entries for final_model, customML_inference_support_labels, model_validation_results
    
    #regarding postprocess_dict data structure
    #this function consolidates derived model_validation_results with postprocess_dict['miscparameters_results']
    #this function will populate and revert back to {} entry for customML_inference_support
    #this function returns new entries in 'final_model', include:
    #final_trained_model, final_model_ML_cmnd, final_model_autoMLer, final_model_categorylist, final_model_labelctgy
    #if a label Binary consolidation was present, this function will add a new column_dict entry 
    #for the final consolidated label set stored under the first returned consolidation column    

    Don't currently have logger_dict support for automodel
    """

    logger = {}
    
    # #printout display progress
    # # if printstatus in {True, 'summary'}:
    # self.__autoprint('info', "_______________")
    # self.__autoprint('info', "Begin AutoModel training")
    # self.__autoprint('info', "")
      
    #copy external dictionaries or lists to internal state
    for parameter in \
    [ML_cmnd, postprocess_dict]:
      if isinstance(parameter, (list, dict)):
        parameter = self.__autocopy(parameter)

    automodel_decode_valresult = False

    if 'encryption' in postprocess_dict \
    and postprocess_dict['encryption'] is True \
    and encrypt_key is not False:

      postprocess_dict, automodel_decode_valresult = \
      self.__decrypt_postprocess_dict(postprocess_dict, encrypt_key, printstatus)

    if randomseed is False:
      randomseed = int(np.random.Generator(np.random.PCG64()).integers(0, high=2147483647, size=1))
        
    #a few of the postprocess_dict entries may get overwritten for model training
    #we'll access the originals and revert at close
#     ML_cmnd_orig = self.__autocopy(postprocess_dict['ML_cmnd'])
#     autoMLer_orig = self.__autocopy(postprocess_dict['autoMLer'])
    #expected as an empty dictionary
    customML_inference_support_orig = \
    self.__autocopy(postprocess_dict['customML_inference_support'])
    
    #validate valid parameter values received
    model_validation_results = \
    self.__check_model_parameters(train, labels, postprocess_dict, 
                                 ML_cmnd, encrypt_key,
                                 printstatus, randomseed)

    model_validation_results.update({'automodel_decode_valresult' : automodel_decode_valresult})
    
    autoMLer = self.__autocopy(postprocess_dict['autoMLer'])
    
    #need to inspect ML_cmnd for cases where differs from ML_cmnd saved in postprocess_dict
    #if so will reinitialize
    if ML_cmnd is False:
      ML_cmnd = self.__autocopy(postprocess_dict['ML_cmnd'])

      #if autoMLtype is customML, the training functions won't have been saved in postprocess_dict
      autoML_type_customML_valresult = False
      if ML_cmnd['autoML_type'] == 'customML':
        autoML_type_customML_valresult = True
        # if printstatus != 'silent':
        self.__autoprint('warning', "warning of missing entry in postprocess_dict['ML_cmnd']['autoML_type']")
        self.__autoprint('warning', "when customML autoML_type is applied in automunge(.)")
        self.__autoprint('warning', "the training function is not retained") 
        self.__autoprint('warning', "so that it doesn't have to be reinitialized in a seperate notebook")
        self.__autoprint('warning', "to apply customML in this funciton please pass a training and inference function")
        self.__autoprint('warning', "through the ML_cmnd parameter")
        self.__autoprint('warning', "which accepts similar specifications to its use in automunge(.)")
        self.__autoprint('warning', '')

        autoMLer = self.__assemble_autoMLer()
        ML_cmnd['autoML_type'] = 'randomforest'

      model_validation_results.update({'autoML_type_customML_valresult' : autoML_type_customML_valresult})
      
    elif isinstance(ML_cmnd, dict):
      #this will initialize any missing entries for ML_cmnd
      #using the same defaults as were populated in automunge(.)
      ML_cmnd_valresult_2, ML_cmnd = \
      self.__check_ML_cmnd(ML_cmnd, printstatus)
      model_validation_results.update({'ML_cmnd_valresult_2' : ML_cmnd_valresult_2})

      #if new ML_cmnd has different autoML_type than that applied for ML infill
      #then will need to initialize a seperate autoMLer to access train and inference functions
      if ML_cmnd['autoML_type'] != postprocess_dict['ML_cmnd']['autoML_type']:
        autoMLer = self.__assemble_autoMLer()
      
#       #this will be reset to ML_cmnd_orig before return
#       postprocess_dict['ML_cmnd'] = ML_cmnd
      
    #We'll assume validation sets are performed in ML wrappers
    #for an alternate version that includes options for cross validation and tuning of encodings
    #an extension is planned as autotune that will be built on top of this function
    
    #__

    #we'll remove columns from ML_cmnd['full_exclude'] or with MLinfilltype == 'totalexclude'
    #using a simpler method than applied in ML infill for this purpose since we're only training one model

    full_exclude_specified = []
    if 'full_exclude' in ML_cmnd:
      full_exclude_specified = ML_cmnd['full_exclude']

    totalexclude_MLinfilltype = []
    for column_dict_entry in postprocess_dict['column_dict']:
      column_dict_entry_category = postprocess_dict['column_dict'][column_dict_entry]['category']
      column_dict_entry_category_MLinfilltype = postprocess_dict['process_dict'][column_dict_entry_category]['MLinfilltype']
      if column_dict_entry_category_MLinfilltype == 'totalexclude':
        totalexclude_MLinfilltype.append(column_dict_entry)

    nonnumeric_columns = full_exclude_specified + totalexclude_MLinfilltype
    #convert to returtned header format
    nonnumeric_columns = self.__column_convert_support(nonnumeric_columns, postprocess_dict, convert_to='returned')

    nonnumeric_columns = list(set(nonnumeric_columns) & set(train))

    #now drop any potentially nonnumeric columns
    train = train.drop(nonnumeric_columns, axis=1)

    #__
    
    process_dict = postprocess_dict['process_dict']
    
    #we'll assume train and labels set are already validated in __check_model_parameters
    
    #find origcateogry of labels from postprocess_dict
    labelcolumnkey = list(labels)[0]

    if labelcolumnkey in postprocess_dict['column_dict']:

      origcolumn = postprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
      origcategory = postprocess_dict['column_dict'][labelcolumnkey]['origcategory']

      #find labelctgy from process_dict based on this origcategory
      labelctgy = process_dict[origcategory]['labelctgy']

      categorylist = []

      for label_column in labels.columns:

        if label_column in postprocess_dict['column_dict'] \
        and postprocess_dict['column_dict'][label_column]['category'] == labelctgy:

          categorylist = postprocess_dict['column_dict'][label_column]['categorylist']

          #we'll follow convention that if target label category MLinfilltype is concurrent
          #we'll arbitrarily take the first column and use that as target
          if postprocess_dict['process_dict'][labelctgy]['MLinfilltype'] \
          in {'concurrent_act', 'concurrent_nmbr', 'concurrent_ordl'}:

            categorylist = [categorylist[0]]

          break
          
      #same as if labelcolumnkey not in postprocess_dict['column_dict']:
      #only other scenario for labels is column originated from categoric consolidation
      if len(postprocess_dict['labels_Binary_dict']) > 0:

        #for categoric consolidated labels, we are going to edit the column_dict and process_dict
        #as a hack to enable __trainFSmodel support
        #noting that in so doing there is a remote edge case of process_dict edit interfering with user overwrite to processdict
        #so we have some additional accomodation there as well

        label_column = postprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns'][0]

        categorylist = postprocess_dict['labels_Binary_dict'][0]['returned_Binary_columns']

        labelctgy = postprocess_dict['labels_Binary_dict'][0]['Btype']

        #since process_dict only returns entries applied for transformations, there is chance a Binary transform isn't included
        #so as part of the hack we'll add the associated Binary transform for purposes of inspecting the MLinfilltype
        #postprocess_dict['process_dict'] will only be inspected for MLinfilltype in __trainFSmodel and __shuffleaccuracy

        #in cases where labelctgy is already an entry in process_dict we'll add suffix until unique
        remote_overwrite_edge_case_marker = False
        if labelctgy in postprocess_dict['process_dict']:
          remote_overwrite_edge_case_marker = True

        if labelctgy == 'ord3':

          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in postprocess_dict['process_dict']:
              labelctgy += 'B'

          postprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'singlct'}})

        if labelctgy == 'onht':

          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in postprocess_dict['process_dict']:
              labelctgy += 'B'

          postprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : 'multirt'}})

        if labelctgy == '1010':

          if remote_overwrite_edge_case_marker is True:
            #this is in case user specified by chance the same string in processdict
            while labelctgy in postprocess_dict['process_dict']:
              labelctgy += 'B'

          postprocess_dict['process_dict'].update({labelctgy : {'MLinfilltype' : '1010'}})

        #this is a hack to support __trainFSmodel with categoric consolidations
        postprocess_dict['column_dict'].update({label_column : {'categorylist' : categorylist,
                                                                'columnslist' : list(labels), 
                                                                'category' : labelctgy,
                                                                'origcategory' : labelctgy}})

      if len(categorylist) == 0:
        # if printstatus != 'silent':
        #this is a remote edge case, printout added for troubleshooting support
        self.__autoprint('warning', "Label root category processdict entry contained a labelctgy entry not found in family tree")
        self.__autoprint('warning', "Model training will not run without valid labelgctgy processdict entry")
        self.__autoprint('warning', '')

        labelctgy_not_found_in_familytree_pm_valresult = True
        model_validation_results.update({'labelctgy_not_found_in_familytree_pm_valresult' : labelctgy_not_found_in_familytree_pm_valresult})

      elif len(categorylist) == 1:
        labels = pd.DataFrame(labels[categorylist[0]])

      else:
        labels = labels[categorylist]
      
      #validate that data is all valid numeric
      numeric_data_result, all_valid_entries_result = \
      self.__validate_allvalidnumeric(train, printstatus)

      model_validation_results.update({'numeric_data_result': numeric_data_result})
      model_validation_results.update({'all_valid_entries_result': all_valid_entries_result})
      
      #train the model
      model, postprocess_dict = \
      self.__trainFSmodel(train, labels, randomseed, \
                          process_dict, postprocess_dict, labelctgy, ML_cmnd, \
                          printstatus, logger, FSautoMLer=autoMLer)
      
      #we'll store the model and a few entrires to support inference
      #in postprocess_dict['final_model']
      
      postprocess_dict['final_model'] = {}
      postprocess_dict['final_model']['final_trained_model'] = model
      postprocess_dict['final_model']['final_model_ML_cmnd'] = self.__autocopy(ML_cmnd)
      postprocess_dict['final_model']['final_model_autoMLer'] = self.__autocopy(autoMLer)
      postprocess_dict['final_model']['final_model_categorylist'] = categorylist
      postprocess_dict['final_model']['final_model_labelctgy'] = labelctgy

      #___
      #some cleanups to returned data strucutes in postprocess_dict['final_model']
      #so don't have to reinitialize training functions
      #similar to what is done in automunge(.) with __autoMLer_cleanup

      if postprocess_dict['final_model']['final_model_ML_cmnd']['autoML_type'] == 'customML':
        #modeltarget is e.g. booleanclassification, ordinalclassification, etc
        for modeltarget in postprocess_dict['final_model']['final_model_autoMLer']['customML']:
          if 'train' in postprocess_dict['final_model']['final_model_autoMLer']['customML'][modeltarget]:
            del postprocess_dict['final_model']['final_model_autoMLer']['customML'][modeltarget]['train']

      #we'll also strike any populated customML training functions passed to ML_cmnd['customML']
      if 'customML' in postprocess_dict['final_model']['final_model_ML_cmnd']:
        if 'customML_Classifier_train' in postprocess_dict['final_model']['final_model_ML_cmnd']['customML']:
          postprocess_dict['final_model']['final_model_ML_cmnd']['customML']['customML_Classifier_train'] = True
        if 'customML_Regressor_train' in postprocess_dict['final_model']['final_model_ML_cmnd']['customML']:
          postprocess_dict['final_model']['final_model_ML_cmnd']['customML']['customML_Regressor_train'] = True
      #___

#       postprocess_dict['final_label_column'] = label_column
      
      #the inference functions inspect customML_inference_support in ML_cmnd
      #we'll have convention for postprocess_dict['customML_inference_support']
      #that for labels we'll store in postprocess_dict['customML_inference_support_labels']
      #and then transfer to ML_cmnd for any inference
      postprocess_dict['customML_inference_support_labels'] = \
      self.__autocopy(postprocess_dict['customML_inference_support'])
      
      #reset this to empty dictionary, is used to support some of data conversions in training/inference
      postprocess_dict['customML_inference_support'] = {}
      
#       postprocess_dict['ML_cmnd'] = ML_cmnd_orig
#       postprocess_dict['autoMLer'] = autoMLer_orig
#       #expected as an empty dictionary
#       postprocess_dict['customML_inference_support'] = customML_inference_support_orig
      
      #save validation results
      postprocess_dict['miscparameters_results'].update(model_validation_results)

      #if postprocess_dict was received as entrypted encrypt again before return
      #perform postprocess_dict encryption if elected by encrypt_key
      if encrypt_key is not False:
        postprocess_dict = \
        self.__encrypt_postprocess_dict(postprocess_dict, encrypt_key, postprocess_dict['privacy_encode'], printstatus)

    return postprocess_dict

  #__FunctionBlock: model inference Function Blocks

  def autoinference(self, test, postprocess_dict, encrypt_key = False, 
                    printstatus = True, randomseed = False):
    """
    ***Please note this function is considered experimental / not fully audited

    This function can be applied to test data returned from automugne or postmunge
    In conjunction with the postprocess_dict
    and runs inference on the model saved from automodel in postprocess_dictzs
    """

    if 'encryption' in postprocess_dict \
    and postprocess_dict['encryption'] is True \
    and encrypt_key is not False:

      postprocess_dict = self.__autocopy(postprocess_dict)

      postprocess_dict, decode_valresult = \
      self.__decrypt_postprocess_dict(postprocess_dict, encrypt_key, printstatus)
    
    if 'final_model' not in postprocess_dict or postprocess_dict['final_model'] is False:
      df_predictions = pd.DataFrame() 
    
    else:

      if randomseed is False:
        randomseed = int(np.random.Generator(np.random.PCG64()).integers(0, high=2147483647, size=1))
      
      #initialize a few variables accessed from postprocess_dict
      
      ML_cmnd = self.__autocopy(postprocess_dict['final_model']['final_model_ML_cmnd'])
      
      autoML_type = ML_cmnd['autoML_type']
      
      autoMLer = self.__autocopy(postprocess_dict['final_model']['final_model_autoMLer'])
      
      model = self.__autocopy(postprocess_dict['final_model']['final_trained_model'])
      
      ML_cmnd.update({'customML_inference_support' : postprocess_dict['customML_inference_support_labels']})
      
      label_categorylist = postprocess_dict['final_model']['final_model_categorylist']
      
      process_dict = postprocess_dict['process_dict']
      
      labelscategory = postprocess_dict['final_model']['final_model_labelctgy']
      
      #the type of inference operation is based on MLinfillitype of the labelscategory
      
      MLinfilltype = process_dict[labelscategory]['MLinfilltype']
      
      if MLinfilltype in {'numeric', 'concurrent_nmbr', 'integer'}:
        ML_application = 'regression'
      elif MLinfilltype in {'singlct', 'concurrent_ordl'}:
        ML_application = 'ordinalclassification'
      elif MLinfilltype in {'binary', 'concurrent_act'}:
        ML_application = 'booleanclassification'
      elif MLinfilltype in {'multirt', '1010'}:
        ML_application = 'onehotclassification'
        
      if MLinfilltype not in {'exclude', 'boolexclude', 'ordlexclude', 'totalexclude'}:

        #this produces inference predictions as a numpy array
        np_predictions = \
        autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, test, printstatus, label_categorylist)

        if MLinfilltype == '1010':

          df_predictions = \
          self.__convert_onehot_to_1010(df_predictions)

        #convert infill values to dataframe
        df_predictions = pd.DataFrame(np_predictions, columns = label_categorylist)

        if df_predictions.shape[0] == test.shape[0]:
          df_predictions.index = test.index
        #(else this inference was attempted without a trained model)
        
      else:
        
        df_predictions = pd.DataFrame() 
        
    return df_predictions
