# Module: Preprocess
# Author: Fahad Akbar <m.akbar@queensu.ca>
# License: MIT

import pandas as pd
import numpy as np
import ipywidgets as wg
from IPython.display import display
from ipywidgets import Layout
from sklearn.base import BaseEstimator, TransformerMixin, clone
from sklearn.impute._base import _BaseImputer
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MaxAbsScaler
from sklearn.preprocessing import PowerTransformer
from sklearn.preprocessing import QuantileTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.decomposition import PCA
from sklearn.decomposition import KernelPCA
from sklearn.cross_decomposition import PLSRegression
from sklearn.manifold import TSNE
from sklearn.decomposition import IncrementalPCA
from sklearn.preprocessing import KBinsDiscretizer
from pyod.models.knn import KNN
from pyod.models.iforest import IForest
from pyod.models.pca import PCA as PCA_od
from sklearn import cluster
from scipy import stats
from sklearn.ensemble import RandomForestClassifier as rfc
from sklearn.ensemble import RandomForestRegressor as rfr
from lightgbm import LGBMClassifier as lgbmc
from lightgbm import LGBMRegressor as lgbmr
import sys
import gc
from sklearn.pipeline import Pipeline
from sklearn import metrics
from datetime import datetime
import calendar
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from typing import Optional, Union
from pycaret.internal.logging import get_logger
from pycaret.internal.utils import infer_ml_usecase

from sklearn.utils.validation import check_is_fitted, check_random_state

pd.set_option("display.max_columns", 500)
pd.set_option("display.max_rows", 500)

SKLEARN_EMPTY_STEP = "passthrough"

# _____________________________________________________________________________________________________________________________


def str_if_not_null(x):
    if pd.isnull(x) or (x is None) or pd.isna(x) or (x is not x):
        return x
    return str(x)


def find_id_columns(data, target, numerical_features):
    # some times we have id column in the data set, we will try to find it and then  will drop it if found
    len_samples = len(data)
    id_columns = []
    for i in data.select_dtypes(
        include=["object", "int64", "float64", "float32"]
    ).columns:
        col = data[i]
        if i not in numerical_features and i != target:
            if sum(col.isnull()) == 0:
                try:
                    col = col.astype("int64")
                except:
                    continue
                if col.nunique() == len_samples:
                    # we extract column and sort it
                    features = col.sort_values()
                    # no we subtract i+1-th value from i-th (calculating increments)
                    increments = features.diff()[1:]
                    # if all increments are 1 (with float tolerance), then the column is ID column
                    if sum(np.abs(increments - 1) < 1e-7) == len_samples - 1:
                        id_columns.append(i)
    return id_columns


class DataTypes_Auto_infer(BaseEstimator, TransformerMixin):
    """
    - This will try to infer data types automatically, option to override learent data types is also available.
    - This alos automatically delets duplicate columns (values or same colume name), removes rows where target variable is null and 
      remove columns and rows where all the records are null
  """

    def __init__(
        self,
        target,
        ml_usecase,
        categorical_features=[],
        numerical_features=[],
        time_features=[],
        features_todrop=[],
        id_columns=[],
        display_types=True,
    ):  # nothing to define
        """
    User to define the target (y) variable
      args:
        target: string, name of the target variable
        ml_usecase: string , 'regresson' or 'classification . For now, only supports two  class classification
        - this is useful in case target variable is an object / string . it will replace the strings with integers
        categorical_features: list of categorical features, default None, when None best guess will be used to identify categorical features
        numerical_features: list of numerical features, default None, when None best guess will be used to identify numerical features
        time_features: list of date/time features, default None, when None best guess will be used to identify date/time features    
  """
        self.target = target
        self.ml_usecase = ml_usecase
        self.features_todrop = [str(x) for x in features_todrop]
        self.categorical_features = [
            x for x in categorical_features if x not in self.features_todrop
        ]
        self.numerical_features = [
            x for x in numerical_features if x not in self.features_todrop
        ]
        self.time_features = [x for x in time_features if x not in self.features_todrop]
        self.display_types = display_types
        self.id_columns = id_columns

    def fit(self, dataset, y=None):  # learning data types of all the columns
        """
    Args: 
      data: accepts a pandas data frame
    Returns:
      Panda Data Frame
    """

        data = dataset.copy()

        # also make sure that all the column names are string
        data.columns = [str(i) for i in data.columns]

        # drop any columns that were asked to drop
        data.drop(columns=self.features_todrop, errors="ignore", inplace=True)

        # remove sepcial char from column names
        # data.columns= data.columns.str.replace('[,]','')

        # we will take float as numberic, object as categorical from the begning
        # fir int64, we will check to see what is the proportion of unique counts to the total lenght of the data
        # if proportion is lower, then it is probabaly categorical
        # however, proportion can be lower / disturebed due to samller denominator (total lenghth / number of samples)
        # so we will take the following chart
        # 0-50 samples, threshold is 24%
        # 50-100 samples, th is 12%
        # 50-250 samples , th is 4.8%
        # 250-500 samples, th is 2.4%
        # 500 and above 2% or belwo

        # if there are inf or -inf then replace them with NaN
        data.replace([np.inf, -np.inf], np.NaN, inplace=True)

        # we canc check if somehow everything is object, we can try converting them in float
        for i in data.select_dtypes(include=["object"]).columns:
            try:
                data[i] = data[i].astype("int64")
            except:
                None

        for i in (
            data.select_dtypes(include=["object"])
            .drop(self.target, axis=1, errors="ignore")
            .columns
        ):
            try:
                data[i] = pd.to_datetime(
                    data[i], infer_datetime_format=True, utc=False, errors="raise"
                )
            except:
                continue

        # if data type is bool or pandas Categorical , convert to categorical
        for i in data.select_dtypes(include=["bool", "category"]).columns:
            data[i] = data[i].astype("object")

        # wiith csv , if we have any null in  a colum that was int , panda will read it as float.
        # so first we need to convert any such floats that have NaN and unique values are lower than 20
        for i in data.select_dtypes(include=["float64"]).columns:
            data[i] = data[i].astype("float32")
            # count how many Nas are there
            na_count = sum(data[i].isnull())
            # count how many digits are there that have decimiles
            count_float = np.nansum(
                [False if r.is_integer() else True for r in data[i]]
            )
            # total decimiels digits
            count_float = (
                count_float - na_count
            )  # reducing it because we know NaN is counted as a float digit
            # now if there isnt any float digit , & unique levales are less than 20 and there are Na's then convert it to object
            if (count_float == 0) & (data[i].nunique() <= 20) & (na_count > 0):
                data[i] = data[i].astype("object")

        # should really be an absolute number say 20
        # length = len(data.iloc[:,0])
        # if length in range(0,51):
        #   th=.25
        # elif length in range(51,101):
        #   th=.12
        # elif length in range(101,251):
        #   th=.048
        # elif length in range(251,501):
        #   th=.024
        # elif length > 500:
        #   th=.02

        # if column is int and unique counts are more than two, then: (exclude target)
        for i in data.select_dtypes(include=["int64"]).columns:
            if i != self.target:
                if data[i].nunique() <= 20:  # hard coded
                    data[i] = data[i].apply(str_if_not_null)
                else:
                    data[i] = data[i].astype("float32")
        # # if colum is objfloat  and only have two unique counts , this is probabaly one hot encoded
        # # make it object
        for i in data.select_dtypes(include=["float32"]).columns:
            if data[i].nunique() == 2:
                data[i] = data[i].apply(str_if_not_null)

        # for time & dates
        # self.drop_time = [] # for now we are deleting time columns

        # now in case we were given any specific columns dtypes in advance , we will over ride theos
        for i in self.categorical_features:
            try:
                data[i] = data[i].apply(str_if_not_null)
            except:
                data[i] = dataset[i].apply(str_if_not_null)

        for i in self.numerical_features:
            try:
                data[i] = data[i].astype("float32")
            except:
                data[i] = dataset[i].astype("float32")

        for i in self.time_features:
            try:
                data[i] = pd.to_datetime(
                    data[i], infer_datetime_format=True, utc=False, errors="raise"
                )
            except:
                data[i] = pd.to_datetime(
                    dataset[i], infer_datetime_format=True, utc=False, errors="raise"
                )

        for i in data.select_dtypes(
            include=["datetime64", "datetime64[ns, UTC]"]
        ).columns:
            data[i] = data[i].astype("datetime64[ns]")

        # table of learent types
        self.learned_dtypes = data.dtypes
        # self.training_columns = data.drop(self.target,axis=1).columns

        # if there are inf or -inf then replace them with NaN
        data = data.replace([np.inf, -np.inf], np.NaN).astype(self.learned_dtypes)

        # lets remove duplicates
        # remove duplicate columns (columns with same values)
        # (too expensive on bigger data sets)
        # data_c = data.T.drop_duplicates()
        # data = data_c.T
        # remove columns with duplicate name
        data = data.loc[:, ~data.columns.duplicated()]
        # Remove NAs
        data.dropna(axis=0, how="all", inplace=True)
        data.dropna(axis=1, how="all", inplace=True)
        # remove the row if target column has NA
        try:
            data.dropna(subset=[self.target], inplace=True)
        except KeyError:
            pass

        # self.training_columns = data.drop(self.target,axis=1).columns

        # since due to transpose , all data types have changed, lets change the dtypes to original---- not required any more since not transposing any more
        # for i in data.columns: # we are taking all the columns in test , so we dot have to worry about droping target column
        #   data[i] = data[i].astype(self.learned_dtypes[self.learned_dtypes.index==i])

        if self.display_types == True:
            display(
                wg.Text(
                    value="Following data types have been inferred automatically, if they are correct press enter to continue or type 'quit' otherwise.",
                    layout=Layout(width="100%"),
                ),
                display_id="m1",
            )

            dt_print_out = pd.DataFrame(
                self.learned_dtypes, columns=["Feature_Type"]
            ).drop("UNSUPERVISED_DUMMY_TARGET", errors="ignore")
            dt_print_out["Data Type"] = ""

            for i in dt_print_out.index:
                if i != self.target:
                    if i in self.id_columns:
                        dt_print_out.loc[i, "Data Type"] = "ID Column"
                    elif dt_print_out.loc[i, "Feature_Type"] == "object":
                        dt_print_out.loc[i, "Data Type"] = "Categorical"
                    elif dt_print_out.loc[i, "Feature_Type"] == "float32":
                        dt_print_out.loc[i, "Data Type"] = "Numeric"
                    elif dt_print_out.loc[i, "Feature_Type"] == "datetime64[ns]":
                        dt_print_out.loc[i, "Data Type"] = "Date"
                    # elif dt_print_out.loc[i,'Feature_Type'] == 'int64':
                    #  dt_print_out.loc[i,'Data Type'] = 'Categorical'
                else:
                    dt_print_out.loc[i, "Data Type"] = "Label"

            # if we added the dummy  target column , then drop it
            dt_print_out.drop(index="dummy_target", errors="ignore", inplace=True)

            display(dt_print_out[["Data Type"]])
            self.response = input()

            if self.response in [
                "quit",
                "Quit",
                "exit",
                "EXIT",
                "q",
                "Q",
                "e",
                "E",
                "QUIT",
                "Exit",
            ]:
                sys.exit(
                    "Read the documentation of setup to learn how to overwrite data types over the inferred types. setup function must run again before you continue modeling."
                )

        # drop time columns
        # data.drop(self.drop_time,axis=1,errors='ignore',inplace=True)

        # drop id columns
        data.drop(self.id_columns, axis=1, errors="ignore", inplace=True)

        return data

    def transform(self, dataset, y=None):
        """
      Args: 
        data: accepts a pandas data frame
      Returns:
        Panda Data Frame
    """

        data = dataset.copy()

        # also make sure that all the column names are string
        data.columns = [str(i) for i in data.columns]

        # drop any columns that were asked to drop
        data.drop(columns=self.features_todrop, errors="ignore", inplace=True)
        data = data[self.final_training_columns]

        # also make sure that all the column names are string
        data.columns = [str(i) for i in data.columns]

        # if there are inf or -inf then replace them with NaN
        data.replace([np.inf, -np.inf], np.NaN, inplace=True)

        try:
            data.dropna(subset=[self.target], inplace=True)
        except KeyError:
            pass

        # remove sepcial char from column names
        # data.columns= data.columns.str.replace('[,]','')

        # very first thing we need to so is to check if the training and test data hace same columns

        for i in self.final_training_columns:
            if i not in data.columns:
                raise TypeError(
                    f"test data does not have column {i} which was used for training."
                )

        # just keep picking the data and keep applying to the test data set (be mindful of target variable)
        for (
            i
        ) in (
            data.columns
        ):  # we are taking all the columns in test , so we dot have to worry about droping target column
            if i == self.target and (
                (self.ml_usecase == "classification")
                and (self.learned_dtypes[self.target] == "object")
            ):
                data[i] = self.le.transform(data[i].apply(str).astype("object"))
                data[i] = data[i].astype("int64")
            else:
                if self.learned_dtypes[i].name == "datetime64[ns]":
                    data[i] = pd.to_datetime(
                        data[i], infer_datetime_format=True, utc=False, errors="coerce"
                    )
                data[i] = data[i].astype(self.learned_dtypes[i])

        # drop time columns
        # data.drop(self.drop_time,axis=1,errors='ignore',inplace=True)

        # drop id columns
        data.drop(self.id_columns, axis=1, errors="ignore", inplace=True)

        return data

    # fit_transform
    def fit_transform(self, dataset, y=None):

        data = dataset

        # since this is for training , we dont nees any transformation since it has already been transformed in fit
        data = self.fit(data)

        # additionally we just need to treat the target variable
        # for ml use ase
        if (self.ml_usecase == "classification") & (
            data[self.target].dtype == "object"
        ):
            self.le = LabelEncoder()
            data[self.target] = self.le.fit_transform(
                data[self.target].apply(str).astype("object")
            )
            self.replacement = _get_labelencoder_reverse_dict(self.le)

            # self.u = list(pd.unique(data[self.target]))
            # self.replacement = np.arange(0,len(self.u))
            # data[self.target]= data[self.target].replace(self.u,self.replacement)
            # data[self.target] = data[self.target].astype('int64')
            # self.replacement = pd.DataFrame(dict(target_variable=self.u,replaced_with=self.replacement))

        # drop time columns
        # data.drop(self.drop_time,axis=1,errors='ignore',inplace=True)

        # drop id columns
        data.drop(self.id_columns, axis=1, errors="ignore", inplace=True)
        # finally save a list of columns that we would need from test data set
        self.final_training_columns = data.columns.to_list()
        self.final_training_columns.remove(self.target)

        return data


# _______________________________________________________________________________________________________________________
# Imputation


class Simple_Imputer(_BaseImputer):
    """
    Imputes all type of data (numerical,categorical & Time).
      Highly recommended to run Define_dataTypes class first
      Numerical values can be imputed with mean or median or filled with zeros
      categorical missing values will be replaced with "Other"
      Time values are imputed with the most frequesnt value
      Ignores target (y) variable    
      Args: 
        Numeric_strategy: string , all possible values {'mean','median','zero'}
        categorical_strategy: string , all possible values {'not_available','most frequent'}
        target: string , name of the target variable

  """

    _numeric_strategies = {
        "mean": "mean",
        "median": "median",
        "most frequent": "most_frequent",
        "most_frequent": "most_frequent",
        "zero": "constant",
    }
    _categorical_strategies = {
        "most frequent": "most_frequent",
        "most_frequent": "most_frequent",
        "not_available": "constant",
    }

    def __init__(
        self,
        numeric_strategy,
        categorical_strategy,
        target_variable,
        fill_value_numerical=0,
        fill_value_categorical="not_available",
    ):
        if numeric_strategy not in self._numeric_strategies:
            numeric_strategy = "zero"
        self.numeric_strategy = numeric_strategy
        self.target = target_variable
        if categorical_strategy not in self._categorical_strategies:
            categorical_strategy = "most_frequent"
        self.categorical_strategy = categorical_strategy
        self.numeric_imputer = SimpleImputer(
            strategy=self._numeric_strategies[self.numeric_strategy],
            fill_value=fill_value_numerical,
        )
        self.categorical_imputer = SimpleImputer(
            strategy=self._categorical_strategies[self.categorical_strategy],
            fill_value=fill_value_categorical,
        )
        self.most_frequent_time = []

    def fit(self, dataset, y=None):  #
        try:
            data = dataset.drop(self.target, axis=1)
        except:
            data = dataset
        self.numeric_columns = data.select_dtypes(include=["float32", "int64"]).columns
        self.categorical_columns = data.select_dtypes(include=["object"]).columns
        self.time_columns = data.select_dtypes(include=["datetime64[ns]"]).columns

        statistics = []

        if not self.numeric_columns.empty:
            self.numeric_imputer.fit(data[self.numeric_columns])
            statistics.append((self.numeric_imputer.statistics_, self.numeric_columns))
        if not self.categorical_columns.empty:
            self.categorical_imputer.fit(data[self.categorical_columns])
            statistics.append(
                (self.categorical_imputer.statistics_, self.categorical_columns)
            )
        if not self.time_columns.empty:
            self.most_frequent_time = []
            for col in self.time_columns:
                self.most_frequent_time.append(data[col].mode()[0])
            statistics.append((self.most_frequent_time, self.time_columns))

        self.statistics_ = np.zeros(shape=len(data.columns), dtype=object)
        columns = list(data.columns)
        for s, index in statistics:
            for i, j in enumerate(index):
                self.statistics_[columns.index(j)] = s[i]

        return

    def transform(self, dataset, y=None):
        data = dataset
        imputed_data = []
        if not self.numeric_columns.empty:
            numeric_data = pd.DataFrame(
                self.numeric_imputer.transform(data[self.numeric_columns]),
                columns=self.numeric_columns,
                index=data.index,
            )
            imputed_data.append(numeric_data)
        if not self.categorical_columns.empty:
            categorical_data = pd.DataFrame(
                self.categorical_imputer.transform(data[self.categorical_columns]),
                columns=self.categorical_columns,
                index=data.index,
            )
            for col in categorical_data.columns:
                categorical_data[col] = categorical_data[col].apply(str)
            imputed_data.append(categorical_data)
        if not self.time_columns.empty:
            time_data = data[self.time_columns]
            for i, col in enumerate(time_data.columns):
                time_data[col].fillna(self.most_frequent_time[i])
            imputed_data.append(time_data)

        if imputed_data:
            data.update(pd.concat(imputed_data, axis=1))
        data.astype(dataset.dtypes)

        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        self.fit(data)
        return self.transform(data)


# _______________________________________________________________________________________________________________________
# Imputation with surrogate columns
class Surrogate_Imputer(_BaseImputer):
    """
    Imputes feature with surrogate column (numerical,categorical & Time).
      - Highly recommended to run Define_dataTypes class first
      - it is also recommended to only apply this to features where it makes business sense to creat surrogate column
      - feature name has to be provided
      - only able to handle one feature at a time
      - Numerical values can be imputed with mean or median or filled with zeros
      - categorical missing values will be replaced with "Other"
      - Time values are imputed with the most frequesnt value
      - Ignores target (y) variable    
      Args: 
        feature_name: string, provide features name
        feature_type: string , all possible values {'numeric','categorical','date'}
        strategy: string ,all possible values {'mean','median','zero','not_available','most frequent'}
        target: string , name of the target variable

  """

    def __init__(self, numeric_strategy, categorical_strategy, target_variable):
        self.numeric_strategy = numeric_strategy
        self.target = target_variable
        self.categorical_strategy = categorical_strategy

    def fit(self, dataset, y=None):  #
        def zeros(x):
            return 0

        data = dataset
        # make a table for numerical variable with strategy stats
        if self.numeric_strategy == "mean":
            self.numeric_stats = (
                data.drop(self.target, axis=1)
                .select_dtypes(include=["float32", "int64"])
                .apply(np.nanmean)
            )
        elif self.numeric_strategy == "median":
            self.numeric_stats = (
                data.drop(self.target, axis=1)
                .select_dtypes(include=["float32", "int64"])
                .apply(np.nanmedian)
            )
        else:
            self.numeric_stats = (
                data.drop(self.target, axis=1)
                .select_dtypes(include=["float32", "int64"])
                .apply(zeros)
            )

        self.numeric_columns = (
            data.drop(self.target, axis=1)
            .select_dtypes(include=["float32", "int64"])
            .columns
        )
        # also need to learn if any columns had NA in training
        self.numeric_na = pd.DataFrame(columns=self.numeric_columns)
        for i in self.numeric_columns:
            if data[i].isnull().any() == True:
                self.numeric_na.loc[0, i] = True
            else:
                self.numeric_na.loc[0, i] = False

        # for Catgorical ,
        if self.categorical_strategy == "most frequent":
            self.categorical_columns = (
                data.drop(self.target, axis=1).select_dtypes(include=["object"]).columns
            )
            self.categorical_stats = pd.DataFrame(
                columns=self.categorical_columns
            )  # place holder
            for i in self.categorical_stats.columns:
                self.categorical_stats.loc[0, i] = data[i].value_counts().index[0]
            # also need to learn if any columns had NA in training, but this is only valid if strategy is "most frequent"
            self.categorical_na = pd.DataFrame(columns=self.categorical_columns)
            for i in self.categorical_columns:
                if sum(data[i].isnull()) > 0:
                    self.categorical_na.loc[0, i] = True
                else:
                    self.categorical_na.loc[0, i] = False
        else:
            self.categorical_columns = (
                data.drop(self.target, axis=1).select_dtypes(include=["object"]).columns
            )
            self.categorical_na = pd.DataFrame(columns=self.categorical_columns)
            self.categorical_na.loc[
                0, :
            ] = False  # (in this situation we are not making any surrogate column)

        # for time, there is only one way, pick up the most frequent one
        self.time_columns = (
            data.drop(self.target, axis=1)
            .select_dtypes(include=["datetime64[ns]"])
            .columns
        )
        self.time_stats = pd.DataFrame(columns=self.time_columns)  # place holder
        self.time_na = pd.DataFrame(columns=self.time_columns)
        for i in self.time_columns:
            self.time_stats.loc[0, i] = data[i].value_counts().index[0]

        # learn if time columns were NA
        for i in self.time_columns:
            if data[i].isnull().any() == True:
                self.time_na.loc[0, i] = True
            else:
                self.time_na.loc[0, i] = False

        return data  # nothing to return

    def transform(self, dataset, y=None):
        data = dataset
        # for numeric columns
        for i, s in zip(data[self.numeric_columns].columns, self.numeric_stats):
            array = data[i].isnull()
            data[i].fillna(s, inplace=True)
            # make a surrogate column if there was any
            if self.numeric_na.loc[0, i] == True:
                data[i + "_surrogate"] = array
                # make it string
                data[i + "_surrogate"] = data[i + "_surrogate"].apply(str)

        # for categorical columns
        if self.categorical_strategy == "most frequent":
            for i in self.categorical_stats.columns:
                # data[i].fillna(self.categorical_stats.loc[0,i],inplace=True)
                array = data[i].isnull()
                data[i] = data[i].fillna(self.categorical_stats.loc[0, i])
                data[i] = data[i].apply(str)
                # make surrogate column
                if self.categorical_na.loc[0, i] == True:
                    data[i + "_surrogate"] = array
                    # make it string
                    data[i + "_surrogate"] = data[i + "_surrogate"].apply(str)
        else:  # this means replace na with "not_available"
            for i in self.categorical_columns:
                data[i].fillna("not_available", inplace=True)
                data[i] = data[i].apply(str)
                # no need to make surrogate since not_available is itself a new colum

        # for time
        for i in self.time_stats.columns:
            array = data[i].isnull()
            data[i].fillna(self.time_stats.loc[0, i], inplace=True)
            # make surrogate column
            if self.time_na.loc[0, i] == True:
                data[i + "_surrogate"] = array
                # make it string
                data[i + "_surrogate"] = data[i + "_surrogate"].apply(str)

        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        data = self.fit(data)
        return self.transform(data)


class Iterative_Imputer(_BaseImputer):
    def __init__(
        self,
        regressor: BaseEstimator,
        classifier: BaseEstimator,
        *,
        target=None,
        missing_values=np.nan,
        initial_strategy_numeric: str = "mean",
        initial_strategy_categorical: str = "most_frequent",
        ordinal_columns: Optional[list] = None,
        max_iter: int = 10,
        warm_start: bool = False,
        imputation_order: str = "ascending",
        verbose: int = 0,
        random_state: int = None,
        add_indicator: bool = False,
    ):
        super().__init__(missing_values=missing_values, add_indicator=add_indicator)

        self.regressor = regressor
        self.classifier = classifier
        self.initial_strategy_numeric = initial_strategy_numeric
        self.initial_strategy_categorical = initial_strategy_categorical
        self.max_iter = max_iter
        self.warm_start = warm_start
        self.imputation_order = imputation_order
        self.verbose = verbose
        self.random_state = random_state
        self.target = target
        if ordinal_columns is None:
            ordinal_columns = []
        self.ordinal_columns = list(ordinal_columns)
        self._column_cleaner = Clean_Colum_Names()

    def _initial_imputation(self, X):
        if self.initial_imputer_ is None:
            self.initial_imputer_ = Simple_Imputer(
                target_variable="__TARGET__",  # dummy value, we don't actually want to drop anything
                numeric_strategy=self.initial_strategy_numeric,
                categorical_strategy=self.initial_strategy_categorical,
            )
            X_filled = self.initial_imputer_.fit_transform(X)
        else:
            X_filled = self.initial_imputer_.transform(X)

        return X_filled

    def _impute_one_feature(self, X, column, X_na_mask, fit):
        if not fit:
            check_is_fitted(self)
        is_classification = (
            X[column].dtype.name == "object" or column in self.ordinal_columns
        )
        if is_classification:
            if column in self.classifiers_:
                time, dummy, le, estimator = self.classifiers_[column]
            elif not fit:
                return X
            else:
                estimator = clone(self._classifier)
                time = Make_Time_Features()
                dummy = Dummify(column)
                le = LabelEncoder()
        else:
            if column in self.regressors_:
                time, dummy, le, estimator = self.regressors_[column]
            elif not fit:
                return X
            else:
                estimator = clone(self._regressor)
                time = Make_Time_Features()
                dummy = Dummify(column)
                le = None

        if fit:
            fit_kwargs = {}
            X_train = X[~X_na_mask[column]]
            y_train = X_train[column]
            # catboost handles categoricals itself
            if "catboost" not in str(type(estimator)).lower():
                X_train = time.fit_transform(X_train)
                X_train = dummy.fit_transform(X_train)
                X_train.drop(column, axis=1, inplace=True)
            else:
                X_train.drop(column, axis=1, inplace=True)
                fit_kwargs["cat_features"] = []
                for i, col in enumerate(X_train.columns):
                    if X_train[col].dtype.name == "object":
                        X_train[col] = pd.Categorical(
                            X_train[col], ordered=column in self.ordinal_columns
                        )
                        fit_kwargs["cat_features"].append(i)
                fit_kwargs["cat_features"] = np.array(
                    fit_kwargs["cat_features"], dtype=int
                )
            X_train = self._column_cleaner.fit_transform(X_train)

            if le:
                y_train = le.fit_transform(y_train)

            try:
                assert self.warm_start
                estimator.partial_fit(X_train, y_train)
            except:
                estimator.fit(X_train, y_train, **fit_kwargs)

        X_test = X.drop(column, axis=1)[X_na_mask[column]]
        X_test = time.transform(X_test)
        # catboost handles categoricals itself
        if "catboost" not in str(type(estimator)).lower():
            X_test = dummy.transform(X_test)
        else:
            for col in X_test.select_dtypes("object").columns:
                X_test[col] = pd.Categorical(
                    X_test[col], ordered=column in self.ordinal_columns
                )
        result = estimator.predict(X_test)
        if le:
            result = le.inverse_transform(result)

        if fit:
            if is_classification:
                self.classifiers_[column] = (time, dummy, le, estimator)
            else:
                self.regressors_[column] = (time, dummy, le, estimator)

        if result.dtype.name == "float64":
            result = result.astype("float32")

        X_test[column] = result
        X.update(X_test[column])

        gc.collect()

        return X

    def _impute(self, X, fit: bool):
        if self.target in X.columns:
            target_column = X[self.target]
            X = X.drop(self.target, axis=1)
        else:
            target_column = None

        original_columns = X.columns
        original_index = X.index

        X = X.reset_index(drop=True)
        X = self._column_cleaner.fit_transform(X)

        self.imputation_sequence_ = (
            X.isnull().sum().sort_values(ascending=self.imputation_order == "ascending")
        )
        self.imputation_sequence_ = [
            col
            for col in self.imputation_sequence_[self.imputation_sequence_ > 0].index
            if X[col].dtype.name != "datetime64[ns]"
        ]

        X_na_mask = X.isnull()

        X_imputed = self._initial_imputation(X.copy())

        for i in range(self.max_iter if fit else 1):
            for feature in self.imputation_sequence_:
                get_logger().info(f"Iterative Imputation: {i+1} cycle | {feature}")
                X_imputed = self._impute_one_feature(X_imputed, feature, X_na_mask, fit)

        X_imputed.columns = original_columns
        X_imputed.index = original_index

        if target_column is not None:
            X_imputed[self.target] = target_column
        return X_imputed

    def transform(self, X, y=None, **fit_params):
        return self._impute(X, fit=False)

    def fit_transform(self, X, y=None, **fit_params):
        self.random_state_ = getattr(
            self, "random_state_", check_random_state(self.random_state)
        )
        if self.regressor is None:
            raise ValueError("No regressor provided")
        else:
            self._regressor = clone(self.regressor)
        try:
            self._regressor.set_param(random_state=self.random_state_)
        except:
            pass
        if self.classifier is None:
            raise ValueError("No classifier provided")
        else:
            self._classifier = clone(self.classifier)
        try:
            self._classifier.set_param(random_state=self.random_state_)
        except:
            pass

        self.classifiers_ = {}
        self.regressors_ = {}

        self.initial_imputer_ = None

        return self._impute(X, fit=True)

    def fit(self, X, y=None, **fit_params):
        self.fit_transform(X, y=y, **fit_params)

        return self


# _______________________________________________________________________________________________________________________
# Zero and Near Zero Variance
class Zroe_NearZero_Variance(BaseEstimator, TransformerMixin):
    """
    - it eliminates the features having zero variance
    - it eliminates the features haveing near zero variance
    - Near zero variance is determined by 
      -1) Count of unique points divided by the total length of the feature has to be lower than a pre sepcified threshold 
      -2) Most common point(count) divided by the second most common point(count) in the feature is greater than a pre specified threshold
      Once both conditions are met , the feature is dropped  
    -Ignores target variable
      
      Args: 
        threshold_1: float (between 0.0 to 1.0) , default is .10 
        threshold_2: int (between 1 to 100), default is 20 
        tatget variable : string, name of the target variable

  """

    def __init__(self, target, threshold_1=0.1, threshold_2=20):
        self.threshold_1 = threshold_1
        self.threshold_2 = threshold_2
        self.target = target

    def fit(
        self, dataset, y=None
    ):  # from training data set we are going to learn what columns to drop
        data = dataset
        self.to_drop = []
        sampl_len = len(data[self.target])
        for i in data.drop(self.target, axis=1).columns:
            # get the number of unique counts
            u = pd.DataFrame(data[i].value_counts()).sort_values(
                by=i, ascending=False, inplace=False
            )
            # take len of u and divided it by the total sample numbers, so this will check the 1st rule , has to be low say 10%
            # import pdb; pdb.set_trace()
            first = len(u) / sampl_len
            # then check if most common divided by 2nd most common ratio is 20 or more
            if (
                len(u[i]) == 1
            ):  # this means that if column is non variance , automatically make the number big to drop it
                second = 100
            else:
                second = u.iloc[0, 0] / u.iloc[1, 0]
            # if both conditions are true then drop the column, however, we dont want to alter column that indicate NA's
            if (first <= 0.10) and (second >= 20) and (i[-10:] != "_surrogate"):
                self.to_drop.append(i)
            # now drop if the column has zero variance
            if (second == 100) and (i[-10:] != "_surrogate"):
                self.to_drop.append(i)

    def transform(
        self, dataset, y=None
    ):  # since it is only for training data set , nothing here
        data = dataset.drop(self.to_drop, axis=1)
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        self.fit(data)
        return self.transform(data)


# ____________________________________________________________________________________________________________________________
# rare catagorical variables
class Catagorical_variables_With_Rare_levels(BaseEstimator, TransformerMixin):
    """
    -Merges levels in catagorical features with more frequent level  if they appear less than a threshold count 
      e.g. Col=[a,a,a,a,b,b,c,c]
      if threshold is set to 2 , then c will be mrged with b because both are below threshold
      There has to be atleast two levels belwo threshold for this to work 
      the process will keep going until all the levels have atleast 2(threshold) counts
    -Only handles catagorical features
    -It is recommended to run the Zroe_NearZero_Variance and Define_dataTypes first
    -Ignores target variable 
      Args: 
        threshold: int , default 10
        target: string , name of the target variable
        new_level_name: string , name given to the new level generated, default 'others'

  """

    def __init__(self, target, new_level_name="others_infrequent", threshold=0.05):
        self.threshold = threshold
        self.target = target
        self.new_level_name = new_level_name

    def fit(
        self, dataset, y=None
    ):  # we will learn for what columnns what are the level to merge as others
        # every level of the catagorical feature has to be more than threshols, if not they will be clubed togather as "others"
        # in order to apply, there should be atleast two levels belwo the threshold !
        # creat a place holder
        data = dataset
        self.ph = pd.DataFrame(
            columns=data.drop(self.target, axis=1)
            .select_dtypes(include="object")
            .columns
        )
        # ph.columns = df.columns# catagorical only
        for i in data[self.ph.columns].columns:
            # determine the infrequebt count
            v_c = data[i].value_counts()
            count_th = round(v_c.quantile(self.threshold))
            a = np.sum(
                pd.DataFrame(data[i].value_counts().sort_values())[i] <= count_th
            )
            if a >= 2:  # rare levels has to be atleast two
                count = pd.DataFrame(data[i].value_counts().sort_values())
                count.columns = ["fre"]
                count = count[count["fre"] <= count_th]
                to_club = list(count.index)
                self.ph.loc[0, i] = to_club
            else:
                self.ph.loc[0, i] = []
        # # also need to make a place holder that keep records of all the levels , and in case a new level appears in test we will change it to others
        # self.ph_level = pd.DataFrame(columns=data.drop(self.target,axis=1).select_dtypes(include="object").columns)
        # for i in self.ph_level.columns:
        #   self.ph_level.loc[0,i] = list(data[i].value_counts().sort_values().index)

    def transform(self, dataset, y=None):  #
        # transorm
        data = dataset
        for i in data[self.ph.columns].columns:
            t_replace = self.ph.loc[0, i]
            data[i].replace(
                to_replace=t_replace, value=self.new_level_name, inplace=True
            )
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        self.fit(data)
        return self.transform(data)


# _______________________________________________________________________________________________________________________
# new catagorical level in test
class New_Catagorical_Levels_in_TestData(BaseEstimator, TransformerMixin):
    """
    -This treats if a new level appears in the test dataset catagorical's feature (i.e a level on whihc model was not trained previously) 
    -It simply replaces the new level in test data set with the most frequent or least frequent level in the same feature in the training data set
    -It is recommended to run the Zroe_NearZero_Variance and Define_dataTypes first
    -Ignores target variable 
      Args: 
        target: string , name of the target variable
        replacement_strategy:string , 'raise exception', 'least frequent' or 'most frequent' (default 'most frequent' )

  """

    def __init__(self, target, replacement_strategy="most frequent"):
        self.target = target
        self.replacement_strategy = replacement_strategy

    def fit(self, data, y=None):
        # need to make a place holder that keep records of all the levels , and in case a new level appears in test we will change it to others
        self.ph_train_level = pd.DataFrame(
            columns=data.drop(self.target, axis=1)
            .select_dtypes(include="object")
            .columns
        )
        for i in self.ph_train_level.columns:
            if self.replacement_strategy == "least frequent":
                self.ph_train_level.loc[0, i] = list(
                    data[i].value_counts().sort_values().index
                )
            else:
                self.ph_train_level.loc[0, i] = list(data[i].value_counts().index)

    def transform(self, data, y=None):  #
        # transorm
        # we need to learn the same for test data , and then we will compare to check what levels are new in there
        self.ph_test_level = pd.DataFrame(
            columns=data.drop(self.target, axis=1, errors="ignore")
            .select_dtypes(include="object")
            .columns
        )
        for i in self.ph_test_level.columns:
            self.ph_test_level.loc[0, i] = list(
                data[i].value_counts().sort_values().index
            )

        # new we have levels for both test and train, we will start comparing and replacing levels in test set (Only if test set has new levels)
        for i in self.ph_test_level.columns:
            new = list(
                (set(self.ph_test_level.loc[0, i]) - set(self.ph_train_level.loc[0, i]))
            )
            # now if there is a difference , only then replace it
            if len(new) > 0:
                if self.replacement_strategy == "raise exception":
                    raise ValueError(
                        f"Column '{i}' contains levels '{new}' which were not present in train data."
                    )
                data[i].replace(new, self.ph_train_level.loc[0, i][0], inplace=True)

        return data

    def fit_transform(
        self, data, y=None
    ):  # There is no transformation happening in training data set, its all about test
        self.fit(data)
        return data


# _______________________________________________________________________________________________________________________
# Group akin features
class Group_Similar_Features(BaseEstimator, TransformerMixin):
    """
    - Given a list of features , it creates aggregate features 
    - features created are Min, Max, Mean, Median, Mode & Std
    - Only works on numerical features
      Args: 
        list_of_similar_features: list of list, string , e.g. [['col',col2],['col3','col4']]
        group_name: list, group name/names to be added as prefix to aggregate features, e.g ['gorup1','group2']
  """

    def __init__(self, group_name=[], list_of_grouped_features=[[]]):
        self.list_of_similar_features = list_of_grouped_features
        self.group_name = group_name
        # if list of list not given
        try:
            np.array(self.list_of_similar_features).shape[0]
        except:
            raise (
                "Group_Similar_Features: list_of_grouped_features is not provided as list of list"
            )

    def fit(self, data, y=None):
        # nothing to learn
        return self

    def transform(self, dataset, y=None):
        data = dataset
        # # only going to process if there is an actual missing value in training data set
        if len(self.list_of_similar_features) > 0:
            for f, g in zip(self.list_of_similar_features, self.group_name):
                data[g + "_Min"] = data[f].apply(np.min, 1)
                data[g + "_Max"] = data[f].apply(np.max, 1)
                data[g + "_Mean"] = data[f].apply(np.mean, 1)
                data[g + "_Median"] = data[f].apply(np.median, 1)
                data[g + "_Mode"] = stats.mode(data[f], 1)[0]
                data[g + "_Std"] = data[f].apply(np.std, 1)

            return data
        else:
            return data

    def fit_transform(self, data, y=None):
        return self.transform(data)


# ____________________________________________________________________________________________________________________________________________________________________
# Binning for Continious
class Binning(BaseEstimator, TransformerMixin):
    """
    - Converts numerical variables to catagorical variable through binning
    - Number of binns are automitically determined through Sturges method
    - Once discretize, original feature will be dropped
        Args:
            features_to_discretize: list of featur names to be binned

  """

    def __init__(self, features_to_discretize):
        self.features_to_discretize = features_to_discretize

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        # only do if features are provided
        if len(self.features_to_discretize) > 0:
            data_t = self.disc.transform(
                np.array(data[self.features_to_discretize]).reshape(
                    -1, self.len_columns
                )
            )
            # make pandas data frame
            data_t = pd.DataFrame(
                data_t, columns=self.features_to_discretize, index=data.index
            )
            # all these columns are catagorical
            data_t = data_t.astype(str)
            # drop original columns
            data.drop(self.features_to_discretize, axis=1, inplace=True)
            # add newly created columns
            data = pd.concat((data, data_t), axis=1)
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        # only do if features are given

        if len(self.features_to_discretize) > 0:

            # place holder for all the features for their binns
            self.binns = []
            for i in self.features_to_discretize:
                # get numbr of binns
                hist, _ = np.histogram(data[i], bins="sturges")
                self.binns.append(len(hist))

            # how many colums to deal with
            self.len_columns = len(self.features_to_discretize)
            # now do fit transform
            self.disc = KBinsDiscretizer(
                n_bins=self.binns, encode="ordinal", strategy="kmeans"
            )
            data_t = self.disc.fit_transform(
                np.array(data[self.features_to_discretize]).reshape(
                    -1, self.len_columns
                )
            )
            # make pandas data frame
            data_t = pd.DataFrame(
                data_t, columns=self.features_to_discretize, index=data.index
            )
            # all these columns are catagorical
            data_t = data_t.astype(str)
            # drop original columns
            data.drop(self.features_to_discretize, axis=1, inplace=True)
            # add newly created columns
            data = pd.concat((data, data_t), axis=1)

        return data


# ______________________________________________________________________________________________________________________
# Scaling & Power Transform
class Scaling_and_Power_transformation(BaseEstimator, TransformerMixin):
    """
    -Given a data set, applies Min Max, Standar Scaler or Power Transformation (yeo-johnson)
    -it is recommended to run Define_dataTypes first
    - ignores target variable 
      Args: 
        target: string , name of the target variable
        function_to_apply: string , default 'zscore' (standard scaler), all other {'minmaxm','yj','quantile','robust','maxabs'} ( min max,yeo-johnson & quantile power transformation, robust and MaxAbs scaler )

  """

    def __init__(self, target, function_to_apply="zscore", random_state_quantile=42):
        self.target = target
        self.function_to_apply = function_to_apply
        self.random_state_quantile = random_state_quantile
        # self.transform_target = transform_target
        # self.ml_usecase = ml_usecase

    def fit(self, dataset, y=None):

        data = dataset
        # we only want to apply if there are numeric columns
        self.numeric_features = (
            data.drop(self.target, axis=1, errors="ignore")
            .select_dtypes(include=["float32", "int64"])
            .columns
        )
        if len(self.numeric_features) > 0:
            if self.function_to_apply == "zscore":
                self.scale_and_power = StandardScaler()
                self.scale_and_power.fit(data[self.numeric_features])
            elif self.function_to_apply == "minmax":
                self.scale_and_power = MinMaxScaler()
                self.scale_and_power.fit(data[self.numeric_features])
            elif self.function_to_apply == "yj":
                self.scale_and_power = PowerTransformer(
                    method="yeo-johnson", standardize=True
                )
                self.scale_and_power.fit(data[self.numeric_features])
            elif self.function_to_apply == "quantile":
                self.scale_and_power = QuantileTransformer(
                    random_state=self.random_state_quantile,
                    output_distribution="normal",
                )
                self.scale_and_power.fit(data[self.numeric_features])
            elif self.function_to_apply == "robust":
                self.scale_and_power = RobustScaler()
                self.scale_and_power.fit(data[self.numeric_features])
            elif self.function_to_apply == "maxabs":
                self.scale_and_power = MaxAbsScaler()
                self.scale_and_power.fit(data[self.numeric_features])

        return self

    def transform(self, dataset, y=None):
        data = dataset

        if len(self.numeric_features) > 0:
            self.data_t = pd.DataFrame(
                self.scale_and_power.transform(data[self.numeric_features])
            )
            # we need to set the same index as original data
            self.data_t.index = data.index
            self.data_t.columns = self.numeric_features
            for i in self.numeric_features:
                data[i] = self.data_t[i]
            return data

        else:
            return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        self.fit(data)
        # convert target if appropriate
        # default behavious is quantile transformer
        # if ((self.ml_usecase == 'regression') and (self.transform_target == True)):
        #   self.scale_and_power_target = QuantileTransformer(random_state=self.random_state_quantile,output_distribution='normal')
        #   data[self.target]=self.scale_and_power_target.fit_transform(np.array(data[self.target]).reshape(-1,1))

        return self.transform(data)


# ______________________________________________________________________________________________________________________
# Scaling & Power Transform
class Target_Transformation(BaseEstimator, TransformerMixin):
    """
    - Applies Power Transformation (yeo-johnson , Box-Cox) to target variable (Applicable to Regression only)
      - 'bc' for Box_Coc & 'yj' for yeo-johnson, default is Box-Cox
    - if target containes negtive / zero values , yeo-johnson is automatically selected 
    
  """

    def __init__(self, target, function_to_apply="bc"):
        self.target = target
        if function_to_apply == "bc":
            function_to_apply = "box-cox"
        else:
            function_to_apply = "yeo-johnson"
        self.function_to_apply = function_to_apply

    def inverse_transform(self, dataset, y=None):
        data = self.p_transform_target.inverse_transform(
            np.array(dataset).reshape(-1, 1)
        )
        return data

    def fit(self, dataset, y=None):
        self.fit_transform(dataset, y=y)

        return self

    def transform(self, dataset, y=None):
        data = dataset
        if self.target in dataset.columns:
            # apply transformation
            data[self.target] = self.p_transform_target.transform(
                np.array(data[self.target]).reshape(-1, 1)
            )
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        # if target has zero or negative values use yj instead
        if any(data[self.target] <= 0):
            self.function_to_apply = "yeo-johnson"
        # apply transformation
        self.p_transform_target = PowerTransformer(method=self.function_to_apply)
        data[self.target] = self.p_transform_target.fit_transform(
            np.array(data[self.target]).reshape(-1, 1)
        )

        return data


# __________________________________________________________________________________________________________________________
# Time feature extractor
class Make_Time_Features(BaseEstimator, TransformerMixin):
    """
    -Given a time feature , it extracts more features
    - Only accepts / works where feature / data type is datetime64[ns]
    - full list of features is:
      ['month','weekday',is_month_end','is_month_start','hour']
    - all extracted features are defined as string / object
    -it is recommended to run Define_dataTypes first
      Args: 
        time_feature: list of feature names as datetime64[ns] , default empty/none , if empty/None , it will try to pickup dates automatically where data type is datetime64[ns]
        list_of_features: list of required features , default value ['month','weekday','is_month_end','is_month_start','hour']

  """

    def __init__(
        self,
        time_feature=None,
        list_of_features=["month", "weekday", "is_month_end", "is_month_start", "hour"],
    ):
        self.time_feature = time_feature
        self.list_of_features_o = set(list_of_features)

    def fit(self, data, y=None):
        if self.time_feature is None:
            self.time_feature = data.select_dtypes(include=["datetime64[ns]"]).columns
            self.has_hour_ = set()
            for i in self.time_feature:
                if "hour" in self.list_of_features_o:
                    if any(x.hour for x in data[i]):
                        self.has_hour_.add(i)
        return self

    def transform(self, dataset, y=None):
        data = dataset.copy()
        # run fit transform first

        def get_time_features(r):
            features = []
            if "month" in self.list_of_features_o:
                features.append(("_month", str(r.month)))
            if "weekday" in self.list_of_features_o:
                features.append(("_weekday", str(r.weekday())))
            if "is_month_end" in self.list_of_features_o:
                features.append(
                    (
                        "_is_month_end",
                        "1"
                        if calendar.monthrange(r.year, r.month)[1] == r.day
                        else "0",
                    )
                )
            if "is_month_start" in self.list_of_features_o:
                features.append(("_is_month_start", "1" if r.day == 1 else "0"))
            return tuple(features)

        # start making features for every column in the time list
        for i in self.time_feature:
            list_of_features = [get_time_features(r) for r in data[i]]

            fd = defaultdict(list)
            for x in list_of_features:
                for k, v in x:
                    fd[k].append(v)

            for k, v in fd.items():
                data[i + k] = v

            # make hour column if choosen
            if "hour" in self.list_of_features_o and i in self.has_hour_:
                h = [r.hour for r in data[i]]
                data[f"{i}_hour"] = h
                data[f"{i}_hour"] = data[f"{i}_hour"].apply(str)

        # we dont need time columns any more
        data.drop(self.time_feature, axis=1, inplace=True)

        return data

    def fit_transform(self, dataset, y=None):
        # if no columns names are given , then pick datetime columns
        self.fit(dataset, y=y)

        return self.transform(dataset, y=y)


# ____________________________________________________________________________________________________________________________________________________________________
# Ordinal transformer
class Ordinal(BaseEstimator, TransformerMixin):
    """
    - converts categorical features into ordinal values 
    - takes a dataframe , and information about column names and ordered categories as dict
    - returns float panda data frame
  """

    def __init__(self, info_as_dict):
        self.info_as_dict = info_as_dict

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        new_data_test = pd.DataFrame(
            self.enc.transform(data[self.info_as_dict.keys()]),
            columns=self.info_as_dict.keys(),
            index=data.index,
        )
        for i in self.info_as_dict.keys():
            data[i] = new_data_test[i]
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        # creat categories from given keys in the data set
        cat_list = []
        for i in self.info_as_dict.values():
            i = [np.array(i)]
            cat_list = cat_list + i

        # now do fit transform
        self.enc = OrdinalEncoder(categories=cat_list)
        new_data_train = pd.DataFrame(
            self.enc.fit_transform(data.loc[:, self.info_as_dict.keys()]),
            columns=self.info_as_dict,
            index=data.index,
        )
        # new_data = pd.DataFrame(self.enc.fit_transform(data.loc[:,self.info_as_dict.keys()]))
        for i in self.info_as_dict.keys():
            data[i] = new_data_train[i]

        return data


# _______________________________________________________________________________________________________________________

# make dummy variables
class Dummify(BaseEstimator, TransformerMixin):
    """
    - makes one hot encoded variables for dummy variable
    - it is HIGHLY recommended to run the Select_Data_Type class first
    - Ignores target variable

      Args: 
        target: string , name of the target variable
  """

    def __init__(self, target):
        self.target = target

        # creat ohe object
        self.ohe = OneHotEncoder(handle_unknown="ignore", dtype=np.float32)

    def fit(self, dataset, y=None):
        data = dataset
        # will only do this if there are categorical variables
        if len(data.select_dtypes(include=("object")).columns) > 0:
            # we need to learn the column names once the training data set is dummify
            # save non categorical data
            self.data_nonc = data.drop(
                self.target, axis=1, errors="ignore"
            ).select_dtypes(exclude=("object"))
            self.target_column = data[[self.target]]
            # # plus we will only take object data types
            categorical_data = data.drop(
                self.target, axis=1, errors="ignore"
            ).select_dtypes(include=("object"))
            # # now fit the trainin column
            self.ohe.fit(categorical_data)
            self.data_columns = self.ohe.get_feature_names(categorical_data.columns)

        return self

    def transform(self, dataset, y=None):
        data = dataset.copy()
        # will only do this if there are categorical variables
        if len(data.select_dtypes(include=("object")).columns) > 0:
            # only for test data
            self.data_nonc = data.drop(
                self.target, axis=1, errors="ignore"
            ).select_dtypes(exclude=("object"))
            # fit without target and only categorical columns
            array = self.ohe.transform(
                data.drop(self.target, axis=1, errors="ignore").select_dtypes(
                    include=("object")
                )
            ).toarray()
            data_dummies = pd.DataFrame(array, columns=self.data_columns)
            data_dummies.index = self.data_nonc.index
            if self.target in data.columns:
                target_column = data[[self.target]]
            else:
                target_column = None
            # now put target , numerical and categorical variables back togather
            data = pd.concat((target_column, self.data_nonc, data_dummies), axis=1)
            del self.data_nonc
            return data
        else:
            return data

    def fit_transform(self, dataset, y=None):
        data = dataset.copy()
        # will only do this if there are categorical variables
        if len(data.select_dtypes(include=("object")).columns) > 0:
            self.fit(data)
            # fit without target and only categorical columns
            array = self.ohe.transform(
                data.drop(self.target, axis=1, errors="ignore").select_dtypes(
                    include=("object")
                )
            ).toarray()
            data_dummies = pd.DataFrame(array, columns=self.data_columns)
            data_dummies.index = self.data_nonc.index
            # now put target , numerical and categorical variables back togather
            data = pd.concat((self.target_column, self.data_nonc, data_dummies), axis=1)
            # remove unwanted attributes
            del (self.target_column, self.data_nonc)
            return data
        else:
            return data


# _______________________________________________________________________________________________________________________
# Outlier
class Outlier(BaseEstimator, TransformerMixin):
    """
    - Removes outlier using ABOD,KNN,IFO,PCA & HOBS using hard voting
    - Only takes numerical / One Hot Encoded features
  """

    def __init__(
        self, target, contamination=0.20, random_state=42, methods=["knn", "iso", "pca"]
    ):
        self.target = target
        self.contamination = contamination
        self.random_state = random_state
        self.methods = methods

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, data, y=None):
        return data

    def fit_transform(self, dataset, y=None):

        # dummify if there are any obects
        if len(dataset.select_dtypes(include="object").columns) > 0:
            self.dummy = Dummify(self.target)
            data = self.dummy.fit_transform(dataset)
        else:
            data = dataset

        data_without_target = data.drop(self.target, axis=1)

        if "knn" in self.methods:
            self.knn = KNN(contamination=self.contamination)
            self.knn.fit(data_without_target)
            knn_predict = self.knn.predict(data_without_target)
            data_without_target["knn"] = knn_predict

        if "iso" in self.methods:
            self.iso = IForest(
                contamination=self.contamination,
                random_state=self.random_state,
                behaviour="new",
            )
            self.iso.fit(data_without_target)
            iso_predict = self.iso.predict(data_without_target)
            data_without_target["iso"] = iso_predict

        if "pca" in self.methods:
            self.pca = PCA_od(
                contamination=self.contamination, random_state=self.random_state
            )
            self.pca.fit(data_without_target)
            pca_predict = self.pca.predict(data_without_target)
            data_without_target["pca"] = pca_predict

        data_without_target["vote_outlier"] = data_without_target[self.methods].sum(
            axis=1
        )

        self.outliers = data_without_target[
            data_without_target["vote_outlier"] == len(self.methods)
        ].index

        return dataset[~dataset.index.isin(self.outliers)]


# ____________________________________________________________________________________________________________________________________________________________________
# Column Name cleaner transformer
class Clean_Colum_Names(BaseEstimator, TransformerMixin):
    """
    - Cleans special chars that are not supported by jason format
  """

    def fit(self, data, y=None):
        return self

    def transform(self, dataset, y=None):
        data = dataset
        data.columns = data.columns.str.replace(r"[\,\}\{\]\[\:\"\']", "")
        return data

    def fit_transform(self, dataset, y=None):
        return self.transform(dataset, y=y)


# __________________________________________________________________________________________________________________________________________________________________________
# Clustering entire data
class Cluster_Entire_Data(BaseEstimator, TransformerMixin):
    """
    - Applies kmeans clustering to the entire data set and produce clusters
    - Highly recommended to run the DataTypes_Auto_infer class first
      Args:
          target_variable: target variable (integer or numerical only)
          check_clusters_upto: to determine optimum number of kmeans clusters, set the uppler limit of clusters
  """

    def __init__(self, target_variable, check_clusters_upto=20, random_state=42):
        self.target = target_variable
        self.check_clusters = check_clusters_upto + 1
        self.random_state = random_state

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        data = data.drop(self.target, axis=1, errors="ignore")
        # first convert to dummy
        if len(data.select_dtypes(include="object").columns) > 0:
            data_t1 = self.dummy.transform(data)
        else:
            data_t1 = data

        # # # now make PLS
        # # data_t1 = self.pls.transform(data_t1)
        # # data_t1 = self.pca.transform(data_t1)
        # # now predict with the clustes
        predict = pd.DataFrame(self.k_object.predict(data_t1), index=data.index)
        data["data_cluster"] = predict
        data["data_cluster"] = data["data_cluster"].astype("object")
        if self.target in dataset.columns:
            data[self.target] = dataset[self.target]
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset.copy()
        # first convert to dummy (if there are objects in data set)
        if len(data.select_dtypes(include="object").columns) > 0:
            self.dummy = Dummify(self.target)
            data_t1 = self.dummy.fit_transform(data)
            data_t1 = data_t1.drop(self.target, axis=1)
        else:
            data_t1 = data.drop(self.target, axis=1)

        # now make PLS
        # self.pls = PLSRegression(n_components=len(data_t1.columns)-1)
        # data_t1 = self.pls.fit_transform(data_t1.drop(self.target,axis=1),data_t1[self.target])[0]
        # self.pca = PCA(n_components=len(data_t1.columns)-1)
        # data_t1 = self.pca.fit_transform(data_t1.drop(self.target,axis=1))

        # we are goign to make a place holder , for 2 to 20 clusters
        self.ph = pd.DataFrame(
            np.arange(2, self.check_clusters, 1), columns=["clusters"]
        )
        self.ph["Silhouette"] = float(0)
        self.ph["calinski"] = float(0)

        # Now start making clusters
        for k in self.ph.index:
            c = self.ph["clusters"][k]
            self.k_object = cluster.KMeans(
                n_clusters=c,
                init="k-means++",
                precompute_distances="auto",
                n_init=10,
                random_state=self.random_state,
            )
            self.k_object.fit(data_t1)
            self.ph.iloc[k, 1] = metrics.silhouette_score(
                data_t1, self.k_object.labels_
            )
            self.ph.iloc[k, 2] = metrics.calinski_harabasz_score(
                data_t1, self.k_object.labels_
            )

        # now standardize the scores and make a total column
        m = MinMaxScaler((-1, 1))
        self.ph["calinski"] = m.fit_transform(
            np.array(self.ph["calinski"]).reshape(-1, 1)
        )
        self.ph["Silhouette"] = m.fit_transform(
            np.array(self.ph["Silhouette"]).reshape(-1, 1)
        )
        self.ph["total"] = self.ph["Silhouette"] + self.ph["calinski"]
        # sort it by total column and take the first row column 0 , that would represent the optimal clusters
        try:
            self.clusters = int(
                self.ph[self.ph["total"] == max(self.ph["total"])]["clusters"]
            )
        except:  # in case there isnt a decisive measure , take calinski as yeard stick
            self.clusters = int(
                self.ph[self.ph["calinski"] == max(self.ph["calinski"])]["clusters"]
            )
        # Now make the final cluster object
        self.k_object = cluster.KMeans(
            n_clusters=self.clusters,
            init="k-means++",
            precompute_distances="auto",
            n_init=10,
            random_state=self.random_state,
        )
        # now do fit predict
        predict = pd.DataFrame(self.k_object.fit_predict(data_t1), index=data.index)
        data["data_cluster"] = predict
        data["data_cluster"] = data["data_cluster"].astype("object")

        if self.target in dataset.columns:
            data[self.target] = dataset[self.target]

        return data


# __________________________________________________________________________________________________________________________________________
# Clustering catagorical data
class Reduce_Cardinality_with_Clustering(BaseEstimator, TransformerMixin):
    """
    - Reduces the level of catagorical column / cardinality through clustering 
    - Highly recommended to run the DataTypes_Auto_infer class first
      Args:
          target_variable: target variable (integer or numerical only)
          catagorical_feature: list of features on which clustering  is to be applied / cardinality to be reduced
          check_clusters_upto: to determine optimum number of kmeans clusters, set the uppler limit of clusters
  """

    def __init__(
        self,
        target_variable,
        catagorical_feature=[],
        check_clusters_upto=30,
        random_state=42,
    ):
        self.target = target_variable
        self.feature = catagorical_feature
        self.check_clusters = check_clusters_upto + 1
        self.random = random_state

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        # we already know which leval belongs to whihc cluster , so all w need is to replace levels with clusters we already have from training data set
        for i, z in zip(self.feature, self.ph_data):
            data[i] = data[i].replace(list(z["levels"]), z["cluster"])
        return data

    def fit_transform(self, dataset, y=None):
        data = dataset.copy()
        # first convert to dummy
        if len(data.select_dtypes(include="object").columns) > 0:
            self.dummy = Dummify(self.target)
            data_t = self.dummy.fit_transform(data.drop(self.feature, axis=1))
            # data_t1 = data_t1.drop(self.target,axis=1)
        else:
            data_t = data.drop(self.feature, axis=1)

        # now make PLS
        self.pls = PLSRegression(
            n_components=2
        )  # since we are only using two componenets to group #PLSRegression(n_components=len(data_t1.columns)-1)
        data_pls = self.pls.fit_transform(
            data_t.drop(self.target, axis=1), data_t[self.target]
        )[0]

        # # now we will take one component and then we calculate mean, median, min, max and sd of that one component grouped by the catagorical levels
        self.ph_data = []
        self.ph_clusters = []
        for i in self.feature:
            data_t1 = pd.DataFrame(
                dict(levels=data[i], comp1=data_pls[:, 0], comp2=data_pls[:, 1]),
                index=data.index,
            )
            # now group by feature
            data_t1 = data_t1.groupby("levels")
            data_t1 = data_t1[["comp1", "comp2"]].agg(
                ["mean", "median", "min", "max", "std"]
            )  # this gives us a df with only numeric columns (min , max ) and level as index
            # some time if a level has only one record  its std will come up as NaN, so convert NaN to 1
            data_t1.fillna(1, inplace=True)

            # now number of clusters cant be more than the number of samples in aggregated data , so
            self.check_clusters = min(self.check_clusters, len(data_t1))

            # # we are goign to make a place holder , for 2 to 20 clusters
            self.ph = pd.DataFrame(
                np.arange(2, self.check_clusters, 1), columns=["clusters"]
            )
            self.ph["Silhouette"] = float(0)
            self.ph["calinski"] = float(0)

            # Now start making clusters
            for k in self.ph.index:
                c = self.ph["clusters"][k]
                self.k_object = cluster.KMeans(
                    n_clusters=c,
                    init="k-means++",
                    precompute_distances="auto",
                    n_init=10,
                    random_state=self.random,
                )
                self.k_object.fit(data_t1)
                self.ph.iloc[k, 1] = metrics.silhouette_score(
                    data_t1, self.k_object.labels_
                )
                self.ph.iloc[k, 2] = metrics.calinski_harabasz_score(
                    data_t1, self.k_object.labels_
                )

            # now standardize the scores and make a total column
            m = MinMaxScaler((-1, 1))
            self.ph["calinski"] = m.fit_transform(
                np.array(self.ph["calinski"]).reshape(-1, 1)
            )
            self.ph["Silhouette"] = m.fit_transform(
                np.array(self.ph["Silhouette"]).reshape(-1, 1)
            )
            self.ph["total"] = self.ph["Silhouette"] + self.ph["calinski"]
            # sort it by total column and take the first row column 0 , that would represent the optimal clusters
            try:
                self.clusters = int(
                    self.ph[self.ph["total"] == max(self.ph["total"])]["clusters"]
                )
            except:  # in case there isnt a decisive measure , take calinski as yeard stick
                self.clusters = int(
                    self.ph[self.ph["calinski"] == max(self.ph["calinski"])]["clusters"]
                )
            self.ph_clusters.append(self.ph)
            # Now make the final cluster object
            self.k_object = cluster.KMeans(
                n_clusters=self.clusters,
                init="k-means++",
                precompute_distances="auto",
                n_init=10,
                random_state=self.random,
            )
            # now do fit predict
            predict = self.k_object.fit_predict(data_t1)
            # put it back with the group by aggregate columns
            data_t1["cluster"] = predict
            data_t1["cluster"] = data_t1["cluster"].apply(str)
            # now we dont need all the columns, only the cluster column is required along with the index (index also has a name , we  groupy as "levels")
            data_t1 = data_t1[["cluster"]]
            # now convert index ot the column
            data_t1.reset_index(
                level=0, inplace=True
            )  # this table now only contains every level and its cluster
            # self.data_t1= data_t1
            # we can now replace cluster with the original level in the original data frame
            data[i] = data[i].replace(list(data_t1["levels"]), data_t1["cluster"])
            self.ph_data.append(data_t1)

        if self.target in dataset.columns:
            data[self.target] = dataset[self.target]
        return data


# ____________________________________________________________________________________________________________________________________________
# Clustering catagorical data
class Reduce_Cardinality_with_Counts(BaseEstimator, TransformerMixin):
    """
    - Reduces the level of catagorical column by replacing levels with their count & converting objects into float
      Args:
          catagorical_feature: list of features on which clustering is to be applied
  """

    def __init__(self, catagorical_feature=[]):
        self.feature = catagorical_feature

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        # we already know level counts
        for i, z, k in zip(self.feature, self.ph_data, self.ph_u):
            data[i] = data[i].replace(k, z["counts"])
            data[i] = data[i].astype("float32")

        return data

    def fit_transform(self, dataset, y=None):
        data = dataset
        #
        self.ph_data = []
        self.ph_u = []
        for i in self.feature:
            data_t1 = pd.DataFrame(
                dict(
                    levels=data[i].groupby(data[i], sort=False).count().index,
                    counts=data[i].groupby(data[i], sort=False).count().values,
                )
            )
            u = data[i].unique()
            # replace levels with counts
            data[i].replace(u, data_t1["counts"], inplace=True)
            data[i] = data[i].astype("float32")
            self.ph_data.append(data_t1)
            self.ph_u.append(u)

        return data


# ____________________________________________________________________________________________________________________________________________
# take noneliner transformations
class Make_NonLiner_Features(BaseEstimator, TransformerMixin):
    """
    - convert numerical features into polynomial features
    - it is HIGHLY recommended to run the Autoinfer_Data_Type class first
    - Ignores target variable
    - it picks up data type float32 as numerical 
    - for multiclass classification problem , set subclass arg to 'multi'

      Args: 
        target: string , name of the target variable
        Polynomial_degree: int ,default 2  
  """

    def __init__(
        self,
        target,
        ml_usecase="classification",
        Polynomial_degree=2,
        other_nonliner_features=["sin", "cos", "tan"],
        top_features_to_pick=0.20,
        random_state=42,
        subclass="ignore",
        n_jobs=1,
    ):
        self.target = target
        self.Polynomial_degree = Polynomial_degree
        self.ml_usecase = ml_usecase
        self.other_nonliner_features = other_nonliner_features
        self.top_features_to_pick = top_features_to_pick
        self.random_state = random_state
        self.subclass = subclass
        self.n_jobs = n_jobs

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):  # same application for test and train
        data = dataset

        self.numeric_columns = (
            data.drop(self.target, axis=1, errors="ignore")
            .select_dtypes(include="float32")
            .columns
        )
        if self.Polynomial_degree >= 2:  # dont run anything if powr is les than 2
            # self.numeric_columns = data.drop(self.target,axis=1,errors='ignore').select_dtypes(include="float32").columns
            # start taking powers
            for i in range(2, self.Polynomial_degree + 1):
                ddc_power = np.power(data[self.numeric_columns], i)
                ddc_col = list(ddc_power.columns)
                ii = str(i)
                ddc_col = [ddc_col + "_Power" + ii for ddc_col in ddc_col]
                ddc_power.columns = ddc_col
                # put it back with data dummy
                # data = pd.concat((data,ddc_power),axis=1)
        else:
            ddc_power = pd.DataFrame()

        # take sin:
        if "sin" in self.other_nonliner_features:
            ddc_sin = np.sin(data[self.numeric_columns])
            ddc_col = list(ddc_sin.columns)
            ddc_col = ["sin(" + i + ")" for i in ddc_col]
            ddc_sin.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_sin),axis=1)
        else:
            ddc_sin = pd.DataFrame()

        # take cos:
        if "cos" in self.other_nonliner_features:
            ddc_cos = np.cos(data[self.numeric_columns])
            ddc_col = list(ddc_cos.columns)
            ddc_col = ["cos(" + i + ")" for i in ddc_col]
            ddc_cos.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_cos),axis=1)
        else:
            ddc_cos = pd.DataFrame()

        # take tan:
        if "tan" in self.other_nonliner_features:
            ddc_tan = np.tan(data[self.numeric_columns])
            ddc_col = list(ddc_tan.columns)
            ddc_col = ["tan(" + i + ")" for i in ddc_col]
            ddc_tan.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_tan),axis=1)
        else:
            ddc_tan = pd.DataFrame()

        # dummy_all
        dummy_all = pd.concat((data, ddc_power, ddc_sin, ddc_cos, ddc_tan), axis=1)
        # we can select top features using RF
        # # and we only want to do this if the dummy all have more than 50 features
        # if len(dummy_all.columns) > 71:

        dummy_all = dummy_all[self.columns_to_keep]
        if self.target in dataset.columns:
            dummy_all[self.target] = dataset[self.target]
        return dummy_all

    def fit_transform(self, dataset, y=None):

        data = dataset

        self.numeric_columns = (
            data.drop(self.target, axis=1, errors="ignore")
            .select_dtypes(include="float32")
            .columns
        )
        if self.Polynomial_degree >= 2:  # dont run anything if powr is les than 2
            # self.numeric_columns = data.drop(self.target,axis=1,errors='ignore').select_dtypes(include="float32").columns
            # start taking powers
            for i in range(2, self.Polynomial_degree + 1):
                ddc_power = np.power(data[self.numeric_columns], i)
                ddc_col = list(ddc_power.columns)
                ii = str(i)
                ddc_col = [ddc_col + "_Power" + ii for ddc_col in ddc_col]
                ddc_power.columns = ddc_col
                # put it back with data dummy
                # data = pd.concat((data,ddc_power),axis=1)
        else:
            ddc_power = pd.DataFrame()

        # take sin:
        if "sin" in self.other_nonliner_features:
            ddc_sin = np.sin(data[self.numeric_columns])
            ddc_col = list(ddc_sin.columns)
            ddc_col = ["sin(" + i + ")" for i in ddc_col]
            ddc_sin.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_sin),axis=1)
        else:
            ddc_sin = pd.DataFrame()

        # take cos:
        if "cos" in self.other_nonliner_features:
            ddc_cos = np.cos(data[self.numeric_columns])
            ddc_col = list(ddc_cos.columns)
            ddc_col = ["cos(" + i + ")" for i in ddc_col]
            ddc_cos.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_cos),axis=1)
        else:
            ddc_cos = pd.DataFrame()

        # take tan:
        if "tan" in self.other_nonliner_features:
            ddc_tan = np.tan(data[self.numeric_columns])
            ddc_col = list(ddc_tan.columns)
            ddc_col = ["tan(" + i + ")" for i in ddc_col]
            ddc_tan.columns = ddc_col
            # put it back with data dummy
            # data = pd.concat((data,ddc_tan),axis=1)
        else:
            ddc_tan = pd.DataFrame()

        # dummy_all
        dummy_all = pd.concat(
            (data[[self.target]], ddc_power, ddc_sin, ddc_cos, ddc_tan), axis=1
        )
        # we can select top features using our Feature Selection Classic transformer
        afs = Advanced_Feature_Selection_Classic(
            target=self.target,
            ml_usecase=self.ml_usecase,
            top_features_to_pick=self.top_features_to_pick,
            random_state=self.random_state,
            subclass=self.subclass,
            n_jobs=self.n_jobs,
        )
        dummy_all_t = afs.fit_transform(dummy_all)

        data = pd.concat((data, dummy_all_t), axis=1)
        # # making sure no duplicated columns are there
        data = data.loc[:, ~data.columns.duplicated()]
        self.columns_to_keep = data.drop(self.target, axis=1).columns
        return data


# ______________________________________________________________________________________________________________________________________________________
# Feature Selection
class Advanced_Feature_Selection_Classic(BaseEstimator, TransformerMixin):
    """
    - Selects important features and reduces the feature space. Feature selection is based on Random Forest , Light GBM and Correlation
    - to run on multiclass classification , set the subclass argument to 'multi'
  """

    def __init__(
        self,
        target,
        ml_usecase="classification",
        top_features_to_pick=0.10,
        random_state=42,
        subclass="ignore",
        n_jobs=1,
    ):
        self.target = target
        self.ml_usecase = ml_usecase
        self.top_features_to_pick = 1 - top_features_to_pick
        self.random_state = random_state
        self.subclass = subclass
        self.n_jobs = n_jobs

    def fit(self, dataset, y=None):
        self.fit_transform(dataset, y=y)
        return self

    def transform(self, dataset, y=None):
        # return the data with onlys specific columns
        data = dataset
        # self.selected_columns.remove(self.target)
        data = data[self.selected_columns_test]
        if self.target in dataset.columns:
            data[self.target] = dataset[self.target]
        return data

    def fit_transform(self, dataset, y=None):

        dummy_all = dataset.copy()
        dummy_all[self.target] = dummy_all[self.target].astype("float32")

        # Random Forest
        max_fe = min(70, int(np.sqrt(len(dummy_all.columns))))
        max_sa = min(1000, int(np.sqrt(len(dummy_all))))

        if self.ml_usecase == "classification":
            m = rfc(
                100,
                max_depth=5,
                max_features=max_fe,
                n_jobs=self.n_jobs,
                max_samples=max_sa,
                random_state=self.random_state,
            )
        else:
            m = rfr(
                100,
                max_depth=5,
                max_features=max_fe,
                n_jobs=self.n_jobs,
                max_samples=max_sa,
                random_state=self.random_state,
            )

        m.fit(dummy_all.drop(self.target, axis=1), dummy_all[self.target])
        # self.fe_imp_table= pd.DataFrame(m.feature_importances_,columns=['Importance'],index=dummy_all.drop(self.target,axis=1).columns).sort_values(by='Importance',ascending= False)
        self.fe_imp_table = pd.DataFrame(
            m.feature_importances_,
            columns=["Importance"],
            index=dummy_all.drop(self.target, axis=1).columns,
        )
        self.fe_imp_table = self.fe_imp_table[
            self.fe_imp_table["Importance"]
            >= self.fe_imp_table.quantile(self.top_features_to_pick)[0]
        ]
        top = self.fe_imp_table.index
        dummy_all_columns_RF = dummy_all[top].columns

        # LightGBM
        max_fe = min(70, int(np.sqrt(len(dummy_all.columns))))
        max_sa = min(
            float(1000 / len(dummy_all)),
            float(np.sqrt(len(dummy_all) / len(dummy_all))),
        )

        if self.ml_usecase == "classification":
            m = lgbmc(
                n_estimators=100,
                max_depth=5,
                n_jobs=self.n_jobs,
                subsample=max_sa,
                random_state=self.random_state,
            )
        else:
            m = lgbmr(
                n_estimators=100,
                max_depth=5,
                n_jobs=self.n_jobs,
                subsample=max_sa,
                random_state=self.random_state,
            )
        m.fit(dummy_all.drop(self.target, axis=1), dummy_all[self.target])
        # self.fe_imp_table= pd.DataFrame(m.feature_importances_,columns=['Importance'],index=dummy_all.drop(self.target,axis=1).columns).sort_values(by='Importance',ascending= False)
        self.fe_imp_table = pd.DataFrame(
            m.feature_importances_,
            columns=["Importance"],
            index=dummy_all.drop(self.target, axis=1).columns,
        )
        self.fe_imp_table = self.fe_imp_table[
            self.fe_imp_table["Importance"]
            >= self.fe_imp_table.quantile(self.top_features_to_pick)[0]
        ]
        top = self.fe_imp_table.index
        dummy_all_columns_LGBM = dummy_all[top].columns

        # we can now select top correlated feature
        if self.subclass != "multi":
            corr = pd.DataFrame(np.corrcoef(dummy_all.T))
            corr.columns = dummy_all.columns
            corr.index = dummy_all.columns
            # corr = corr[self.target].abs().sort_values(ascending=False)[0:self.top_features_to_pick+1]
            corr = corr[self.target].abs()
            corr = corr[corr.index != self.target]  # drop the target column
            corr = corr[corr >= corr.quantile(self.top_features_to_pick)]
            corr = pd.DataFrame(dict(features=corr.index, value=corr)).reset_index(
                drop=True
            )
            corr = corr.drop_duplicates(subset="value")
            corr = corr["features"]
            # corr = pd.DataFrame(dict(features=corr.index,value=corr)).reset_index(drop=True)
            # corr = corr.drop_duplicates(subset='value')[0:self.top_features_to_pick+1]
            # corr = corr['features']
        else:
            corr = list()

        self.dummy_all_columns_RF = dummy_all_columns_RF
        self.dummy_all_columns_LGBM = dummy_all_columns_LGBM
        self.corr = corr

        self.selected_columns = list(
            set(
                [self.target]
                + list(dummy_all_columns_RF)
                + list(corr)
                + list(dummy_all_columns_LGBM)
            )
        )

        self.selected_columns_test = (
            dataset[self.selected_columns].drop(self.target, axis=1).columns
        )
        return dataset[self.selected_columns]


# _

# ______________________________________________________________________________________________________________________________________________________
# Boruta Feature Selection algorithm
# Base on: https://github.com/scikit-learn-contrib/boruta_py/blob/master/boruta/boruta_py.py
class Boruta_Feature_Selection(BaseEstimator, TransformerMixin):
    """
          Boruta selection algorithm based on borutaPy sklearn-contrib and
          Miron B Kursa, https://m2.icm.edu.pl/boruta/
          Selects the most important features.
            Args:
              target (str): target column name
              ml_usecase (str): case: classification or regression
              top_features_to_pick: to make...
              max_iteration {int): overall iterations of shuffle and train forests 
              alpha {float): p-value on which 
              the option to favour one measur to another. e.g. if value is .6 , during feature selection tug of war, correlation target measure will have a higher say.
              A value of .5 means both measure have equal say 
  """

    def __init__(
        self,
        target,
        ml_usecase="classification",
        top_features_to_pick=1.0,
        max_iteration=200,
        n_iter_no_change=25,
        alpha=0.05,
        random_state=42,
        subclass="ignore",
        n_jobs=1,
    ):
        self.target = target
        self.ml_usecase = ml_usecase
        self.top_features_to_pick = top_features_to_pick
        self.random_state = random_state
        self.subclass = subclass
        self.max_iteration = max_iteration
        self.n_iter_no_change = n_iter_no_change
        self.alpha = alpha
        self.selected_columns_test = []
        self.n_jobs = n_jobs

    @property
    def selected_columns(self):
        return self.selected_columns_test + [self.target]

    def fit(self, dataset, y=None):
        from .patches.boruta_py import BorutaPyPatched

        dummy_data = dataset
        X, y = dummy_data.drop(self.target, axis=1), dummy_data[self.target].values
        y = y.astype("float32")
        X_cols = X.columns
        X = X.values

        if self.ml_usecase == "classification":
            m = rfc(
                100,
                max_depth=5,
                n_jobs=self.n_jobs,
                random_state=self.random_state,
                class_weight="balanced",
            )
        else:
            m = rfr(
                100, max_depth=5, n_jobs=self.n_jobs, random_state=self.random_state,
            )

        feat_selector = BorutaPyPatched(
            m,
            n_estimators="auto",
            perc=int(self.top_features_to_pick * 100),
            max_iter=self.max_iteration,
            random_state=self.random_state,
            early_stopping=(self.n_iter_no_change > 0),
            n_iter_no_change=self.n_iter_no_change,
        )

        try:
            feat_selector.fit(X, y)
            self.selected_columns_test = list(X_cols[feat_selector.support_])
        except:
            # boruta may errors out if all features are selected
            self.selected_columns_test = list(X_cols)

        return self

    def transform(self, dataset, y=None):
        if self.target in dataset.columns:
            return dataset[self.selected_columns]
        else:
            return dataset[self.selected_columns_test]

    def fit_transform(self, dataset, y=None):
        self.fit(dataset, y=y)
        return self.transform(dataset, y=y)


# _________________________________________________________________________________________________________________________________________
class Fix_multicollinearity(BaseEstimator, TransformerMixin):
    """
          Fixes multicollinearity between predictor variables , also considering the correlation between target variable.
          Only applies to regression or two class classification ML use case
          Takes numerical and one hot encoded variables only
            Args:
              threshold (float): The utmost absolute pearson correlation tolerated beyween featres from 0.0 to 1.0
              target_variable (str): The target variable/column name
              correlation_with_target_threshold: minimum absolute correlation required between every feature and the target variable , default 1.0 (0.0 to 1.0)
              correlation_with_target_preference: float (0.0 to 1.0), default .08 ,while choosing between a pair of features w.r.t multicol & correlation target , this gives 
              the option to favour one measur to another. e.g. if value is .6 , during feature selection tug of war, correlation target measure will have a higher say.
              A value of .5 means both measure have equal say 
  """

    # mamke a constructer

    def __init__(
        self,
        threshold,
        target_variable,
        correlation_with_target_threshold=0.0,
        correlation_with_target_preference=1.0,
    ):
        self.threshold = threshold
        self.target_variable = target_variable
        self.correlation_with_target_threshold = correlation_with_target_threshold
        self.target_corr_weight = correlation_with_target_preference
        self.multicol_weight = 1 - correlation_with_target_preference

    # Make fit method

    def fit(self, data, y=None):
        """
        Args:
            data = takes preprocessed data frame
        Returns:
            None
        """

        if data[self.target_variable].dtype not in ["int32", "int64"]:
            raise ValueError('dtype for the target variable should be int32 or int64 only')

        # global data1
        data1 = data.select_dtypes(include=["int32", "int64", "float32", "float64"])
        # try:
        #   self.data1 = self.data1.astype('float16')
        # except:
        #   None
        # make an correlation db with abs correlation db
        # self.data_c = self.data1.T.drop_duplicates()
        # self.data1 = self.data_c.T
        corr = pd.DataFrame(np.corrcoef(data1.T))
        corr.columns = data1.columns
        corr.index = data1.columns
        # corr_matrix = abs(data1.corr())
        corr_matrix = abs(corr)

        # for every diagonal value, make it Nan
        corr_matrix.values[
            tuple([np.arange(corr_matrix.shape[0])] * 2)
        ] = np.NaN

        # Now Calculate the average correlation of every feature with other, and get a pandas data frame
        avg_cor = pd.DataFrame(corr_matrix.mean())
        avg_cor["feature"] = avg_cor.index
        avg_cor.reset_index(drop=True, inplace=True)
        avg_cor.columns = ["avg_cor", "features"]

        # Calculate the correlation with the target
        targ_cor = pd.DataFrame(corr_matrix[self.target_variable].dropna())
        targ_cor["feature"] = targ_cor.index
        targ_cor.reset_index(drop=True, inplace=True)
        targ_cor.columns = ["target_variable", "features"]

        # Now, add a column for variable name and drop index
        corr_matrix["column"] = corr_matrix.index
        corr_matrix.reset_index(drop=True, inplace=True)

        # now we need to melt it , so that we can correlation pair wise , with two columns
        cols = corr_matrix.column
        melt = (
            corr_matrix.melt(id_vars=["column"], value_vars=cols)
            .sort_values(by="value", ascending=False)
            .dropna()
        )

        # now bring in the avg correlation for first of the pair
        merge = pd.merge(
            melt, avg_cor, left_on="column", right_on="features"
        ).drop("features", axis=1)

        # now bring in the avg correlation for second of the pair
        merge = pd.merge(
            merge, avg_cor, left_on="variable", right_on="features"
        ).drop("features", axis=1)

        # now bring in the target correlation for first of the pair
        merge = pd.merge(
            merge, targ_cor, left_on="column", right_on="features"
        ).drop("features", axis=1)

        # now bring in the avg correlation for second of the pair
        merge = pd.merge(
            merge, targ_cor, left_on="variable", right_on="features"
        ).drop("features", axis=1)

        # sort and save
        merge = merge.sort_values(by="value", ascending=False)

        # we need to now eleminate all the pairs that are actually duplicate e.g cor(x,y) = cor(y,x) , they are the same , we need to find these and drop them
        merge["all_columns"] = merge["column"] + merge["variable"]

        # this puts all the coresponding pairs of features togather , so that we can only take one, since they are just the duplicates
        merge["all_columns"] = [sorted(i) for i in merge["all_columns"]]

        # now sort by new column
        merge = merge.sort_values(by="all_columns")

        # take every second colums
        merge = merge.iloc[::2, :]

        # make a ranking column to eliminate features
        merge["rank_x"] = round(
            self.multicol_weight * (merge["avg_cor_y"] - merge["avg_cor_x"])
            + self.target_corr_weight
            * (merge["target_variable_x"] - merge["target_variable_y"]),
            6,
        )  # round it to 6 digits

        ## Now there will be rows where the rank will be exactly zero, these is where the value (corelartion between features) is exactly one ( like price and price^2)
        ## so in that case , we can simply pick one of the variable
        # but since , features can be in either column, we will drop one column (say 'column') , only if the feature is not in the second column (in variable column)
        # both equations below will return the list of columns to drop from here
        # this is how it goes

        ## For the portion where correlation is exactly one !
        one = merge[merge["rank_x"] == 0]

        # this portion is complicated
        # table one have all the paired variable having corelation of 1
        # in a nutshell, we can take any column (one side of pair) and delete the other columns (other side of the pair)
        # however one varibale can appear more than once on any of the sides , so we will run for loop to find all pairs...
        # here it goes
        # take a list of all (but unique ) variables that have correlation 1 for eachother, we will make two copies
        u_all = list(
            pd.unique(pd.concat((one["column"], one["variable"]), axis=0))
        )
        u_all_1 = list(
            pd.unique(pd.concat((one["column"], one["variable"]), axis=0))
        )
        # take a list of features (unique) for the first side of the pair
        u_column = pd.unique(one["column"])

        # now we are going to start picking each variable from one column (one side of the pair) , check it against the other column (other side of the pair)
        # to pull all coresponding / paired variables  , and delete thoes newly varibale names from all unique list

        for i in u_column:
            # print(i)
            r = one[one["column"] == i]["variable"]
            for q in r:
                if q in u_all:
                    # print("_"+q)
                    u_all.remove(q)

        # now the unique column contains the varibales that should remain, so in order to get the variables that should be deleted :
        to_drop = list(set(u_all_1) - set(u_all))

        # to_drop_a =(list(set(one['column'])-set(one['variable'])))
        # to_drop_b =(list(set(one['variable'])-set(one['column'])))
        # to_drop = to_drop_a + to_drop_b

        ## now we are to treat where rank is not Zero and Value (correlation) is greater than a specific threshold
        non_zero = merge[
            (merge["rank_x"] != 0.0) & (merge["value"] >= self.threshold)
        ]

        # pick the column to delete
        non_zero_list = list(
            np.where(
                non_zero["rank_x"] < 0,
                non_zero["column"],
                non_zero["variable"],
            )
        )

        # add two list
        self.to_drop = to_drop + non_zero_list

        # make sure that target column is not a part of the list
        try:
            self.to_drop.remove(self.target_variable)
        except:
            pass

        # now we want to keep only the columns that have more correlation with traget by a threshold
        self.to_drop_taret_correlation = []
        if self.correlation_with_target_threshold != 0.0:
            corr = pd.DataFrame(
                np.corrcoef(data.drop(self.to_drop, axis=1).T),
                columns=data.drop(self.to_drop, axis=1).columns,
                index=data.drop(self.to_drop, axis=1).columns,
            )
            self.to_drop_taret_correlation = corr[self.target_variable].abs()
            # to_drop_taret_correlation = data.drop(self.to_drop,axis=1).corr()[target_variable].abs()
            self.to_drop_taret_correlation = self.to_drop_taret_correlation[
                self.to_drop_taret_correlation < self.correlation_with_target_threshold
            ]
            self.to_drop_taret_correlation = list(self.to_drop_taret_correlation.index)
            # to_drop = corr + to_drop
            try:
                self.to_drop_taret_correlation.remove(self.target_variable)
            except:
                pass
        return self

    # now Transform
    def transform(self, dataset, y=None):
        """
        Args:f
            data = takes preprocessed data frame
        Returns:
            data frame
    """
        data = dataset
        data = data.drop(self.to_drop, axis=1)
        # now drop less correlated data
        data.drop(self.to_drop_taret_correlation, axis=1, inplace=True, errors="ignore")
        return data

    # fit_transform
    def fit_transform(self, data, y=None):

        """
        Args:
            data = takes preprocessed data frame
        Returns:
            data frame
    """
        self.fit(data)
        return self.transform(data)


# ____________________________________________________________________________________________________________________________________________________________________
# handle perfect multicollinearity
class Remove_100(BaseEstimator, TransformerMixin):
    """
    - Takes DF, return data frame while removing features that are perfectly correlated (droping one)
  """

    def __init__(self, target):
        self.target = target
        self.columns_to_drop = []

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        return dataset.drop(self.columns_to_drop, axis=1)

    def fit_transform(self, dataset, y=None):
        data = dataset

        targetless_data = data.drop(self.target, axis=1)

        # correlation should be calculated between at least two features, if there is only 1, there is nothing to delete
        if len(targetless_data.columns) <= 1:
            return data

        corr = pd.DataFrame(np.corrcoef(targetless_data.T))
        corr.columns = targetless_data.columns
        corr.index = targetless_data.columns
        corr_matrix = abs(corr)

        # Now, add a column for variable name and drop index
        corr_matrix["column"] = corr_matrix.index
        corr_matrix.reset_index(drop=True, inplace=True)

        # now we need to melt it , so that we can correlation pair wise , with two columns
        cols = corr_matrix.column
        melt = corr_matrix.melt(id_vars=["column"], value_vars=cols).sort_values(
            by="value", ascending=False
        )  # .dropna()
        melt["value"] = round(melt["value"], 2)  # round it to two digits

        # now pick variables where value is one and 'column' != variabe ( both columns are not same)
        c1 = melt["value"] == 1.00
        c2 = melt["column"] != melt["variable"]
        melt = melt[((c1 == True) & (c2 == True))]

        # we need to now eleminate all the pairs that are actually duplicate e.g cor(x,y) = cor(y,x) , they are the same , we need to find these and drop them
        melt["all_columns"] = melt["column"] + melt["variable"]

        # this puts all the coresponding pairs of features togather , so that we can only take one, since they are just the duplicates
        melt["all_columns"] = [sorted(i) for i in melt["all_columns"]]

        # # now sort by new column
        melt = melt.sort_values(by="all_columns")

        # # take every second colums
        melt = melt.iloc[::2, :]

        # lets keep the columns on the left hand side of the table
        self.columns_to_drop = melt["variable"]

        return data.drop(self.columns_to_drop, axis=1)


# _______________________________________________________________________________________________________________________________________________________________________________________________
# custome DFS
class DFS_Classic(BaseEstimator, TransformerMixin):
    """
    - Automated feature interactions using multiplication, division , addition & substraction
    - Only accepts numeric / One Hot Encoded features
    - Takes DF, return same DF 
    - for Multiclass classification problem , set subclass arg as 'multi'
  """

    def __init__(
        self,
        target,
        ml_usecase="classification",
        interactions=["multiply", "divide", "add", "subtract"],
        top_features_to_pick_percentage=0.05,
        random_state=42,
        subclass="ignore",
        n_jobs=1,
    ):
        self.target = target
        self.interactions = interactions
        self.top_n_correlated = top_features_to_pick_percentage  # (this will be 1- top_features , but handled in the Advance_feature_selection )
        self.ml_usecase = ml_usecase
        self.random_state = random_state
        self.subclass = subclass
        self.n_jobs = n_jobs

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):

        data = dataset

        data_without_target = data.drop(self.target, axis=1, errors="ignore")
        # for multiplication:
        # we need bot catagorical and numerical columns

        if "multiply" in self.interactions:

            data_multiply = pd.concat(
                [
                    data_without_target.mul(col[1], axis="index")
                    for col in data_without_target.iteritems()
                ],
                axis=1,
            )
            data_multiply.columns = [
                "_multiply_".join([i, j])
                for j in data_without_target.columns
                for i in data_without_target.columns
            ]
            # we dont need to apply rest of conditions
            data_multiply.index = data.index
        else:
            data_multiply = pd.DataFrame()

        # for division, we only want it to apply to numerical columns
        if "divide" in self.interactions:

            data_divide = pd.concat(
                [
                    data_without_target[self.numeric_columns].div(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_divide.columns = [
                "_divide_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            data_divide.replace([np.inf, -np.inf], 0, inplace=True)
            data_divide.fillna(0, inplace=True)
            data_divide.index = data.index
        else:
            data_divide = pd.DataFrame()

        # for addition, we only want it to apply to numerical columns
        if "add" in self.interactions:

            data_add = pd.concat(
                [
                    data_without_target[self.numeric_columns].add(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_add.columns = [
                "_add_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            data_add.index = data.index
        else:
            data_add = pd.DataFrame()

        # for substraction, we only want it to apply to numerical columns
        if "subtract" in self.interactions:

            data_substract = pd.concat(
                [
                    data_without_target[self.numeric_columns].sub(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_substract.columns = [
                "_subtract_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            data_substract.index = data.index
        else:
            data_substract = pd.DataFrame()

        # get all the dummy data combined
        dummy_all = pd.concat(
            (data, data_multiply, data_divide, data_add, data_substract), axis=1
        )
        del data_multiply
        del data_divide
        del data_add
        del data_substract
        # now only return the columns we want:
        dummy_all = dummy_all[self.columns_to_keep]
        if self.target in dataset.columns:
            dummy_all[self.target] = dataset[self.target]
        return dummy_all

    def fit_transform(self, dataset, y=None):

        data = dataset

        data_without_target = data.drop(self.target, axis=1, errors="ignore")

        # we need to seperate numerical and ont hot encoded columns
        # self.ohe_columns = [i if ((len(data[i].unique())==2) & (data[i].unique()[0] in [0,1]) & (data[i].unique()[1] in [0,1]) ) else None for i in data.drop(self.target,axis=1).columns]
        self.ohe_columns = [
            i
            for i in data.columns
            if data[i].nunique() == 2
            and data[i].unique()[0] in [0, 1]
            and data[i].unique()[1] in [0, 1]
        ]
        # self.ohe_columns = [i for i in self.ohe_columns if i is not None]
        self.numeric_columns = [
            i for i in data_without_target.columns if i not in self.ohe_columns
        ]
        target_variable = data[[self.target]]

        # for multiplication:
        # we need bot catagorical and numerical columns

        if "multiply" in self.interactions:
            data_multiply = pd.concat(
                [
                    data_without_target.mul(col[1], axis="index")
                    for col in data_without_target.iteritems()
                ],
                axis=1,
            )
            data_multiply.columns = [
                "_multiply_".join([i, j])
                for j in data_without_target.columns
                for i in data_without_target.columns
            ]
            # we dont need columns that are self interacted
            col = [
                "_multiply_".join([i, j])
                for j in data_without_target.columns
                for i in data_without_target.columns
                if i != j
            ]
            data_multiply = data_multiply[col]
            # we dont need columns where the sum of the total column is null (to catagorical variables never happening togather)
            col1 = [
                i for i in data_multiply.columns if np.nansum(data_multiply[i]) != 0
            ]
            data_multiply = data_multiply[col1]
            data_multiply.index = data.index
        else:
            data_multiply = pd.DataFrame()

        # for division, we only want it to apply to numerical columns
        if "divide" in self.interactions:
            data_divide = pd.concat(
                [
                    data_without_target[self.numeric_columns].div(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_divide.columns = [
                "_divide_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            # we dont need columns that are self interacted
            col = [
                "_divide_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
                if i != j
            ]
            data_divide = data_divide[col]
            # we dont need columns where the sum of the total column is null (to catagorical variables never happening togather)
            col1 = [i for i in data_divide.columns if np.nansum(data_divide[i]) != 0]
            data_divide = data_divide[col1]
            # additionally we need to fill anll the possible NaNs
            data_divide.replace([np.inf, -np.inf], 0, inplace=True)
            data_divide.fillna(0, inplace=True)
            data_divide.index = data.index
        else:
            data_divide = pd.DataFrame()

        # for addition, we only want it to apply to numerical columns
        if "add" in self.interactions:
            data_add = pd.concat(
                [
                    data_without_target[self.numeric_columns].add(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_add.columns = [
                "_add_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            # we dont need columns that are self interacted
            col = [
                "_add_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
                if i != j
            ]
            data_add = data_add[col]
            # we dont need columns where the sum of the total column is null (to catagorical variables never happening togather)
            col1 = [i for i in data_add.columns if np.nansum(data_add[i]) != 0]
            data_add = data_add[col1]
            data_add.index = data.index
        else:
            data_add = pd.DataFrame()

        # for substraction, we only want it to apply to numerical columns
        if "subtract" in self.interactions:
            data_substract = pd.concat(
                [
                    data_without_target[self.numeric_columns].sub(col[1], axis="index")
                    for col in data_without_target[self.numeric_columns].iteritems()
                ],
                axis=1,
            )
            data_substract.columns = [
                "_subtract_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
            ]
            # we dont need columns that are self interacted
            col = [
                "_subtract_".join([i, j])
                for j in data_without_target[self.numeric_columns].columns
                for i in data_without_target[self.numeric_columns].columns
                if i != j
            ]
            data_substract = data_substract[col]
            # we dont need columns where the sum of the total column is null (to catagorical variables never happening togather)
            col1 = [
                i for i in data_substract.columns if np.nansum(data_substract[i]) != 0
            ]
            data_substract = data_substract[col1]
            data_substract.index = data.index
        else:
            data_substract = pd.DataFrame()

        # get all the dummy data combined
        dummy_all = pd.concat(
            (data_multiply, data_divide, data_add, data_substract), axis=1
        )
        del data_multiply
        del data_divide
        del data_add
        del data_substract

        dummy_all[self.target] = target_variable
        self.dummy_all = dummy_all

        # apply advanced feature selectio
        afs = Advanced_Feature_Selection_Classic(
            target=self.target,
            ml_usecase=self.ml_usecase,
            top_features_to_pick=self.top_n_correlated,
            random_state=self.random_state,
            subclass=self.subclass,
            n_jobs=self.n_jobs,
        )
        dummy_all_t = afs.fit_transform(dummy_all)

        data_fe_final = pd.concat(
            (data, dummy_all_t), axis=1
        )  # self.data_fe[self.corr]
        # # making sure no duplicated columns are there
        data_fe_final = data_fe_final.loc[
            :, ~data_fe_final.columns.duplicated()
        ]  # new added
        # # remove thetarget column
        # # this is the final data we want that includes original , fe data plus impact of top n correlated
        self.columns_to_keep = data_fe_final.drop(self.target, axis=1).columns
        del dummy_all
        del dummy_all_t

        return data_fe_final


# ____________________________________________________________________________________________________________________________________________________________________
# Empty transformer
class Empty(BaseEstimator, TransformerMixin):
    """
    - Takes DF, return same DF 
  """

    def fit(self, data, y=None):
        return self

    def transform(self, data, y=None):
        return data

    def fit_transform(self, data, y=None):
        return self.transform(data)


# ____________________________________________________________________________________________________________________________________
# reduce feature space
class Reduce_Dimensions_For_Supervised_Path(BaseEstimator, TransformerMixin):
    """
    - Takes DF, return same DF with different types of dimensionality reduction modles (pca_liner , pca_kernal, tsne , pls, incremental)
    - except pca_liner, every other method takes integer as number of components 
    - only takes numeric variables (float & One Hot Encoded)
    - it is intended to solve supervised ML usecases , such as classification / regression
  """

    def __init__(
        self,
        target,
        method="pca_liner",
        variance_retained_or_number_of_components=0.99,
        random_state=42,
    ):
        self.target = target
        self.variance_retained = variance_retained_or_number_of_components
        self.random_state = random_state
        self.method = method

    def fit(self, data, y=None):
        self.fit_transform(data, y=y)
        return self

    def transform(self, dataset, y=None):
        data = dataset
        if self.method in [
            "pca_liner",
            "pca_kernal",
            "tsne",
            "incremental",
        ]:  # if self.method in ['pca_liner' , 'pca_kernal', 'tsne' , 'incremental','psa']
            data = data.drop(self.target, axis=1, errors="ignore")
            data_pca = self.pca.transform(data)
            data_pca = pd.DataFrame(data_pca)
            data_pca.columns = [
                "Component_" + str(i) for i in np.arange(1, len(data_pca.columns) + 1)
            ]
            data_pca.index = data.index
            if self.target in dataset.columns:
                data_pca[self.target] = dataset[self.target]
            return data_pca
        else:
            return dataset

    def fit_transform(self, dataset, y=None):
        data = dataset
        if self.method == "pca_liner":
            self.pca = PCA(self.variance_retained, random_state=self.random_state)
            # fit transform
            data_pca = self.pca.fit_transform(data.drop(self.target, axis=1))
            data_pca = pd.DataFrame(data_pca)
            data_pca.columns = [
                "Component_" + str(i) for i in np.arange(1, len(data_pca.columns) + 1)
            ]
            data_pca.index = data.index
            data_pca[self.target] = data[self.target]
            return data_pca
        elif self.method == "pca_kernal":  # take number of components only
            self.pca = KernelPCA(
                self.variance_retained,
                kernel="rbf",
                random_state=self.random_state,
                n_jobs=-1,
            )
            # fit transform
            data_pca = self.pca.fit_transform(data.drop(self.target, axis=1))
            data_pca = pd.DataFrame(data_pca)
            data_pca.columns = [
                "Component_" + str(i) for i in np.arange(1, len(data_pca.columns) + 1)
            ]
            data_pca.index = data.index
            data_pca[self.target] = data[self.target]
            return data_pca
        # elif self.method == 'pls': # take number of components only
        #   self.pca = PLSRegression(self.variance_retained,scale=False)
        #   # fit transform
        #   data_pca = self.pca.fit_transform(data.drop(self.target,axis=1),data[self.target])[0]
        #   data_pca = pd.DataFrame(data_pca)
        #   data_pca.columns = ["Component_"+str(i) for i in np.arange(1,len(data_pca.columns)+1)]
        #   data_pca.index = data.index
        #   data_pca[self.target] = data[self.target]
        #   return(data_pca)
        elif self.method == "tsne":  # take number of components only
            self.pca = TSNE(self.variance_retained, random_state=self.random_state)
            # fit transform
            data_pca = self.pca.fit_transform(data.drop(self.target, axis=1))
            data_pca = pd.DataFrame(data_pca)
            data_pca.columns = [
                "Component_" + str(i) for i in np.arange(1, len(data_pca.columns) + 1)
            ]
            data_pca.index = data.index
            data_pca[self.target] = data[self.target]
            return data_pca
        elif self.method == "incremental":  # take number of components only
            self.pca = IncrementalPCA(self.variance_retained)
            # fit transform
            data_pca = self.pca.fit_transform(data.drop(self.target, axis=1))
            data_pca = pd.DataFrame(data_pca)
            data_pca.columns = [
                "Component_" + str(i) for i in np.arange(1, len(data_pca.columns) + 1)
            ]
            data_pca.index = data.index
            data_pca[self.target] = data[self.target]
            return data_pca
        else:
            return dataset


# ___________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________
# preprocess_all_in_one
def Preprocess_Path_One(
    train_data,
    target_variable,
    ml_usecase=None,
    test_data=None,
    categorical_features=[],
    numerical_features=[],
    time_features=[],
    features_todrop=[],
    display_types=True,
    imputation_type="simple",
    numeric_imputation_strategy="mean",
    categorical_imputation_strategy="not_available",
    imputation_classifier=None,
    imputation_regressor=None,
    imputation_max_iter=10,
    imputation_warm_start=False,
    imputation_order="ascending",
    apply_zero_nearZero_variance=False,
    club_rare_levels=False,
    rara_level_threshold_percentage=0.05,
    apply_untrained_levels_treatment=False,
    untrained_levels_treatment_method="least frequent",
    apply_ordinal_encoding=False,
    ordinal_columns_and_categories={},
    apply_cardinality_reduction=False,
    cardinal_method="cluster",
    cardinal_features=[],
    apply_binning=False,
    features_to_binn=[],
    apply_grouping=False,
    group_name=[],
    features_to_group_ListofList=[[]],
    apply_polynomial_trigonometry_features=False,
    max_polynomial=2,
    trigonometry_calculations=["sin", "cos", "tan"],
    top_poly_trig_features_to_select_percentage=0.20,
    scale_data=False,
    scaling_method="zscore",
    Power_transform_data=False,
    Power_transform_method="quantile",
    remove_outliers=False,
    outlier_contamination_percentage=0.01,
    outlier_methods=["pca", "iso", "knn"],
    dummify_categoricals=True,
    apply_feature_selection=False,
    feature_selection_top_features_percentage=0.80,
    feature_selection_method="classic",
    remove_multicollinearity=False,
    maximum_correlation_between_features=0.90,
    remove_perfect_collinearity=False,
    apply_feature_interactions=False,
    feature_interactions_to_apply=["multiply", "divide", "add", "subtract"],
    feature_interactions_top_features_to_select_percentage=0.01,
    cluster_entire_data=False,
    range_of_clusters_to_try=20,
    apply_pca=False,
    pca_method="pca_liner",
    pca_variance_retained_or_number_of_components=0.99,
    random_state=42,
    n_jobs=-1,
):

    """
    Follwoing preprocess steps are taken:
      - 1) Auto infer data types 
      - 2) Impute (simple or with surrogate columns)
      - 3) Ordinal Encoder
      - 4) Drop categorical variables that have zero variance or near zero variance
      - 5) Club categorical variables levels togather as a new level (other_infrequent) that are rare / at the bottom 5% of the variable distribution
      - 6) Club unseen levels in test dataset with most/least frequent levels in train dataset 
      - 7) Reduce high cardinality in categorical features using clustering or counts
      - 8) Generate sub features from time feature such as 'month','weekday',is_month_end','is_month_start' & 'hour'
      - 9) Group features by calculating min, max, mean, median & sd of similar features
      -10) Make nonliner features (polynomial, sin , cos & tan)
      -11) Scales & Power Transform (zscore,minmax,yeo-johnson,quantile,maxabs,robust) , including option to transform target variable
      -12) Apply binning to continious variable when numeric features are provided as a list 
      -13) Detect & remove outliers using isolation forest, knn and PCA
      -14) Apply clusters to segment entire data
      -15) One Hot / Dummy encoding
      -16) Remove special characters from column names such as commas, square brackets etc to make it competible with jason dependednt models
      -17) Feature Selection throuh Random Forest , LightGBM and Pearson Correlation / Boruta algorithm
      -18) Fix multicollinearity
      -19) Feature Interaction (DFS) , multiply , divided , add and substract features
      -20) Apply diamension reduction techniques such as pca_liner, pca_kernal, incremental, tsne 
          - except for pca_liner, all other method only takes number of component (as integer) i.e no variance explaination metohd available  
  """

    # also make sure that all the column names are string
    train_data.columns = [str(i) for i in train_data.columns]
    if test_data is not None:
        test_data.columns = [str(i) for i in test_data.columns]

    if target_variable is None:
        ml_usecase = "regression"
    else:
        # WE NEED TO AUTO INFER the ml use case
        inferred_ml_usecase, subcase = infer_ml_usecase(train_data[target_variable])
        if ml_usecase is None:
            ml_usecase = inferred_ml_usecase

    dtypes = DataTypes_Auto_infer(
        target=target_variable,
        ml_usecase=ml_usecase,
        categorical_features=categorical_features,
        numerical_features=numerical_features,
        time_features=time_features,
        features_todrop=features_todrop,
        display_types=display_types,
        id_columns=find_id_columns(
            train_data, target_variable, numerical_features=numerical_features
        ),
    )

    # for imputation
    # imputation_type = "A"
    if imputation_type == "simple":
        imputer = Simple_Imputer(
            numeric_strategy=numeric_imputation_strategy,
            target_variable=target_variable,
            categorical_strategy=categorical_imputation_strategy,
        )
    # elif imputation_type == "surrogate imputer":
    #  imputer = Surrogate_Imputer(numeric_strategy=numeric_imputation_strategy,categorical_strategy=categorical_imputation_strategy,target_variable=target_variable)
    else:
        imputer = Iterative_Imputer(
            classifier=imputation_classifier,
            regressor=imputation_regressor,
            target=target_variable,
            initial_strategy_numeric=numeric_imputation_strategy,
            max_iter=imputation_max_iter,
            warm_start=imputation_warm_start,
            imputation_order=imputation_order,
            random_state=random_state,
            ordinal_columns=ordinal_columns_and_categories.keys(),
        )

    # for zero_near_zero
    if apply_zero_nearZero_variance == True:
        znz = Zroe_NearZero_Variance(target=target_variable)
    else:
        znz = SKLEARN_EMPTY_STEP

    # for rare levels clubbing:

    if club_rare_levels == True:
        club_R_L = Catagorical_variables_With_Rare_levels(
            target=target_variable, threshold=rara_level_threshold_percentage
        )
    else:
        club_R_L = SKLEARN_EMPTY_STEP

    # untrained levels in test
    if apply_untrained_levels_treatment == True:
        new_levels = New_Catagorical_Levels_in_TestData(
            target=target_variable,
            replacement_strategy=untrained_levels_treatment_method,
        )
    else:
        new_levels = New_Catagorical_Levels_in_TestData(
            target=target_variable, replacement_strategy="raise exception",
        )

    # untrained levels in test(ordinal specific)
    if apply_untrained_levels_treatment == True:
        new_levels1 = New_Catagorical_Levels_in_TestData(
            target=target_variable,
            replacement_strategy=untrained_levels_treatment_method,
        )
    else:
        new_levels1 = New_Catagorical_Levels_in_TestData(
            target=target_variable, replacement_strategy="raise exception",
        )

    # cardinality:
    if apply_cardinality_reduction == True and cardinal_method == "cluster":
        cardinality = Reduce_Cardinality_with_Clustering(
            target_variable=target_variable,
            catagorical_feature=cardinal_features,
            check_clusters_upto=50,
            random_state=random_state,
        )
    elif apply_cardinality_reduction == True and cardinal_method == "count":
        cardinality = Reduce_Cardinality_with_Counts(
            catagorical_feature=cardinal_features
        )
    else:
        cardinality = SKLEARN_EMPTY_STEP

    # ordinal coding
    if apply_ordinal_encoding == True:
        # we need to make sure that if the columns chosen by user have NA & imputer strategy is not_availablle then we add that to the category first
        for i in ordinal_columns_and_categories.keys():
            if sum(train_data[i].isnull()) > 0:
                if categorical_imputation_strategy == "not_available":
                    lis = ["not_available"] + ordinal_columns_and_categories[i]
                    ordinal_columns_and_categories.update({i: lis})

        ordinal = Ordinal(info_as_dict=ordinal_columns_and_categories)
    else:
        ordinal = SKLEARN_EMPTY_STEP

    # grouping
    if apply_grouping == True:
        group = Group_Similar_Features(
            group_name=group_name, list_of_grouped_features=features_to_group_ListofList
        )
    else:
        group = SKLEARN_EMPTY_STEP

    # non_liner_features
    if apply_polynomial_trigonometry_features == True:
        nonliner = Make_NonLiner_Features(
            target=target_variable,
            ml_usecase=ml_usecase,
            Polynomial_degree=max_polynomial,
            other_nonliner_features=trigonometry_calculations,
            top_features_to_pick=top_poly_trig_features_to_select_percentage,
            random_state=random_state,
            subclass=subcase,
            n_jobs=n_jobs,
        )
    else:
        nonliner = SKLEARN_EMPTY_STEP

    # binning
    if apply_binning == True:
        binn = Binning(features_to_discretize=features_to_binn)
    else:
        binn = SKLEARN_EMPTY_STEP

    # scaling & power transform
    if scale_data == True:
        scaling = Scaling_and_Power_transformation(
            target=target_variable,
            function_to_apply=scaling_method,
            random_state_quantile=random_state,
        )
    else:
        scaling = SKLEARN_EMPTY_STEP

    if Power_transform_data == True:
        P_transform = Scaling_and_Power_transformation(
            target=target_variable,
            function_to_apply=Power_transform_method,
            random_state_quantile=random_state,
        )
    else:
        P_transform = SKLEARN_EMPTY_STEP

    # for Time Variables
    feature_time = Make_Time_Features()

    if dummify_categoricals == True:
        dummy = Dummify(target_variable)
    else:
        dummy = SKLEARN_EMPTY_STEP

    # remove putliers
    if remove_outliers == True:
        rem_outliers = Outlier(
            target=target_variable,
            contamination=outlier_contamination_percentage,
            random_state=random_state,
            methods=outlier_methods,
        )
    else:
        rem_outliers = SKLEARN_EMPTY_STEP

    # cluster all data:
    if cluster_entire_data == True:
        cluster_all = Cluster_Entire_Data(
            target_variable=target_variable,
            check_clusters_upto=range_of_clusters_to_try,
            random_state=random_state,
        )
    else:
        cluster_all = SKLEARN_EMPTY_STEP

    # clean column names for special char
    clean_names = Clean_Colum_Names()

    # feature selection
    if apply_feature_selection:
        # TODO: add autoselect
        if feature_selection_method == "boruta":
            feature_select = Boruta_Feature_Selection(
                target=target_variable,
                ml_usecase=ml_usecase,
                top_features_to_pick=feature_selection_top_features_percentage,
                random_state=random_state,
                subclass=subcase,
                n_jobs=n_jobs,
            )
        else:
            feature_select = Advanced_Feature_Selection_Classic(
                target=target_variable,
                ml_usecase=ml_usecase,
                top_features_to_pick=feature_selection_top_features_percentage,
                random_state=random_state,
                subclass=subcase,
                n_jobs=n_jobs,
            )
    else:
        feature_select = SKLEARN_EMPTY_STEP

    # removing multicollinearity
    if remove_multicollinearity == True and subcase != "multi":
        fix_multi = Fix_multicollinearity(
            target_variable=target_variable,
            threshold=maximum_correlation_between_features,
        )
    elif remove_multicollinearity == True and subcase == "multi":
        fix_multi = Fix_multicollinearity(
            target_variable=target_variable,
            threshold=maximum_correlation_between_features,
            correlation_with_target_preference=0.0,
        )
    else:
        fix_multi = SKLEARN_EMPTY_STEP

    # remove 100% collinearity
    if remove_perfect_collinearity == True:
        fix_perfect = Remove_100(target=target_variable)
    else:
        fix_perfect = SKLEARN_EMPTY_STEP

    # apply dfs
    if apply_feature_interactions == True:
        dfs = DFS_Classic(
            target=target_variable,
            ml_usecase=ml_usecase,
            interactions=feature_interactions_to_apply,
            top_features_to_pick_percentage=feature_interactions_top_features_to_select_percentage,
            random_state=random_state,
            subclass=subcase,
            n_jobs=n_jobs,
        )
    else:
        dfs = SKLEARN_EMPTY_STEP

    # apply pca
    if apply_pca == True:
        pca = Reduce_Dimensions_For_Supervised_Path(
            target=target_variable,
            method=pca_method,
            variance_retained_or_number_of_components=pca_variance_retained_or_number_of_components,
            random_state=random_state,
        )
    else:
        pca = SKLEARN_EMPTY_STEP

    pipe = Pipeline(
        [
            ("dtypes", dtypes),
            ("imputer", imputer),
            (
                "new_levels1",
                new_levels1,
            ),  # specifically used for ordinal, so that if a new level comes in a feature that was marked ordinal can be handled
            ("ordinal", ordinal),
            ("cardinality", cardinality),
            ("znz", znz),
            ("club_R_L", club_R_L),
            ("new_levels", new_levels),
            ("feature_time", feature_time),
            ("group", group),
            ("nonliner", nonliner),
            ("scaling", scaling),
            ("P_transform", P_transform),
            ("binn", binn),
            ("rem_outliers", rem_outliers),
            ("cluster_all", cluster_all),
            ("dummy", dummy),
            ("fix_perfect", fix_perfect),
            ("clean_names", clean_names),
            ("feature_select", feature_select),
            ("fix_multi", fix_multi),
            ("dfs", dfs),
            ("pca", pca),
        ]
    )

    return pipe


# ______________________________________________________________________________________________________________________________________________________
# preprocess_all_in_one_unsupervised
def Preprocess_Path_Two(
    train_data,
    ml_usecase=None,
    test_data=None,
    categorical_features=[],
    numerical_features=[],
    time_features=[],
    features_todrop=[],
    display_types=False,
    imputation_type="simple",
    numeric_imputation_strategy="mean",
    categorical_imputation_strategy="not_available",
    imputation_classifier=None,
    imputation_regressor=None,
    imputation_max_iter=10,
    imputation_warm_start=False,
    imputation_order="ascending",
    apply_zero_nearZero_variance=False,
    club_rare_levels=False,
    rara_level_threshold_percentage=0.05,
    apply_untrained_levels_treatment=False,
    untrained_levels_treatment_method="least frequent",
    apply_cardinality_reduction=False,
    cardinal_method="cluster",
    cardinal_features=[],
    apply_ordinal_encoding=False,
    ordinal_columns_and_categories={},
    apply_binning=False,
    features_to_binn=[],
    apply_grouping=False,
    group_name=[],
    features_to_group_ListofList=[[]],
    scale_data=False,
    scaling_method="zscore",
    Power_transform_data=False,
    Power_transform_method="quantile",
    remove_outliers=False,
    outlier_contamination_percentage=0.01,
    outlier_methods=["pca", "iso", "knn"],
    remove_multicollinearity=False,
    maximum_correlation_between_features=0.90,
    remove_perfect_collinearity=False,
    apply_pca=False,
    pca_method="pca_liner",
    pca_variance_retained_or_number_of_components=0.99,
    random_state=42,
    n_jobs=-1,
):

    """
    Follwoing preprocess steps are taken:
      - THIS IS BUILt FOR UNSUPERVISED LEARNING
      - 1) Auto infer data types 
      - 2) Impute (simple or with surrogate columns)
      - 3) Ordinal Encoder
      - 4) Drop categorical variables that have zero variance or near zero variance
      - 5) Club categorical variables levels togather as a new level (other_infrequent) that are rare / at the bottom 5% of the variable distribution
      - 6) Club unseen levels in test dataset with most/least frequent levels in train dataset 
      - 7) Reduce high cardinality in categorical features using clustering or counts
      - 8) Generate sub features from time feature such as 'month','weekday',is_month_end','is_month_start' & 'hour'
      - 9) Group features by calculating min, max, mean, median & sd of similar features
      -10) Scales & Power Transform (zscore,minmax,yeo-johnson,quantile,maxabs,robust) , including option to transform target variable
      -11) Apply binning to continious variable when numeric features are provided as a list 
      -12) Detect & remove outliers using isolation forest, knn and PCA
      -13) One Hot / Dummy encoding
      -14) Remove special characters from column names such as commas, square brackets etc to make it competible with jason dependednt models
      -15) Fix multicollinearity
      -16) Apply diamension reduction techniques such as pca_liner, pca_kernal, incremental, tsne 
          - except for pca_liner, all other method only takes number of component (as integer) i.e no variance explaination metohd available 
  """
    return Preprocess_Path_One(
        train_data=train_data,
        ml_usecase=ml_usecase,
        target_variable=None,
        test_data=test_data,
        categorical_features=categorical_features,
        numerical_features=numerical_features,
        time_features=time_features,
        features_todrop=features_todrop,
        display_types=display_types,
        imputation_type=imputation_type,
        numeric_imputation_strategy=numeric_imputation_strategy,
        categorical_imputation_strategy=categorical_imputation_strategy,
        imputation_classifier=imputation_classifier,
        imputation_regressor=imputation_regressor,
        imputation_max_iter=imputation_max_iter,
        imputation_warm_start=imputation_warm_start,
        imputation_order=imputation_order,
        apply_zero_nearZero_variance=apply_zero_nearZero_variance,
        club_rare_levels=club_rare_levels,
        rara_level_threshold_percentage=rara_level_threshold_percentage,
        apply_untrained_levels_treatment=apply_untrained_levels_treatment,
        untrained_levels_treatment_method=untrained_levels_treatment_method,
        apply_ordinal_encoding=apply_ordinal_encoding,
        ordinal_columns_and_categories=ordinal_columns_and_categories,
        apply_cardinality_reduction=apply_cardinality_reduction,
        cardinal_method=cardinal_method,
        cardinal_features=cardinal_features,
        apply_binning=apply_binning,
        features_to_binn=features_to_binn,
        apply_grouping=apply_grouping,
        group_name=group_name,
        features_to_group_ListofList=features_to_group_ListofList,
        scale_data=scale_data,
        scaling_method=scaling_method,
        Power_transform_data=Power_transform_data,
        Power_transform_method=Power_transform_method,
        remove_outliers=remove_outliers,
        outlier_contamination_percentage=outlier_contamination_percentage,
        outlier_methods=outlier_methods,
        dummify_categoricals=False,
        remove_multicollinearity=remove_multicollinearity,
        maximum_correlation_between_features=maximum_correlation_between_features,
        remove_perfect_collinearity=remove_perfect_collinearity,
        apply_pca=apply_pca,
        pca_method=pca_method,
        pca_variance_retained_or_number_of_components=pca_variance_retained_or_number_of_components,
        random_state=random_state,
        n_jobs=n_jobs,
    )


def _get_labelencoder_reverse_dict(le: LabelEncoder) -> dict:
    # now get the replacement dict
    rev = le.inverse_transform(range(0, len(le.classes_)))
    rep = np.array(range(0, len(le.classes_)))
    replacement = {}
    for i, k in zip(rev, rep):
        replacement[i] = k
    return replacement
