import numpy as np
import pandas as pd
from keras.models import Sequential
from sklearn.model_selection import KFold, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from pandas import read_csv
from keras.datasets import mnist
from keras.utils.np_utils import to_categorical  # convert to one-hot-encoding
from keras.models import Sequential
from keras.initializers import RandomNormal, glorot_normal
from sklearn.model_selection import StratifiedKFold
from keras.layers import Activation

from keras import activations
from keras import backend as K
from keras import initializers, regularizers, constraints
from keras.initializers import RandomNormal, glorot_normal
from keras.layers import Input, Embedding, Dense, concatenate, InputSpec
from keras.layers import Activation, concatenate, InputSpec
from keras.utils import conv_utils, np_utils
from keras.utils.generic_utils import (
    func_load,
    deserialize_keras_object,
    has_arg,
    get_custom_objects,
)
from keras.utils.generic_utils import deserialize_keras_object, func_dump

np.random.seed(7)
from keras.engine.base_layer import Layer


p1 = 1
p2 = 2
p3 = 6
p4 = 24
p5 = 120
p6 = 720


def create_model_classic(active1, active2):
    # create model
    model = Sequential()
    model.add(Dense(12, input_dim=8, activation=active1))
    model.add(Dense(12, activation=active2))
    model.add(Dense(20))
    model.add(Dense(1, activation="sigmoid"))
    # Compile model
    model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
    return model


# Defining New Activation functions
############################################################################
def X_1(x):
    return K.pow(x, 1)


get_custom_objects().update({"X_1": Activation(X_1)})
############################################################################
def X_2(x):
    return (K.pow(x, 2)) / 2


get_custom_objects().update({"X_2": Activation(X_2)})
############################################################################
def X_3(x):
    return (K.pow(x, 3)) / 6


get_custom_objects().update({"X_3": Activation(X_3)})
############################################################################
def X_4(x):
    return (K.pow(x, 4)) / 24


get_custom_objects().update({"X_4": Activation(X_4)})
############################################################################
def X_5(x):
    return (K.pow(x, 5)) / 120


get_custom_objects().update({"X_5": Activation(X_5)})
###############################################################################
def X_6(x):
    return (K.pow(x, 6)) / 720


get_custom_objects().update({"X_6": Activation(X_6)})


# Defining New Activation functions
############################################################################
def S_X_1(x):
    return K.sin(x)


get_custom_objects().update({"S_X_1": Activation(S_X_1)})
############################################################################
def S_X_2(x):
    return K.sin(2 * x)


get_custom_objects().update({"S_X_2": Activation(S_X_2)})
############################################################################
def S_X_3(x):
    return K.sin(3 * x)


get_custom_objects().update({"S_X_3": Activation(S_X_3)})
############################################################################
def S_X_4(x):
    return K.sin(4 * x)


get_custom_objects().update({"S_X_4": Activation(S_X_4)})
############################################################################
def S_X_5(x):
    return K.sin(5 * x)


get_custom_objects().update({"S_X_5": Activation(S_X_5)})
###############################################################################
def S_X_6(x):
    return K.sin(6 * x)


get_custom_objects().update({"S_X_6": Activation(S_X_6)})
###############################################################################


# Defining New Activation functions
############################################################################
def C_a_X_1(x):
    return (K.pow(x, 1)) / p1


get_custom_objects().update({"C_a_X_1": Activation(C_a_X_1)})
############################################################################
def C_a_X_2(x):
    return ((2 * K.pow(x, 2)) - 1) / p2


get_custom_objects().update({"C_a_X_2": Activation(C_a_X_2)})
############################################################################
def C_a_X_3(x):
    return ((4 * K.pow(x, 3)) - (3 * x)) / p3


get_custom_objects().update({"C_a_X_3": Activation(C_a_X_3)})
############################################################################
def C_a_X_4(x):
    return ((8 * K.pow(x, 4)) - (8 * K.pow(x, 2)) + 1) / p4


get_custom_objects().update({"C_a_X_4": Activation(C_a_X_4)})
############################################################################
def C_a_X_5(x):
    return ((16 * K.pow(x, 5)) - (20 * K.pow(x, 3)) + (5 * x)) / p5


get_custom_objects().update({"C_a_X_5": Activation(C_a_X_5)})
############################################################################
def C_a_X_6(x):
    return ((32 * K.pow(x, 6)) - (48 * K.pow(x, 4)) + (18 * K.pow(x, 2)) - 1) / p6


get_custom_objects().update({"C_a_X_6": Activation(C_a_X_6)})


# Defining New Activation functions
############################################################################
def C_b_X_1(x):
    return (2 * K.pow(x, 1)) / p1


get_custom_objects().update({"C_b_X_1": Activation(C_b_X_1)})
############################################################################
def C_b_X_2(x):
    return ((4 * K.pow(x, 2)) - 1) / p2


get_custom_objects().update({"C_b_X_2": Activation(C_b_X_2)})
############################################################################
def C_b_X_3(x):
    return ((8 * K.pow(x, 3)) - (4 * x)) / p3


get_custom_objects().update({"C_b_X_3": Activation(C_b_X_3)})
############################################################################
def C_b_X_4(x):
    return ((16 * K.pow(x, 4)) - (12 * K.pow(x, 2)) + 1) / p4


get_custom_objects().update({"C_b_X_4": Activation(C_b_X_4)})
############################################################################
def C_b_X_5(x):
    return ((32 * K.pow(x, 5)) - (32 * K.pow(x, 3)) + 6 * x) / p5


get_custom_objects().update({"C_b_X_5": Activation(C_b_X_5)})
############################################################################
def C_b_X_6(x):
    return ((64 * K.pow(x, 4)) - (80 * K.pow(x, 4)) + (24 * K.pow(x, 2)) - 1) / p6


get_custom_objects().update({"C_b_X_6": Activation(C_b_X_6)})


# Defining New Activation functions
############################################################################
def H_X_1(x):
    return (2 * K.pow(x, 1)) / p1


get_custom_objects().update({"H_X_1": Activation(H_X_1)})
############################################################################
def H_X_2(x):
    return ((4 * K.pow(x, 2)) - 2) / p2


get_custom_objects().update({"H_X_2": Activation(H_X_2)})
############################################################################
def H_X_3(x):
    return ((8 * K.pow(x, 3)) - (12 * x)) / p3


get_custom_objects().update({"H_X_3": Activation(H_X_3)})
############################################################################
def H_X_4(x):
    return ((16 * K.pow(x, 4)) - (48 * K.pow(x, 2)) + 12) / p4


get_custom_objects().update({"H_X_4": Activation(H_X_4)})
############################################################################
def H_X_5(x):
    return ((32 * K.pow(x, 5)) - (160 * K.pow(x, 3)) + 120 * x) / p5


get_custom_objects().update({"H_X_5": Activation(H_X_5)})
############################################################################
def H_X_6(x):
    return ((64 * K.pow(x, 6)) - (480 * K.pow(x, 4)) + (720 * K.pow(x, 2)) - 120) / p6


get_custom_objects().update({"H_X_6": Activation(H_X_6)})


# Defining New Activation functions
############################################################################
def L_X_1(x):
    return (K.pow(x, 1)) / p1


get_custom_objects().update({"L_X_1": Activation(L_X_1)})
############################################################################
def L_X_2(x):
    return ((3 * K.pow(x, 2)) - 1) / (2 * p2)


get_custom_objects().update({"L_X_2": Activation(L_X_2)})
############################################################################
def L_X_3(x):
    return ((5 * K.pow(x, 3)) - (3 * x)) / (2 * p3)


get_custom_objects().update({"L_X_3": Activation(L_X_3)})
############################################################################
def L_X_4(x):
    return ((35 * K.pow(x, 4)) - (32 * K.pow(x, 2)) + 3) / (8 * p4)


get_custom_objects().update({"L_X_4": Activation(L_X_4)})
############################################################################
def L_X_5(x):
    return ((63 * K.pow(x, 5)) - (70 * K.pow(x, 3)) + 15 * x) / (8 * p5)


get_custom_objects().update({"L_X_5": Activation(L_X_5)})
############################################################################
def L_X_6(x):
    return ((231 * K.pow(x, 6)) - (315 * K.pow(x, 4)) + (105 * K.pow(x, 2)) - 5) / (
        16 * p6
    )


get_custom_objects().update({"L_X_6": Activation(L_X_6)})


# ### Dense_Co is a custom Keras Layer which we develop to implement SWAG
#


class Dense_Ch_a(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_Ch_a, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("C_a_X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("C_a_X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("C_a_X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("C_a_X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("C_a_X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("C_a_X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dense_Ch_b(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_Ch_b, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("C_b_X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("C_b_X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("C_b_X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("C_b_X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("C_b_X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("C_b_X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dense_S(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_S, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("S_X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("S_X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("S_X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("S_X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("S_X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("S_X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dense_H(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_H, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("H_X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("H_X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("H_X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("H_X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("H_X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("H_X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dense_L(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_L, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("L_X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("L_X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("L_X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("L_X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("L_X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("L_X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dense_Co(Layer):
    def __init__(
        self,
        units,
        activation=None,
        hidden_dim=None,
        use_bias=True,
        kernel_initializer="glorot_uniform",
        bias_initializer="zeros",
        kernel_regularizer=None,
        bias_regularizer=None,
        activity_regularizer=None,
        kernel_constraint=None,
        bias_constraint=None,
        **kwargs
    ):
        if "input_shape" not in kwargs and "input_dim" in kwargs:
            kwargs["input_shape"] = (kwargs.pop("input_dim"),)
        super(Dense_Co, self).__init__(**kwargs)
        self.units = units
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(min_ndim=2)
        self.supports_masking = True
        if hidden_dim != None:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.units

    def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        ##########################################################################
        self.kernel = self.add_weight(
            shape=(input_dim, self.hidden_dim * 6),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )

        ##########################################################################
        self.kernel_all = self.add_weight(
            shape=(6 * self.hidden_dim, self.units),
            initializer=self.kernel_initializer,
            name="kernel",
            regularizer=self.kernel_regularizer,
            constraint=self.kernel_constraint,
        )
        ##########################################################################
        if self.use_bias:
            self.bias = self.add_weight(
                shape=(self.hidden_dim * 6,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

            ###########################################################################
            self.bias_all = self.add_weight(
                shape=(self.units,),
                initializer=self.bias_initializer,
                name="bias",
                regularizer=self.bias_regularizer,
                constraint=self.bias_constraint,
            )

        else:
            self.bias = None
        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
        self.built = True

    def call(self, inputs):

        output1 = K.bias_add(
            K.dot(inputs, self.kernel[:, : self.hidden_dim]),
            self.bias[: self.hidden_dim],
            data_format="channels_last",
        )
        output2 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim : self.hidden_dim * 2]),
            self.bias[self.hidden_dim : self.hidden_dim * 2],
            data_format="channels_last",
        )
        output3 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 2 : self.hidden_dim * 3]),
            self.bias[self.hidden_dim * 2 : self.hidden_dim * 3],
            data_format="channels_last",
        )
        output4 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 3 : self.hidden_dim * 4]),
            self.bias[self.hidden_dim * 3 : self.hidden_dim * 4],
            data_format="channels_last",
        )
        output5 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 4 : self.hidden_dim * 5]),
            self.bias[self.hidden_dim * 4 : self.hidden_dim * 5],
            data_format="channels_last",
        )
        output6 = K.bias_add(
            K.dot(inputs, self.kernel[:, self.hidden_dim * 5 :]),
            self.bias[self.hidden_dim * 5 :],
            data_format="channels_last",
        )

        self.activation = activations.get("X_1")
        output1 = self.activation(output1)

        self.activation = activations.get("X_2")
        output2 = self.activation(output2)

        self.activation = activations.get("X_3")
        output3 = self.activation(output3)

        self.activation = activations.get("X_4")
        output4 = self.activation(output4)

        self.activation = activations.get("X_5")
        output5 = self.activation(output5)

        self.activation = activations.get("X_6")
        output6 = self.activation(output6)
        output_all = concatenate([output1, output2, output3, output4, output5, output6])

        output_all = K.dot(output_all, self.kernel_all)
        output_all = K.bias_add(output_all, self.bias_all, data_format="channels_last")
        self.activation = activations.get("linear")
        output_all = self.activation(output_all)

        return output_all

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) >= 2
        assert input_shape[-1]
        output_shape = list(input_shape)
        output_shape[-1] = self.units
        return tuple(output_shape)

    def get_config(self):
        config = {
            "units": self.units,
            "activation": activations.serialize(self.activation),
            "use_bias": self.use_bias,
            "kernel_initializer": initializers.serialize(self.kernel_initializer),
            "bias_initializer": initializers.serialize(self.bias_initializer),
            "kernel_regularizer": regularizers.serialize(self.kernel_regularizer),
            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
            "activity_regularizer": regularizers.serialize(self.activity_regularizer),
            "kernel_constraint": constraints.serialize(self.kernel_constraint),
            "bias_constraint": constraints.serialize(self.bias_constraint),
        }
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


# Define normalize Functions
def normalize(d):
    # d is a (n x dimension) np array
    d -= np.min(d, axis=0)
    d /= np.ptp(d, axis=0)
    return d


# Define Rescaling Functions from 0-1 to 0.1-0.9
def rescale_range(d):
    # d is a (n x dimension) np array
    d = np.multiply(d, 0.89)
    d = np.add(d, 0.01)
    return d


def build_model_v1(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(Dense_Co(output_dim, hidden_dim=hidden_dim, input_dim=input_dim))
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def SWAG(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_Co(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def Ch_First(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_Ch_a(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def Ch_second(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_Ch_b(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def Hermite(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_H(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def Legendre(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_L(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model


def Sin(input_dim, hidden_dim, output_dim):
    model = Sequential()
    model.add(
        Dense_S(
            output_dim,
            hidden_dim=hidden_dim,
            input_dim=input_dim,
            kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
            bias_initializer=RandomNormal(mean=0.0, stddev=0.04),
        )
    )
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
    return model