r"""
.. versionadded:: 0.3

In this module is stored everything related to Multi-layer perceptron (MLP).
This neural network can be used for classification and regression.


Minimal Working Example
************************

.. code-block:: python

    import numpy as np
    import padasip as pa

    # data creation
    x = np.array([
            [0,0,0,0], [1,0,0,0], [0,1,0,0], [1,1,0,0],
            [0,0,1,0], [1,0,1,0], [0,1,1,0], [1,1,1,0],
            [0,0,0,1], [1,0,0,1], [0,1,0,1], [1,1,0,1],
            [0,0,1,1], [1,0,1,1], [0,1,1,1], [1,1,1,1]
        ])
    d = np.array([0,1,1,0,0,1,0,0,1,0,1,0,1,1,1,0])
    N = len(d)
    n = 4

    # creation of neural network
    nn = pa.ann.NetworkMLP([5,6], n, outputs=1, activation="tanh", mu="auto")    

    # training
    e, mse = nn.train(x, d, epochs=200, shuffle=True)    

    # get results
    y = nn.run(x)

And the result (pairs: target, output) can look like

>>> for i in zip(d, y): print i
... 
(0, 0.0032477183193071906)
(1, 1.0058082383308447)
(1, 1.0047503447788306)
(0, 0.0046026142618665845)
(0, 0.0003037425037410007)
(1, 1.0017672193832869)
(0, 0.0015817734995124679)
(0, 0.0019115885715706904)
(1, 0.99342117275580499)
(0, 0.00069114178424850147)
(1, 1.0021789943501729)
(0, 0.0021355836851727717)
(1, 0.99809312951378826)
(1, 1.0071488717506856)
(1, 1.0067500768423701)
(0, -0.0045962250501771244)
>>> 



Learning Rate Selection
**************************

If you select the learning rate (:math:`\mu` in equations,
or `mu` in code) manually, it will be used the same value for all nodes,
otherwise it is selected automatically :cite:`lecun2012efficient` as follows

:math:`\mu_{ij} = m^{-0.5}`

where the :math:`m` is the amount of nodes on input of given node.
The automatic selection is recomended and default option.


Default Values of Weights
****************************

The distribution from what the weights are taken is chosen automatically
:cite:`lecun2012efficient`, it has zero mean and
the standard derivation estimated as follows

:math:`\sigma_{w} = m^{-0.5}`

where the :math:`m` is the amount of nodes on input of given node.


References
***************

.. bibliography:: mlp.bib
    :style: plain

Code Explanation
******************
"""
import numpy as np
import padasip.consts as co

class Layer():
    """
    This class represents a single hidden layer of the MLP.

    Args:

    * `n_layer` : size of the layer (int)

    * `n_input` : how many inputs the layer have (int)

    * `activation_f` : what function should be used as activation function (str)

    * `mu` : learning rate (float or str), it can be directly the float value,
        or string `auto` for automatic selection of learning rate
        :cite:`lecun2012efficient`

    """
    
    def __init__(self, n_layer, n_input, activation_f, mu):
        sigma = n_input**(-0.5)
        if mu == "auto":
            self.mu = sigma
        else:
            self.mu = mu
        self.n_input = n_input
        self.w = np.random.normal(0, sigma, (n_layer, n_input+1))
        self.x = np.ones(n_input+1)
        self.y = np.zeros(n_input+1)
        self.f = activation_f

    def activation(self, x, f="sigmoid", der=False):
        """
        This function process values of layer outputs with activation function.

        **Args:**

        * `x` : array to process (1-dimensional array) 

        **Kwargs:**

        * `f` : activation function

        * `der` : normal output, or its derivation (bool)

        **Returns:**

        * values processed with activation function (1-dimensional array)
        
        """
        if f == "sigmoid":
            if der:
                return x * (1 - x)
            return 1. / (1 + np.exp(-x))
        elif f == "tanh":
            if der:
                return 1 - x**2 
            return (2. / (1 + np.exp(-2*x))) - 1       
                
    def predict(self, x):
        """
        This function make forward pass through this layer (no update).

        **Args:**

        * `x` : input vector (1-dimensional array)

        **Returns:**
        
        * `y` : output of MLP (float or 1-diemnsional array).
            Size depends on number of nodes in this layer.
            
        """
        self.x[1:] = x
        self.y = self.activation(np.sum(self.w*self.x, axis=1), f=self.f)
        return self.y
    
    def update(self, w, e):
        """
        This function make update according provided target
        and the last used input vector.

        **Args:**

        * `d` : target (float or 1-dimensional array).
            Size depends on number of MLP outputs.

        **Returns:**

        * `w` : weights of the layers (2-dimensional layer).
            Every row represents one node.
        
        * `e` : error used for update (float or 1-diemnsional array).
            Size correspond to size of input `d`.
        """
        if len(w.shape) == 1:
            e = self.activation(self.y, f=self.f, der=True) * e * w
            dw = self.mu * np.outer(e, self.x)
        else:
            e = self.activation(self.y, f=self.f, der=True) * (1 - self.y) * np.dot(e, w)
            dw = self.mu * np.outer(e, self.x)
        w = self.w[:,1:]
        self.w += dw
        return w, e
        
        
class NetworkMLP():
    """
    This class represents a Multi-layer Perceptron neural network.

    *Args:**

    * `layers` : array describing hidden layers of network
        (1-dimensional array of integers). Every number in array represents
        one hidden layer. For example [3, 6, 2] create
        network with three hidden layers. First layer will have 3 nodes,
        second layer will have 6 nodes and the last hidden layer
        will have 2 nodes.

    * `n_input` : number of network inputs (int). 

    **Kwargs:**

    * `outputs` : number of network outputs (int). Default is 1.

    * `activation` : activation function (str)

        * "sigmoid" - sigmoid
    
        * "tanh" : hyperbolic tangens

    * `mu` : learning rate (float or str), it can be:
        * float value - value is directly used as `mu`

        * "auto" - this will trigger automatic selection of learning rate
        according to :cite:`lecun2012efficient`

    """

    def __init__(self, layers, n_input, outputs=1, activation="sigmoid", mu="auto"):
        sigma = layers[-1]**(-0.5)
        # set learning rate
        if mu == "auto":
            self.mu = sigma
        else:
            try:
                param = float(mu)            
            except:
                raise ValueError(
                    'Parameter mu is not float or similar'
                    )
            self.mu = mu
        self.n_input = n_input
        # create output layer
        self.outputs = outputs
        if self.outputs == 1:
            self.w = np.random.normal(0, sigma, layers[-1]+1)
        else:
            self.w = np.random.normal(0, sigma, (outputs, layers[-1]+1))
        self.x = np.ones(layers[-1]+1)
        self.y = 0
        # create hidden layers
        self.n_layers = len(layers)
        self.layers = []
        for n in range(self.n_layers):
            if n == 0:
                l = Layer(layers[n], n_input, activation, mu)
                self.layers.append(l)
            else:
                l = Layer(layers[n], layers[n-1], activation, mu)
                self.layers.append(l)
 
    def train(self, x, d, epochs=10, shuffle=False):
        """
        Function for batch training of MLP.

        **Args:**

        * `x` : input array (2-dimensional array).
            Every row represents one input vector (features).

        * `d` : input array (n-dimensional array).
            Every row represents target for one input vector.
            Target can be one or more values (in case of multiple outputs).

        **Kwargs:**
        
        * `epochs` : amount of epochs (int). That means how many times
            the MLP will iterate over the passed set of data (`x`, `d`).

        * `shuffle` : if true, the order of inputs and outpust are shuffled (bool).
            That means the pairs input-output are in different order in every epoch.

        **Returns:**
        
        * `e`: output vector (m-dimensional array). Every row represents
            error (or errors) for an input and output in given epoch.
            The size of this array is length of provided data times
            amount of epochs (`N*epochs`).

        * `MSE` : mean squared error (1-dimensional array). Every value
            stands for MSE of one epoch.
            
        """
        # measure the data and check if the dimmension agree
        N = len(x)
        if not len(d) == N:
            raise ValueError('The length of vector d and matrix x must agree.')  
        if not len(x[0]) == self.n_input:
            raise ValueError('The number of network inputs is not correct.')
        if self.outputs == 1:
            if not len(d.shape) == 1:
                raise ValueError('For one output MLP the d must have one dimension')
        else:
            if not d.shape[1] == self.outputs:
                raise ValueError('The number of outputs must agree with number of columns in d')
        try:    
            x = np.array(x)
            d = np.array(d)
        except:
            raise ValueError('Impossible to convert x or d to a numpy array')
        # create empty arrays
        if self.outputs == 1:
            e = np.zeros(epochs*N)
        else:
            e = np.zeros((epochs*N, self.outputs))
        MSE = np.zeros(epochs)
        # shuffle data if demanded
        if shuffle:
            randomize = np.arange(len(x))
            np.random.shuffle(randomize)
            x = x[randomize]
            d = d[randomize]
        # adaptation loop
        for epoch in range(epochs):
            for k in range(N):
                self.predict(x[k])
                e[(epoch*N)+k] = self.update(d[k])
            MSE[epoch] = np.sum(e[epoch*N:(epoch+1)*N-1]**2) / N
        return e, MSE

    def run(self, x):
        """
        Function for batch usage of already trained and tested MLP.

        **Args:**

        * `x` : input array (2-dimensional array).
            Every row represents one input vector (features).

        **Returns:**
        
        * `y`: output vector (n-dimensional array). Every row represents
            output (outputs) for an input vector.
            
        """
        # measure the data and check if the dimmension agree
        try:    
            x = np.array(x)
        except:
            raise ValueError('Impossible to convert x to a numpy array')
        N = len(x)   
        # create empty arrays     
        if self.outputs == 1:
            y = np.zeros(N)
        else:
            y = np.zeros((N, self.outputs))
        # predict data in loop        
        for k in range(N):
            y[k] = self.predict(x[k])
        return y

    def test(self, x, d):
        """
        Function for batch test of already trained MLP.

        **Args:**

        * `x` : input array (2-dimensional array).
            Every row represents one input vector (features).

        * `d` : input array (n-dimensional array).
            Every row represents target for one input vector.
            Target can be one or more values (in case of multiple outputs).

        **Returns:**
        
        * `e`: output vector (n-dimensional array). Every row represents
            error (or errors) for an input and output.
            
        """
        # measure the data and check if the dimmension agree
        N = len(x)
        if not len(d) == N:
            raise ValueError('The length of vector d and matrix x must agree.')  
        if not len(x[0]) == self.n_input:
            raise ValueError('The number of network inputs is not correct.')
        if self.outputs == 1:
            if not len(d.shape) == 1:
                raise ValueError('For one output MLP the d must have one dimension')
        else:
            if not d.shape[1] == self.outputs:
                raise ValueError('The number of outputs must agree with number of columns in d')
        try:    
            x = np.array(x)
            d = np.array(d)
        except:
            raise ValueError('Impossible to convert x or d to a numpy array')
        # create empty arrays
        if self.outputs == 1:
            y = np.zeros(N)
        else:
            y = np.zeros((N, self.outputs))
        # measure in loop        
        for k in range(N):
            y[k] = self.predict(x[k])
        return d - y    

    def predict(self, x):
        """
        This function make forward pass through MLP (no update).

        **Args:**

        * `x` : input vector (1-dimensional array)

        **Returns:**
        
        * `y` : output of MLP (float or 1-diemnsional array).
            Size depends on number of MLP outputs.
            
        """
        # forward pass to hidden layers
        for l in self.layers:
            x = l.predict(x)
        self.x[1:] = x
        # forward pass to output layer
        if self.outputs == 1:
            self.y = np.dot(self.w, self.x)
        else: 
            self.y = np.sum(self.w*self.x, axis=1)
        return self.y
    
    def update(self, d):
        """
        This function make update according provided target
        and the last used input vector.

        **Args:**

        * `d` : target (float or 1-dimensional array).
            Size depends on number of MLP outputs.

        **Returns:**
        
        * `e` : error used for update (float or 1-diemnsional array).
            Size correspond to size of input `d`.
            
        """
        # update output layer
        e = d - self.y
        error = np.copy(e)
        if self.outputs == 1:
            dw = self.mu * e * self.x   
            w = np.copy(self.w)[1:]
        else:
            dw = self.mu * np.outer(e, self.x)
            w = np.copy(self.w)[:,1:]
        self.w += dw
        # update hidden layers
        for l in reversed(self.layers):
            w, e = l.update(w, e) 
        return error


