# -*- coding: utf-8 -*-
#cython: embedsignature=True
#cython: profile=True
#cython: language_level=3

"""
A neural network for NLP tagging tasks.
It employs feature tables to store feature vectors for each token.
"""

import numpy as np
cimport numpy as np
cimport cython
from cpython cimport bool

#from itertools import izip
import logging

#ctypedef float FLOAT_t
ctypedef np.float_t FLOAT_t
#ctypedef int INT_t
ctypedef np.int_t INT_t
ctypedef np.double_t DOUBLE_t

# ----------------------------------------------------------------------
# Math functions

cdef logsumexp(np.ndarray a, axis=None):
    """Compute the log of the sum of exponentials of input elements.
    like: scipy.misc.logsumexp

    Parameters
    ----------
    a : array_like
        Input array.
    axis : int, optional
        Axis over which the sum is taken. By default `axis` is None,
        and all elements are summed.

    Returns
    -------
    res : ndarray
        The result, ``np.log(np.sum(np.exp(a)))`` calculated in a numerically
        more stable way.
    """
    if axis is None:
        a = a.ravel()
    else:
        a = np.rollaxis(a, axis)
    a_max = a.max(axis=0)
    return np.log(np.sum(np.exp(a - a_max), axis=0)) + a_max

cdef hardtanh(np.ndarray weights, inplace=False):
    """
    Hard hyperbolic tangent.
    If inplace is True, modifies the input weights, which will be faster.
    """
    if inplace:
        out = weights
    else:
        out = np.copy(weights)
    inds_greater = weights > 1
    inds_lesser = weights < -1
    out[inds_greater] = 1
    out[inds_lesser] = -1
    
    return out

cdef hardtanhd(np.ndarray[FLOAT_t, ndim=2] weights):
    """derivative of hardtanh"""
    cdef np.ndarray out = np.zeros_like(weights)
    inds = np.logical_and(-1.0 <= weights, weights <= 1.0)
    out[inds] = 1.0
    
    return out

# ----------------------------------------------------------------------

cdef class Network:
    
    # sizes and learning rates
    cdef readonly int word_window_size, input_size, hidden_size, output_size
    cdef public float learning_rate, learning_rate_features
    cdef public float decay_factor
    cdef public bool use_learning_rate_decay
    cdef readonly int features_per_token
    
    # lookup for fast access to all the token embeddings in a sentence
    cdef np.ndarray sentence_lookup
    
    # padding stuff
    cdef np.ndarray padding_left, padding_right
    cdef public np.ndarray pre_padding, pos_padding
    
    # weights, biases, calculated values
    cdef readonly np.ndarray hidden_weights, output_weights
    cdef readonly np.ndarray hidden_bias, output_bias
    cdef readonly np.ndarray input_values, hidden_values, layer2_values
    
    # feature tables
    cdef public list feature_tables
    
    # transitions
    cdef public float learning_rate_trans
    cdef public np.ndarray transitions
    
    # the score for a given path
    cdef readonly float answer_score
    
    # gradients
    cdef readonly np.ndarray net_gradients, trans_gradients
    cdef readonly np.ndarray input_sent_values, hidden_sent_values, layer2_sent_values
    
    # data for statistics during training. 
    cdef float error, accuracy, float_errors, sentence_accuracy
    cdef int num_tokens, skips
        
    # file where the network is saved
    cdef public str network_filename
    cdef public str network_text_filename
    
    # validation
    cdef list validation_sentences
    cdef list validation_tags

    @classmethod
    def create_new(cls, feature_tables, int word_window, int hidden_size, 
                 int output_size):
        """
        Creates a new neural network.
        """
        # sum the number of features in all tables 
        cdef int input_size = sum(table.shape[1] for table in feature_tables)
        input_size *= word_window
        
        # creates the weight matrices

        # set the seed for replicability
        #np.random.seed(42)

        # SENNA: centered uniform distribution with variance = 1/sqrt(fanin)
        # variance = 1/12 interval ^ 2
        # interval = 3.46 / fanin ^ 1/4
        #high = 1.732 / np.power(input_size, 0.25) # SENNA: 0.416
        high = 2.38 / np.sqrt(input_size) # [Bottou-88]
        #high = 0.1              # Fonseca
        hidden_weights = np.random.uniform(-high, high, (hidden_size, input_size))
        hidden_bias = np.random.uniform(-high, high, (hidden_size))
        #high = 1.732 / np.power(hidden_size, 0.25) # SENNA
        high = 2.38 / np.sqrt(hidden_size) # [Bottou-88]
        #high = 0.1              # Fonseca
        output_weights = np.random.uniform(-high, high, (output_size, hidden_size))
        output_bias = np.random.uniform(-high, high, (output_size))
        
        high = 1.0
        # +1 is due for the initial transition
        transitions = np.random.uniform(-high, high, (output_size + 1, output_size))

        net = Network(word_window, input_size, hidden_size, output_size,
                      hidden_weights, hidden_bias, output_weights, output_bias,
                      transitions)
        net.feature_tables = feature_tables
        
        return net
        
    def __init__(self, word_window, input_size, hidden_size, output_size,
                 hidden_weights, hidden_bias, output_weights, output_bias,
                 transitions=None):
        """
        This function isn't expected to be directly called.
        Instead, use the classmethods load_from_file or 
        create_new.
        
        :param transitions: transition weights. If None uses
            Window Level Likelihood instead of Sentence Level Likelihood.
        """
        self.learning_rate = 0
        self.learning_rate_features = 0
        
        self.word_window_size = word_window
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.features_per_token = input_size / word_window
        
        # A_i_j score for jumping from tag i to j
        # A_0_i = transitions[-1]
        self.transitions = transitions

        self.hidden_weights = hidden_weights
        self.hidden_bias = hidden_bias
        self.output_weights = output_weights
        self.output_bias = output_bias
        
        self.validation_sentences = None
        self.validation_tags = None
        
        self.use_learning_rate_decay = False

    def description(self):
        """
        Returns a textual description of the network.
        """
        table_dims = [str(t.shape[1]) for t in self.feature_tables]
        table_dims =  ', '.join(table_dims)
        
        desc = """
Word window size: %d
Feature table sizes: %s
Input layer size: %d
Hidden layer size: %d
Output size: %d
""" % (self.word_window_size, table_dims, self.input_size, self.hidden_size, self.output_size)
        
        return desc
    
    def _create_sentence_lookup(self, np.ndarray sentence):
        """
        Create a lookup matrix with the embeddings values for all tokens in a sentence.
        """        
        #print (sentence)
        cdef np.ndarray padded_sentence = np.concatenate((self.pre_padding,
                                                          sentence,
                                                          self.pos_padding))
        
        # make sure it works on 32 bit python installations
        padded_sentence = padded_sentence.astype(int)
        #padded_sentence = padded_sentence.astype(np.int64)
        
        self.sentence_lookup = np.empty((len(padded_sentence), self.features_per_token))
        ind_from = 0
        
        for i, table in enumerate(self.feature_tables):
            num_dims = table.shape[1]
            ind_to = ind_from + num_dims
            
            token_indices = padded_sentence[:, i]
            embeddings = table.take(token_indices, axis=0)
            self.sentence_lookup[:, ind_from:ind_to] = embeddings
            
            ind_from = ind_to
    
    property padding_left:
        """
        The padding element filling the "void" before the beginning
        of the sentence.
        """
        def __get__(self):
            return self.padding_left
    
        def __set__(self, np.ndarray padding_left):
            self.padding_left = padding_left
            self.pre_padding = np.array((self.word_window_size // 2) * [padding_left])
    
    property padding_right:
        """
        The padding element filling the "void" after the end
        of the sentence.
        """
        def __get__(self):
            return self.padding_right
    
        def __set__(self, np.ndarray padding_right):
            self.padding_right = padding_right
            self.pos_padding = np.array((self.word_window_size // 2) * [padding_right])
    
    def tag_sentence(self, np.ndarray sentence):
        """
        Runs the network for each element in the sentence and returns 
        the sequence of tags.
        
        :param sentence: a 2-dim numpy array, where each item encodes a token.
        """
        scores = self._tag_sentence(sentence)
        # computes full score, combining ftheta and A (if SLL)
        return self._viterbi(scores)

    def _tag_sentence(self, np.ndarray sentence, tags=None):
        """
        Runs the network for each element in the sentence and returns 
        the sequence of tags.
        
        :param sentence: a 2-dim numpy array, where each item encodes a token.
        :param tags: the correct tags (needed when training)
        :return: a (len(sentence), output_size) array with the scores for all tokens
        """
        cdef np.ndarray answer
        cdef np.ndarray input_data
        # scores[t, i] = ftheta_i,t = score for i-th tag, t-th word
        cdef np.ndarray scores = np.empty((len(sentence), self.output_size))
        
        training = tags is not None
        if training:
            self.input_sent_values = np.empty((len(sentence), self.input_size))
            # layer2_values at each token in the correct path
            self.layer2_sent_values = np.empty((len(sentence), self.hidden_size))
            # hidden_values at each token in the correct path
            self.hidden_sent_values = np.empty((len(sentence), self.hidden_size))
        
        self._create_sentence_lookup(sentence)

        # run through all windows in the sentence
        for i in range(len(sentence)):
            input_data = self.sentence_lookup[i:i + self.word_window_size].flatten()
            
            # (hidden_size, input_size) . input_size = hidden_size
            self.layer2_values = self.hidden_weights.dot(input_data) + self.hidden_bias
            self.hidden_values = hardtanh(self.layer2_values, inplace=not training)
            output = self.output_weights.dot(self.hidden_values) + self.output_bias
            scores[i] = output
            
            if training:
                self.input_sent_values[i] = input_data
                self.layer2_sent_values[i] = self.layer2_values
                self.hidden_sent_values[i] = self.hidden_values
        
        if training:
            if self._calculate_gradients_sll(tags, scores):
                self._backpropagate(sentence)

        return scores
    
    def _calculate_delta(self, scores):
        """
        Calculates a matrix with the scores for all possible paths at all given
        points (tokens).
        In the returned matrix, delta[i][j] means the sum of all scores 
        ending in token i with tag j (delta_i(j) in eq. 14 in the paper)
        """
        # logadd for first token. the transition score of the starting tag must be used.
        # it turns out that logadd = log(exp(score)) = score
        # (use long double because taking exp's leads to very very big numbers)
        # scores[t][k] = ftheta_k,t
        delta = scores
        
        # transitions[-1] represents initial transition, A_0,i in paper (mispelled as A_i,0)
        # delta_0(k) = ftheta_k,0 + A_0,i
        delta[0] += self.transitions[-1]
        
        # logadd for the remaining tokens
        # delta_t(k) = ftheta_k,t + logadd_i(delta_t-1(i) + A_i,k)
        #            = ftheta_k,t + log(Sum_i(exp(delta_t-1(i) + A_i,k)))
        transitions = self.transitions[:-1].T # A_k,i
        
        for token in xrange(1, len(delta)):
            # sum by rows
            logadd = logsumexp(delta[token - 1] + transitions, 1)
            delta[token] += logadd
        
        return delta

    @cython.boundscheck(False)
    def _calculate_gradients_sll(self, tags, scores):
        """
        Calculates the output and transition deltas for each token, using Sentence Level Likelihood.
        The aim is to minimize the cost:
        C(theta,A) = logadd(scores for all possible paths) - score(correct path)
        
        :returns: if True, normal gradient calculation was performed.
            If False, the error was too low and weight correction should be
            skipped.
        """
        cdef np.ndarray[DOUBLE_t, ndim=2] delta # (len(sentence), output_size)
        cdef np.ndarray[DOUBLE_t, ndim=2] delta_softmax # (output_size, output_size)
        
        # ftheta_i,t = network output for i-th tag, at t-th word
        # s = Sum_i(A_tags[i-1],tags[i] + ftheta_i,i), i < len(sentence)   (12)
        correct_path_score = 0
        last_tag = self.output_size
        for tag, net_scores in zip(tags[:-2], scores[:-2]):
            #print(tags, last_tag, tag, self.transitions)
            trans = 0 if self.transitions is None else self.transitions[last_tag, tag]
            #print ('trans:',trans)
            #print ('net:',tag, net_scores[tag])
            #print
            correct_path_score += trans + net_scores[tag]
            last_tag = tag 
        
        # delta[t] = delta_t in equation (14)
        delta = self._calculate_delta(scores)
        # logadd_i(delta_T(i)) = log(Sum_i(exp(delta_T(i))))
        # Sentence-level Log-Likelihood (SLL)
        # C(ftheta,A) = logadd_j(s(x, j, theta, A)) - score(correct path)
        #print ': ', np.log(np.sum(np.exp(delta[-1])))
        #print '> ', correct_path_score
        error = np.log(np.sum(np.exp(delta[-1]))) - correct_path_score
        #print error
        #error = logsumexp(delta[-1]) - correct_path_score
        self.error += error
        
        # if the error is too low, don't bother training (saves time and avoids
        # overfitting). An error of 0.01 means a log-prob of -0.01 for the right
        # tag, i.e., more than 99% probability
        # error 0.69 -> 50% probability for right tag (minimal threshold)
        # error 0.22 -> 80%
        # error 0.1  -> 90%
        if error <= 0.01:
            self.skips += 1
            return False
        
        # initialize gradients
        # dC / dftheta
        self.net_gradients = np.zeros((len(tags), self.output_size))
        # dC / dA
        #self.trans_gradients = np.zeros_like(self.transitions, float)
        self.trans_gradients = np.zeros_like(self.transitions, np.float64)
        
        # things get nasty from here
        # refer to the papers to understand what exactly is going on
        
        # compute the gradients for the last token
        # dC_logadd / ddelta_T(i) = e(delta_T(i))/Sum_k(e(delta_T(k)))
        # Compute it using the log:
        # log(e(delta_T(i))/Sum_k(e(delta_T(k)))) =
        # log(e(delta_T(i))) - log(Sum_k(e(delta_T(k)))) =
        # delta_T(i) - logsumexp(delta_T(k))
        # dC_logadd / ddelta_T(i) = e(delta_T(i) - logsumexp(delta_T(k)))
        sumlogadd = logsumexp(delta[-1])
        # negative gradients
        self.net_gradients[-1] = -np.exp(delta[-1] - sumlogadd)

        transitions_t = 0 if self.transitions is None else self.transitions[:-1].T
        
        # delta[i][j]: sum of scores of all path that assign tag j to ith-token

        # now compute the gradients for the other tokens, from last to first
        for t in range(len(scores) - 2, -1, -1):
            
            # sum the scores for all paths ending with each tag i at token t
            # with the transitions from tag i to the next tag j
            # Obtained by transposing twice
            # [delta_t-1(i)+A_j,i]T
            path_scores = (delta[t] + transitions_t).T

            # normalize over all possible tag paths using a softmax,
            # computed using log.
            # the log of the sums of exps, summed by column
            log_sum_scores = logsumexp(path_scores, 0)
            
            # softmax is the division of an exponential by the sum of all exponentials
            # (yields a probability)
            # e(delta_t-1(i)+A_i,j) / Sum_k e(delta_t-1(k)+A_k,j)
            delta_softmax = np.exp(path_scores - log_sum_scores)

            # multiply each value in the softmax by the gradient at the next tag
            # dC_logadd / ddelta_t(i) * delta_softmax
            # Attardi: negative since net_gradients[t + 1] already negative
            grad_times_softmax = self.net_gradients[t + 1] * delta_softmax
            # dC / dA_i,j
            self.trans_gradients[:-1, :] += grad_times_softmax
            
            # sum all transition gradients by row to find the network gradients
            # Sum_j(dC_logadd / ddelta_t(j) * delta_softmax)
            # Attardi: negative since grad_times_softmax already negative
            self.net_gradients[t] = np.sum(grad_times_softmax, 1)

        # find the gradients for the starting transition
        # there is only one possibility to come from, which is the sentence start
        self.trans_gradients[-1] = self.net_gradients[0]
        
        # now, add +1 to the correct path
        last_tag = self.output_size
        for token, tag in enumerate(tags):
            self.net_gradients[token][tag] += 1 # negative gradient
            if self.transitions is not None:
                self.trans_gradients[last_tag][tag] += 1 # negative gradient
            last_tag = tag
        
        return True

    @cython.boundscheck(False)
    def _calculate_gradients_wll(self, tags, scores):
        """
        Calculates the output for each token, using Word Level Likelihood.
        The aim is to minimize the word-level log-likelihood:
        C(ftheta) = logadd_j(ftheta_j) - ftheta_y,
        where y is the sequence of correct tags
        
        :returns: if True, normal gradient calculation was performed.
            If False, the error was too low and weight correction should be
            skipped.
        """
        # compute the negative gradient with respect to ftheta
        # dC / dftheta_i = e(ftheta_i)/Sum_k(e(ftheta_k))
        exponentials = np.exp(scores)
        # FIXME: use logsumexp
        # ((len(sentence), self.output_size))
        self.net_gradients = -(exponentials.T / exponentials.sum(1)).T

        # correct path and its gradient
        correct_path_score = 0
        token = 0
        for tag, net_scores in zip(tags, scores):
            self.net_gradients[token][tag] += 1 # negative gradient
            token += 1
            correct_path_score += net_scores[tag]

        # C(ftheta) = logadd_j(ftheta_j) - score(correct path)
        #error = np.log(np.sum(np.exp(scores))) - correct_path_score
        error = logsumexp(scores) - correct_path_score
        # approximate
        #error = np.max(scores) - correct_path_score
        self.error += error

        return True

    @cython.boundscheck(False)
    def _viterbi(self, np.ndarray[FLOAT_t, ndim=2] scores):
        """
        Performs a Viterbi search over the scores for each tag using
        the transitions matrix. If a matrix wasn't supplied, 
        it will return the tags with the highest scores individually.
        """
        # pretty straightforward
        if self.transitions is None or len(scores) == 1:
            return scores.argmax(1)
            
        path_scores = np.empty_like(scores)
        #path_backtrack = np.empty_like(scores, np.int64)
        path_backtrack = np.empty_like(scores, int)
        
        # now the actual Viterbi algorithm
        # first, get the scores for each tag at token 0
        # the last row of the transitions table has the scores for the first tag
        path_scores[0] = scores[0] + self.transitions[-1]
        
        output_range = np.arange(self.output_size) # outside loop. Attardi
        transitions = self.transitions[:-1]        # idem

        cdef int i
        for i in xrange(1, len(scores)):
            
            # each line contains the score until each tag t plus the transition to each other tag t'
            prev_score_and_trans = (path_scores[i - 1] + transitions.T).T
            
            # find the previous tag that yielded the max score
            path_backtrack[i] = prev_score_and_trans.argmax(0)
            path_scores[i] = prev_score_and_trans[path_backtrack[i], 
                                                  output_range] + scores[i]
            
        # now find the maximum score for the last token and follow the backtrack
        #answer = np.empty(len(scores), dtype=np.int64)
        answer = np.empty(len(scores), dtype=int)
        answer[-1] = path_scores[-1].argmax()
        self.answer_score = path_scores[-1][answer[-1]]
        previous_tag = path_backtrack[-1][answer[-1]]
        
        for i in range(len(scores) - 2, 0, -1):
            answer[i] = previous_tag
            previous_tag = path_backtrack[i][previous_tag]
        
        answer[0] = previous_tag
        return answer
    
    def set_validation_data(self, list validation_sentences=None,
                            list validation_tags=None):
        """
        Sets the data to be used during validation. If this function is not
        called before training, the training data is used to measure performance.
        
        :param validation_sentences: sentences to be used in validation.
        :param validation_tags: tags for the validation sentences.
        """
        self.validation_sentences = validation_sentences
        self.validation_tags = validation_tags 
    
    def set_learning_rate_decay(self, float decay_factor=1.0):
        """
        Sets the network to use learning rate decay. 
        
        The learning rate at each iteration t is determined as:
        initial_rate / (1 + t * decay_factor)
        
        with t starting from 0
        """
        self.use_learning_rate_decay = True
        self.decay_factor = decay_factor
    
    def decrease_learning_rates(self, epoch):
        """
        Apply the learning rate decay, if the network was configured to use it.
        """
        if not self.use_learning_rate_decay or epoch == 0:
            return
        
        # multiplying the last rate by this adjustment is equivalent to
        # initial_rate / (1 + t * decay_factor)
        # and we don't need to store the initial rates
        factor = (1.0 + (epoch - 1) * self.decay_factor) / (1 + epoch * self.decay_factor)
        self.learning_rate *= factor
        self.learning_rate_features *= factor
        if self.transitions is not None:
            self.learning_rate_trans *= factor
    
    def train(self, list sentences, list tags, 
              int epochs, int epochs_between_reports=0,
              float desired_accuracy=0):
        """
        Trains the network to tag sentences.
        
        :param sentences: a list of 2-dim numpy arrays, where each item 
            encodes a sentence. Each item in a sentence has the 
            indices to its features.
        :param tags: a list of 1-dim numpy arrays, where each item has
            the tags of the sentences.
        :param epochs: number of training epochs
        :param epochs_between_reports: number of epochs to wait between
            reports about the training performance. 0 means no reports.
        :param desired_accuracy: training stops if the desired accuracy
            is reached. Ignored if 0.
        """
        logger = logging.getLogger("Logger")
        logger.info("Training for up to %d epochs" % epochs)
        top_accuracy = 0
        last_accuracy = 0
        last_error = np.Infinity
        self.num_tokens = sum(len(sent) for sent in sentences)
        
        if self.validation_sentences is None:
            self.validation_sentences = sentences
            self.validation_tags = tags
        
        for i in range(epochs):
            self.decrease_learning_rates(i)      
            self._train_epoch(sentences, tags)
            self._validate()
            
            # normalize error
            self.error = self.error / self.num_tokens if self.num_tokens else np.Infinity
            
            # Attardi: save model
            if self.accuracy > top_accuracy:
                top_accuracy = self.accuracy
                self.save()
                logger.debug("Saved model")
            elif self.use_learning_rate_decay:
                # this iteration didn't bring improvements; load the last saved model
                # before continuing training with a lower rate
                self._load_parameters()
                        
            if (epochs_between_reports > 0 and i % epochs_between_reports == 0) \
                or self.accuracy >= desired_accuracy > 0 \
                or (self.accuracy < last_accuracy and self.error > last_error):
                
                self._print_epoch_report(i + 1)
                
                if self.accuracy >= desired_accuracy > 0\
                        or (self.error > last_error and self.accuracy < last_accuracy):
                    break
                
            last_accuracy = self.accuracy
            last_error = self.error
        
        self.num_tokens = 0
            
    def _print_epoch_report(self, int num):
        """
        Reports the status of the network in the given training
        epoch, including error and accuracy.
        """
        logger = logging.getLogger("Logger")
        logger.info("%d epochs   Error: %f   Accuracy: %f   " \
            "%d corrections skipped   " \
            "learning rate: %f" % (num,
                                   self.error,
                                   self.accuracy,
                                   self.skips,
                                   self.learning_rate))
    
    def _train_epoch(self, list sentences, list tags):
        """
        Trains for one epoch with all examples.
        """
        self.error = 0
        self.skips = 0
        self.float_errors = 0
        
        # shuffle data
        # get the random number generator state in order to shuffle
        # sentences and their tags in the same order
        random_state = np.random.get_state()
        np.random.shuffle(sentences)
        np.random.set_state(random_state)
        np.random.shuffle(tags)
        
        for sent, sent_tags in zip(sentences, tags):
            try:
                self._tag_sentence(sent, sent_tags)
            except FloatingPointError:
                # just ignore the sentence in case of an overflow
                self.float_errors += 1

    def _validate(self):
        """Perform validation on validation data and estimate accuracy"""
        hits = 0
        
        for sent, gold_tags in zip(self.validation_sentences, self.validation_tags):
            answer = self.tag_sentence(sent)
            hits += np.count_nonzero(answer == gold_tags)
        
        # self.num_tokens stores number of tokens in training sentences
        num_tokens = sum(len(sent) for sent in self.validation_sentences)
        self.accuracy = float(hits) / num_tokens

    def _backpropagate(self, sentence):
        """
        Backpropagate the gradients of the cost.
        """
        # f_1 = input_sent_values
        # f_2 = M_1 f_1 + b_2 = layer2_values
        # f_3 = hardTanh(f_2) = hidden_values
        # f_4 = M_2 f_3 + b_4

        # For l = 4..1 do:
        # dC / dtheta_l = df_l / dtheta_l dC / df_l		(19)
        # dC / df_l-1 = df_l / df_l-1 dC / df_l			(20)

        """
        Compute the gradients of the cost for each layer
        """
        # layer 4: output layer
        # dC / dW_4 = dC / df_4 f_3.T				(22)
        # (len, output_size).T (len, hidden_size) = (output_size, hidden_size)
        cdef np.ndarray[FLOAT_t, ndim=2] output_deltas
        output_deltas = self.net_gradients.T.dot(self.hidden_sent_values)

        # dC / db_4 = dC / df_4					(22)
        # (output_size) += ((len(sentence), output_size))
        # sum by column, i.e. all changes through the sentence
        output_bias_deltas = self.net_gradients.sum(0)

        # dC / df_3 = M_2.T dC / df_4				(23)
        #  (len, output_size) (output_size, hidden_size) = (len, hidden_size)
        dCdf_3 = self.net_gradients.dot(self.output_weights)

        # layer 3: HardTanh layer
        # no weights to adjust

        # dC / df_2 = hardtanhd(f_2) * dC / df_3
        # (len, hidden_size) (len, hidden_size)
        # FIXME: this goes quickly to 0.
        dCdf_2 = hardtanhd(self.layer2_sent_values) * dCdf_3

        # df_2 / df_1 = M_1

        # layer 2: linear layer
        # dC / dW_2 = dC / df_2 f_1.T				(22)
        cdef np.ndarray[FLOAT_t, ndim=2] hidden_deltas
        # (len, hidden_size).T (len, input_size) = (hidden_size, input_size)
        hidden_deltas = dCdf_2.T.dot(self.input_sent_values)

        # dC / db_2 = dC / df_2					(22)
        # sum by column contribution by each token
        hidden_bias_deltas = dCdf_2.sum(0)

        # dC / df_1 = M_1.T dC / df_2
        cdef np.ndarray[FLOAT_t, ndim=2] input_gradients
        # (len, hidden_size) (hidden_size, input_size) = (len, input_size)
        input_gradients = dCdf_2.dot(self.hidden_weights)

        """
        Adjust the weights. 
        """
        self.output_weights += output_deltas * self.learning_rate
        self.output_bias += output_bias_deltas * self.learning_rate
        self.hidden_weights += hidden_deltas * self.learning_rate
        self.hidden_bias += hidden_bias_deltas * self.learning_rate

        """
        Adjust the features indexed by the input window.
        """
        # the deltas that will be applied to the feature tables
        # they are in the same sequence as the network receives them, i.e.
        # [token1-table1][token1-table2][token2-table1][token2-table2] (...)
        # input_size = num features * window (e.g. 60 * 5). Attardi
        cdef np.ndarray[FLOAT_t, ndim=2] input_deltas
        # (len, input_size)
        input_deltas = input_gradients * self.learning_rate_features
        
        padded_sentence = np.concatenate((self.pre_padding,
                                          sentence,
                                          self.pos_padding))
        
        cdef np.ndarray[INT_t, ndim=1] features
        cdef np.ndarray[FLOAT_t, ndim=2] table
        cdef int start, end, t
        cdef int i, j

        for i, w_deltas in enumerate(input_deltas):
            # for each window (w_deltas: 300, features: 5)
            # this tracks where the deltas for the next table begins
            start = 0
            for features in padded_sentence[i:i+self.word_window_size]:
                # select the columns for each feature_tables (t: 3)
                for t, table in enumerate(self.feature_tables):
                    end = start + table.shape[1]
                    table[features[t]] += w_deltas[start:end]
                    start = end

        # Adjusts the transition scores table with the calculated gradients.
        if self.transitions is not None:
            self.transitions += self.trans_gradients * self.learning_rate_trans

    def _load_parameters(self):
        """
        Loads weights, feature tables and transition tables previously saved.
        """
        data = np.load(self.network_filename)
        self.hidden_weights = data['hidden_weights']
        self.hidden_bias = data['hidden_bias']
        self.output_weights = data['output_weights']
        self.output_bias = data['output_bias']
        self.feature_tables = list(data['feature_tables'])
        
        # check if transitions isn't None (numpy saves everything as an array)
        if data['transitions'].shape != ():
            self.transitions = data['transitions']
        else:
            self.transitions = None
    
    def save(self):
        """
        Saves the neural network to a file.
        It will save the weights, biases, sizes, padding, 
        and feature tables.
        """
        np.savez(self.network_filename, hidden_weights=self.hidden_weights,
                 output_weights=self.output_weights,
                 hidden_bias=self.hidden_bias, output_bias=self.output_bias,
                 word_window_size=self.word_window_size, 
                 input_size=self.input_size, hidden_size=self.hidden_size,
                 output_size=self.output_size, padding_left=self.padding_left,
                 padding_right=self.padding_right, transitions=self.transitions,
                 feature_tables=self.feature_tables)


    @classmethod
    def load_from_file(cls, filename):
        """
        Loads the neural network from a file.
        It will load weights, biases, sizes, padding, 
        and feature tables.
        """
        data = np.load(filename)
        #print filename
        # cython classes don't have the __dict__ attribute
        # so we can't do an elegant self.__dict__.update(data)
        hidden_weights = data['hidden_weights']
        hidden_bias = data['hidden_bias']
        output_weights = data['output_weights']
        output_bias = data['output_bias']
        
        word_window_size = data['word_window_size']
        input_size = data['input_size']
        hidden_size = data['hidden_size']
        output_size = data['output_size']
        if 'transitions' in data:
            transitions = data['transitions']
        else:
            transitions = None

        nn = Network(word_window_size, input_size, hidden_size, output_size,
                     hidden_weights, hidden_bias, output_weights, output_bias,
                     transitions)
        
        nn.padding_left = data['padding_left']
        nn.padding_right = data['padding_right']
        nn.pre_padding = np.array((nn.word_window_size // 2) * [nn.padding_left])
        nn.pos_padding = np.array((nn.word_window_size // 2) * [nn.padding_right])
        nn.feature_tables = list(data['feature_tables'])
        nn.network_filename = filename
        
        return nn 
        
# include the files for other networks
# this comes here after the Network class has already been defined
include "networkconv.pyx"
include "networkdependencyconv.pyx"
