import os
import math
import time

import numpy as np

from matplotlib import pyplot as plt
from sklearn import svm
from scipy.stats import multivariate_normal, norm
from IPython.display import clear_output

from .enumtypes import LossType
from .model import Model
from .report import Report
from .utils import is_notebook

def cls():
  """
  Clears the terminal screen.
  """
  if os.name == 'nt':  # For Windows
    _ = os.system('cls')
  else:  # For Linux and Mac
    _ = os.system('clear')


def h_error_rate(h_bias, h_weights, cov):
  """
  This function calculates the probability of error for a linear classifier :math:`h` trained by Dataset :math:`D` generated by a Gaussian distribution, given:
    - weights vector :math:`\mathbf{w}` of the classifier :math:`h`;
    - the bias :math:`b` of the classifier :math:`h`;
    - the covariance matrix :math:`\Sigma` used to generate the trainning dataset :math:`D`.

  Linear Classifier?
    - ":math:`h(\mathbf{x}) = \mathbf{w} \cdot \mathbf{x} + b`" is a linear classifier.
    - The hyperplane  :math:`h` defined by this equation represents the decision boundary that separates classes in a linearly separable space.
    - The hypothesis space :math:`H` consists of all possible linear classifiers.

  The linear classifier function :math:`h(\mathbf{x})` returns a scalar value that represents a linear combination of the input data point :math:`\mathbf{x}` and the weight vector :math:`\mathbf{w}`, plus a bias term :math:`b`. In the context of binary classification (wich is the case in this experiment), this value can be interpreted as the raw score or decision boundary for classifying an input. If you apply a threshold, such as 0, you can make a binary decision:
    - If :math:`h(\mathbf{x}) \geq 0`, then classify the input as one class (e.g., class 1).
    - If :math:`h(\mathbf{x}) < 0`, then classify the input as the other class (e.g., class 0).


  The minimum probability of error for a linear classifier :math:`h` trained by Dataset :math:`D` generated by a Gaussian distribution is given by:
    - :math:`P(Error) = \\frac{1}{2} (1 - \Phi({Distance}(+))) + \\frac{1}{2}(1 - \Phi(Distance(-)))`
    - :math:`P(Error) = \\frac{1}{2} (1 - \Phi( \\frac{|1 + b - (w_0 + ... + w_{d-1})|}{\sqrt{\Delta}})) + \\frac{1}{2}(1 - \Phi(\\frac{|1 - b - (w_0 + ... + w_{d-1})|}{\sqrt{\Delta}}))`

  Where:
    - :math:`d` is the dimensionality of the classifier :math:`h`
    - :math:`\mathbf{w}` is the weights vector of the classifier :math:`h`
    - :math:`\mathbf{w} = [w_0, ..., w_{d-1}]`
    - :math:`b` is the bias of the classifier :math:`h`
    - :math:`\lambda` is the Lower-triangular Cholesky factor of :math:`\Sigma`
    - :math:`\mathbf{\delta} = [\delta_i]` for :math:`i \in [1,...,d] = [\delta_0,...,\delta_{d}]`
    - :math:`\mathbf{\delta} = (\mathbf{w}|[-1]) \cdot \lambda^T = [w_0, ..., w_{d-1},-1]\cdot \lambda^T`
    - :math:`\Delta = \sum_{\delta_i \in \\mathbf{\delta}} \delta_i^2 = \delta_0^2 + ... + \delta_{d}^2`
    - :math:`\Phi(\cdot)` is the cumulative distribution function of a Gaussian with zero mean and variance one.
    - :math:`Distance(+)` and :math:`Distance(-)` are the distances from the hyperplane :math:`h` to the mean of the positive and negative classes, :math:`\mathbf{\mu_+}` and :math:`\mathbf{\mu_-}`, respectively.
    - :math:`\mathbf{\mu_+}` : mean vector for class :math:`(+)`
    - :math:`\mathbf{\mu_-}` : mean vector for class :math:`(-)`
    - :math:`\mathbf{\mu_+} = \\bigcup_{i=1}^{d} \mu_{+i}`
    - :math:`\mathbf{\mu_+} = \\bigcup_{i=1}^{d} \mu_{-i}`
    - :math:`\mu_{+i} = 1` and :math:`\mu_{-i} = -1`, for :math:`i \in [1,...,d]`
    - :math:`\mathbf{\mu_-} = [\mu_{-i}]` for :math:`i \in [1,...,d] = [\mu_{-1}, ..., \mu_{-d}] = [1,...,1]`
    - :math:`\mathbf{\mu_+} = [\mu_{+i}]` for :math:`i \in [1,...,d] = [\mu_{+1}, ..., \mu_{+d}] = [-1,...,-1]`


  Parameters:
    h_bias (float): bias of the classifier :math:`h(\mathbf{x})`
    h_weights (list): weights of the classifier :math:`h(\mathbf{x})`
    cov (list[list]): covariance matrix of the gaussian samples used to train the model

  Returns:
    float: probability of error for the classifier :math:`h(\mathbf{x})` trained with the gaussian samples generated with the covariance matrix :math:`\Sigma` and means :math:`\mu_+=[1,...,1]` and :math:`\mu_=[-1,...,-1]`



  """

  lamb = np.linalg.cholesky(cov)

  h_weights_and_minus_one = h_weights + [-1]
  delta = np.sum(np.power([np.dot(h_weights_and_minus_one, np.array(lamb).T[i]) for i in range(len(h_weights_and_minus_one))], 2))

  arg1 = abs(np.sum(h_weights_and_minus_one) + h_bias) / math.sqrt(delta)
  arg2 = abs(np.sum(h_weights_and_minus_one) - h_bias) / math.sqrt(delta)
  pr1 = 1-norm.cdf(arg1)
  pr2 = 1-norm.cdf(arg2)
  pr = 0.5*(pr1+pr2)

  return pr


def loss_empirical(clf, dataset_test):
  """
  this function calculates an empirical loss :math:`\hat{L}(h)`, given:
    - SVM trained model containing the best empirical classifier :math:`\hat{h}^{(D)}` in the dictionary :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples.
    - a test set :math:`D_{test} = D_{(+)test} \cup D_{(-)test}`

  For each classifier :math:`h` in :math:`H`, we can calculate its empirical error rate :math:`\hat{L}(h)`, i.e., its proportion of misclassified observations in the dataset :math:`D`:
    - :math:`\hat{L}(h) = \\frac{1}{n} \sum_{i=1}^{n} 1(y_i \\neq h(x_i))`

  Let :math:`\hat{h}^{(D)}` be the best empirical classifier in the dictionary :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples, i.e., the classifier in :math:`H` with the minimum empirical error rate. Under these conditions, we can define two error rates:
    - :math:`\hat{L}(\hat{h}^{(D)}) =` the empirical error rate of the classifier :math:`\hat{h}^{(D)}`, when :math:`D_{test} = D_{train}`
    - :math:`\hat{L}(\hat{h}^{(D)},D') =` the empirical error rate of the classifier :math:`\hat{h}^{(D)}` using a test dataset :math:`D'`, when :math:`D_{test} \\neq D_{train}`

  Parameters:
    clf (sklearn.svm.SVC): SVM trained model containing the best empirical classifier :math:`\hat{h}^{(D)}` in the dictionary :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples.
    dataset_test (dict): a test set :math:`D_{test} = D_{(+)test} \cup D_{(-)test}`

  Returns:
    float: empirical error rate of the classifier :math:`\hat{h}^{(D)}`, :math:`\hat{L}(\hat{h}^{(D)})` when :math:`D_{test} = D_{train}` or :math:`\hat{L}(\hat{h}^{(D)},D')` when :math:`D_{test} \\neq D_{train}`

  """

  dims_to_remove = [i for i in range(len(dataset_test['data'][0])-1,len(clf.coef_[0])-1,-1)]

  X_test, y_test = np.delete(dataset_test['data'], dims_to_remove, 1) , dataset_test['target']

  return 1 - clf.score(X_test,y_test)


def linear_svm_train(dim, training_set):
  """This function trains a linear SVM model and find the best empirical linear classifier :math:`\hat{h}^{(D)}` in the hypothesis space :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples, given:
    - Training set :math:`D = D_{(+)train} \cup D_{(-)train}`
    - dimensionality :math:`d`, :math:`d` < nuber of features in the dataset

  Linear Classifier?
    - ":math:`h(\mathbf{x}) = \mathbf{w} \cdot \mathbf{x} + b`" is a linear classifier.
    - The hyperplane :math:`h` defined by this equation represents the decision boundary that separates classes in a linearly separable space.
    - The hypothesis space :math:`H` consists of all possible linear classifiers.

  :param self: this simulator
  :type self: Simulator
  :param dim: dimensionality of the model, :math:`d` features
  :type dim: int
  :param training_set: dataset :math:`D` to train the model
  :type training_set: dict
  :return: trained SVM model containing the best empirical hyperplane :math:`h` in the hypothesis space :math:`H` for the given dataset :math:`D` with :math:`d` features and :math:`n` samples
  :rtype: sklearn.svm._classes.SVC

  """

  # remove k columns from sample_data for model.dim-k features
  dims_to_remove = [i for i in range(len(training_set['data'][0]) - 1, dim - 1, -1)]

  X_train, y_train = np.delete(training_set['data'], dims_to_remove, 1) , training_set['target']

  # train
  C = 1.0  # SVM regularization parameter
  clf = svm.SVC(kernel='linear', C=C, probability=True) #set svm linear separator
  clf.fit(X_train, y_train.ravel()) #train model

  return clf


def loss_theoretical(h_clf, cov):
  """
  Using probability theory, calculates the theoretical loss for given:
    - sklearn SVM trained model containing the best linear classifier :math:`h` in the hypothesis space :math:`H` for the given training dataset :math:`D` with :math:`d` features and :math:`n` samples
    - covariance matrix of the gaussian samples used to train the SVM model

  Linear Classifier?
    - ":math:`h(\mathbf{x}) = \mathbf{w} \cdot \mathbf{x} + b`" is a linear classifier.
    - The hyperplane :math:`h` defined by this equation represents the decision boundary that separates classes in a linearly separable space.
    - The hypothesis space :math:`H` consists of all possible linear classifiers.

  Let :math:`\hat{h}^{(D)}` be the best empirical classifier in the dictionary :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples, i.e., the classifier in :math:`H` with the minimum empirical error rate. Under these conditions, we can define theoretical error rate as:
    - :math:`L(\hat{h}^{(D)}) =` the theoretical error rate of the classifier :math:`\hat{h}^{(D)}`

  Parameters:
    h_clf(sklearn.svm._classes.SVC): SVM trained classifier
    cov(list[list] or np.ndarray): covariance matrix :math:`\Sigma` of the gaussian samples used to train the SVM model

  Returns:
    float: theoretical loss for classifier :math:`\hat{h}^{(D)}`

  Raises:
    TypeError:
      if h_clf is not a sklearn.svm._classes.SVC
      if cov is not a list of lists or np.ndarray

    ValueError:
      if h_clf is not a linear classifier
      if h_clf is not a trained scikit-learn SVM model
      if cov is not a positive definite matrix

  """

  if not isinstance(h_clf, svm.SVC):
    raise TypeError('h_clf must be a sklearn.svm._classes.SVC')

  if not isinstance(cov, list) and not isinstance(cov, np.ndarray):
    raise TypeError('cov must be a list of lists or np.ndarray')

  if not np.all(np.linalg.eigvals(cov) > 0):
    raise ValueError('cov must be a positive definite matrix')

  if h_clf.kernel != 'linear':
    raise ValueError('h_clf must be a linear classifier')

  if not hasattr(h_clf, 'coef_'):
    raise ValueError('h_clf must be a trained scikit-learn linear SVM model')

  coef_ = h_clf.coef_[0]

  bias = (h_clf.intercept_[0]) / coef_[len(coef_) - 1]

  weights = [-coef_[i]/coef_[len(coef_)-1] for i in range(len(coef_)-1)]

  return h_error_rate(bias, weights, cov)


def loss_bayes_analytical(cov):
  """calculate the theoretical Bayes Error Rate (or Bayes Risk) analytically, given:
    - covariance matrix :math:`\Sigma` of a Gaussian distribution

  The Bayes Error (also: Bayes Risk) is defined as the minimum theoretical achievable error rate:
    - :math:`\min_{h\in H} L(h) =` the smallest theoretically achievable error rate for a classifier :math:`h` in dictionary :math:`H`.

  Warning:
    This function is only valid for 1D, 2D and 3D gaussian models.

  Parameters:
    cov(list[list] or np.ndarray): covariance matrix :math:`\Sigma`

  Returns:
    float: The smallest theoretically achievable error rate for a classifier :math:`h` in dictionary :math:`H` for the given covariance matrix :math:`\Sigma`, also called Bayes Error Rate (or Bayes Risk), represented by :math:`\min_{h\in H} L(h)`.

  """
  # Extract sigma (standard deviations)
  sigma = np.sqrt(np.diag(cov))

  # Create the correlation matrix (rho)
  rho = cov / (sigma[:, None] * sigma[None, :])

  # Calculate the Bayes Error Rate (or Bayes Risk)
  if len(cov) == 3:
    if rho[0][1] == 0 and rho[0][2] == 0 and rho[1][2] == 0:
      w_0 = -(sigma[2]**2)/sigma[0]**2
      w_1 = -(sigma[2]**2)/sigma[1]**2

    elif sigma[0] == sigma[1] and rho[0][2] == 0 and rho[1][2] == 0 and rho[0][1] != 0:
      w_0 = w_1 = -(sigma[2]**2)/((sigma[0]**2)*(1+rho[0][1]))

    elif rho[0][1]==0 and rho[0][2] == rho[1][2] and sigma[0] == sigma[1]:
      try:
        w_0 = w_1 = sigma[2]*(sigma[2]-rho[1][2]*sigma[0])/(sigma[0]*(2*rho[0][2]*sigma[2]-sigma[0]))
      except ZeroDivisionError:
        return 0

    elif sigma[0] == sigma[1] and rho[0][1] != 0 and rho[0][2] == rho[1][2] and aw_0s(rho[0][2]) <= math.sqrt((1+rho[0][1])/2):
      w_0 = w_1 = (1-rho[0][2])/(2*rho[0][2] - (1+rho[0][1]))

    else:
      return 0

    return h_error_rate(0, [w_0, w_1], cov)

  elif len(cov) == 2:
    w_0 = (cov[0][1]-cov[1][1])/(cov[0][0] - cov[0][1])
    return h_error_rate(0, [w_0], cov)

  elif len(cov) == 1:
    return h_error_rate(0, [], cov)

  else:
    return 0


def distance_from_origin_to_intersection_between_normalized_ellipsoid_and_main_diagonal(cov):
  """
  This function calculates the distance from origin :math:`[0,0,...,0]` to the intersection point between the normalized ellipsoid and the main diagonal :math:`[1,1,...,1]`, given:
    - the covariance matrix :math:`\Sigma` of the ellipsoid

  Parameters:
    cov (list): covariance matrix :math:`\Sigma` of the ellipsoid

  Returns:
    float: distance from origin :math:`[0,0,...,0]` to the intersection point between the normalized ellipsoid and the main diagonal :math:`[1,1,...,1]`.

  """

  # convert cov to numpy array
  cov = np.array(cov)

  # Extract sigma (standard deviations)
  sigma = np.sqrt(np.diag(cov))

  # Extract dimension
  dim = len(sigma)

  # Extract min sigma
  min_sigma = min(sigma)

  # Define precision
  precision = 1e-4

  # Define support vector for t
  ar = np.arange(min_sigma, 0, step=-precision).tolist()

  line = np.array([[ar[i] for j in range(dim)] for i in range(len(ar))])

  candidates = []

  for i in range(len(ar)):
    X = line[i]
    if round(np.dot(np.dot(X,np.linalg.inv(cov)), X.T),2) == 1:
      candidates.append(X)
    elif candidates:
      intersection_point = np.mean(candidates)
      distance = math.sqrt(2)*(intersection_point)

      return distance


class Simulator:
  """A simulator for Linear classifier Loss analysis in order to evaluate Trade Off Between Samples and Features in Classification Problems on multivariate Gaussian Generated Samples."""

  def __init__(self, model: Model, dims=None, dims_to_compare=None, loss_types = ('EMPIRICAL_TRAIN', 'THEORETICAL', 'EMPIRICAL_TEST'), test_samples_amt=1024, iters_per_step=5, max_steps=200, first_step=100, precision=1e-6, augmentation_until_n = 1024, verbose=True):

    """
    :param model: Linear Classifier Loss Analysis Model
    :param dims: dimensionalities to be simulated
    :type dims: list[int] or tuple[int]
    :param loss_types: types of Loss comparation graphs wich will be compiled by the Simulator
    :type loss_types: list[str] or tuple[str]
    :param test_samples_amt: number of test samples to be genarated for prediction Loss evaluation
    :param iters_per_step: number of datasets generated in one simulation step equals iters_per_step*sqrt(augmentation_until_n)/sqrt(n), if n < augmentation_until_n, else it equals iters_per_step
    :param max_steps: max number of steps per n ∈ N
    :param first_step: min number of steps per n ∈ N
    :param precision: stop criteria. The simulation finishes when 'difference between E[predict_loss] calculated on two consecutive steps' < precision
    :param augmentation_until_n: increase number of generated datasets for n < augmentation_until_n
    :param verbose: if True, prints simulation progress

    :raise TypeError: if model is not a Model object;
                      if dims is not a list or tuple of integers;
                      if loss_types is not a list or tuple of strings;
                      if test_samples_amt is not an integer;
                      if iters_per_step is not an integer;
                      if max_steps is not an integer;
                      if first_step is not an integer;
                      if precision is not a float;
                      if augmentation_until_n is not an integer;
                      if verbose is not a boolean;


    :raise ValueError: if max_steps < first_step;
                      if precision < 0 or if precision > 1;
                      if augmentation_until_n < 1024;
                      if test_samples_amt < 2;
                      if iters_per_step < 1;
                      if max_steps < 10;
                      if first_step < 1
                      if loss_types is not a valid list or tuple of strings (see enumtypes.py for valid strings) or if loss_types is empty;
                      if dims is not a valid list or tuple of integers or if dims is empty;


    :Example:
      >>> from slacgs import Model

      >>> param = [1,1,2,0,0,0]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


      >>> param = [1,1,2,-0.1,0,0]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


      >>> param = [1,1,2,0,-0.4,-0.4]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


      >>> param = [1,1,2,-0.1,-0.4,-0.4]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


      >>> param = [1,2,-0.1]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, dims=(1,2), iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


      >>> param = [1,1,1,2,0,0,0,0,0,0]
      >>> model = Model(param, N=[2**i for i in range(1,11)], max_n=1024)
      >>> slacgs = Simulator(model, dims=(3,4), iters_per_step=1, max_steps=10, first_step=5, precision=1e-4, augmentation_until_n = 1024, verbose=False)
      >>> slacgs.run() # doctest: +ELLIPSIS


    """


    if not isinstance(model, Model):
        raise TypeError('model must be a Model object')

    if dims and not isinstance(dims, list) and not isinstance(dims, tuple):
        raise TypeError('dims must be a list or tuple')

    if not isinstance(loss_types, list) and not isinstance(loss_types, tuple):
        raise TypeError('loss_types must be a list or tuple of strings')

    if not isinstance(test_samples_amt, int):
        raise TypeError('test_samples_amt must be an integer')

    if not isinstance(iters_per_step, int):
        raise TypeError('iters_per_step must be an integer')

    if not isinstance(max_steps, int):
        raise TypeError('max_steps must be an integer')

    if not isinstance(first_step, int):
        raise TypeError('first_step must be an integer')

    if not isinstance(precision, float):
        raise TypeError('precision must be a float')

    if not isinstance(augmentation_until_n, int):
        raise TypeError('augmentation_until_n must be an integer')

    if not all(isinstance(loss_type, str) for loss_type in loss_types):
        raise TypeError('loss_types must be a list of strings')

    if dims and not all(isinstance(dim, int) for dim in dims):
        raise TypeError('dims must be a list of integers')

    if not isinstance(verbose, bool):
        raise TypeError('verbose must be a boolean')

    if max_steps < first_step:
        raise ValueError('first_step must be less than max_steps')

    if precision < 0 or precision > 1:
        raise ValueError('precision must be greater than 0 and less than 1')

    if augmentation_until_n < 1024:
        raise ValueError('augmentation_until_n must be greater than 1024')

    if test_samples_amt < 2:
        raise ValueError('test_samples_amt must be greater than 2')

    if iters_per_step < 1:
        raise ValueError('iters_per_step must be greater than 1')

    if max_steps < 10:
        raise ValueError('max_steps must be greater than 10')

    if first_step < 1:
        raise ValueError('first_step must be greater than 1')

    loss_types_enum_values = [loss_type.value for loss_type in list(LossType)]
    if not all(loss_type in loss_types_enum_values for loss_type in loss_types):
        raise ValueError('invalid loss_types list, implemented loss_types are: ' + str(loss_types_enum_values))


    if dims:
      if not all((dim > 0 and dim <= model.dim) for dim in dims):
          raise ValueError('invalid dims list/tuple for simulation, available dims for this Model are: ' + str([dim for dim in range(1, model.dim+1)]))
      self.dims = sorted(list(dims))
    else:
      self.dims = [dim for dim in range(1, model.dim + 1)]


    self.test_samples_amt = test_samples_amt
    self.iters_per_step = iters_per_step
    self.max_steps = max_steps
    self.first_step = first_step
    self.precision = precision
    self.augmentation_until_n = augmentation_until_n
    self.loss_types = list(loss_types)
    self.model = model
    self.report = Report(self)
    self.time_spent_test = 0
    self.verbose = verbose
    self.is_notebook = is_notebook()
    self.dims_to_compare = dims_to_compare if dims_to_compare else self.dims[-2:]

  def print_N_progress(self,n: int, max_iter: int, iter_per_step: int,datapoints_fig: plt.Figure, datapoints_plot_png_path):

    """Prints the progress of the simulation for a given n ∈ N and a given number of iterations per step (iter_per_step). The progress is printed in the terminal and a plot of the ellipsoids for this model's covariance matrix and a dataset sample with n=1024 sample points is shown.


    :param n: cardinality of the dataset
    :param max_iter: max number of iterations per n ∈ N
    :param iter_per_step: number of datasets generated in one simulation step equals iter_per_step*sqrt(augmentation_until_n)/sqrt(n), if n < augmentation_until_n, else it equals iter_per_step
    :param datapoints_fig: a plot of the ellipsoids for this model's covariance matrix and a dataset sample with n=1024 sample points


    """

    # terminal output setting
    progress_stream = '----------------------------------------------------------------------------------------------------'
    n_index = (self.model.N.index(n)+1)
    N_size = len(self.model.N)
    p = int(n_index*100/N_size)

    if self.is_notebook:
      clear_output()
    else:
      cls()

    if self.is_notebook:
      plt.close()
      if datapoints_fig :
        plt.figure(figsize=(10, 4))
        fm = plt.get_current_fig_manager()
        fm.canvas.figure = datapoints_fig
        datapoints_fig.canvas = fm.canvas

        plt.figure(datapoints_fig.number)
      plt.figure(self.report.plot_with_intersection().number)
      plt.show()



    print(' progress: ', end='')
    print(progress_stream[0:p] , end='')
    print("\033[91m {}\033[00m" .format(progress_stream[p:-1]) +' '+ str(n_index)+ '/' + str(N_size))
    print('n: ' + str(n))
    print('N = ' + str(self.model.N))
    print('max_iter = ' + str(max_iter))
    print('iter_per_step = ', iter_per_step)
    print('Model: ',self.report.model_tag)
    print('Simulator: ',self.report.sim_tag)
    print('d: ', self.report.d)
    print('bayes error rate: ', self.report.loss_bayes)
    print('Data Points Plot: ', datapoints_plot_png_path,'\n')
    for dim in self.dims:
      if self.report.loss_bayes[dim] == 0:
        print('when BR = 0, it will be infered after simulation')
        break

  def print_step_report(self, i, iter_per_step, loss_sum, iter_N):
    """
    :param self: self object of the class Simulator
    :type self: Simulator
    :param i: current iteration
    :param iter_per_step: iterations per step
    :param loss_sum: loss sum
    :param iter_N: iterations per n

    """

    print('step (',int((i+1)/iter_per_step),'): ', {d : {loss_type : loss_sum[d][loss_type]/iter_N[d][loss_type]  for loss_type in self.loss_types} for d in self.dims})

  def loss_bayes_empirical(self, d):
    """This function estimates numerically the bayes error rate for a classifier :math:`h` in dictionary :math:`H`.
      - When cardinality :math:`n` is large enough, a loss function :math:`L(h)` converges to the bayes error rate for a given dimensionality :math:`d`.
      - We can estimate the Bayes Error Rate by averaging loss :math:`L(h)` values for sufficiently large :math:`n`.
      - The best choice is to opt for averaging :math:`\hat{L}(\hat{h}^{(D)})` and :math:`\hat{L}(\hat{h}^{(D)},D')` for a Dataset :math:`D` with sufficiently large cardinality :math:`n`.

    Let :math:`\hat{h}^{(D)}` be the best empirical classifier in the dictionary :math:`H` for the dataset :math:`D` with :math:`d` features and :math:`n` samples, i.e., the classifier :math:`h` in :math:`H` with the minimum empirical error rate. Under these conditions, we can define three error rates:
      - :math:`L(\hat{h}^{(D)}) =` the theoretical error rate of the classifier :math:`\hat{h}^{(D)}`
      - :math:`\hat{L}(\hat{h}^{(D)}) =` the empirical error rate of the classifier :math:`\hat{h}^{(D)}`
      - :math:`\hat{L}(\hat{h}^{(D)},D') =` the empirical error rate of the classifier :math:`\hat{h}^{(D)}` using a test dataset :math:`D'`

    The Bayes Error (also: Bayes Risk) is defined as the minimum theoretical achievable error rate:
      - :math:`\min_{h\in H} L(h) =` the smallest theoretically achievable error rate for a classifier :math:`h` in dictionary :math:`H`.

    Parameters:
      self (Simulator): self object of the class Simulator
      d (int): dimensionality :math:`d` for which the bayes error rate is estimated

    Returns:
      float: An estimation for the smallest theoretically achievable error rate for a classifier :math:`h` in dictionary :math:`H` for the given covariance matrix :math:`\Sigma`, also called Bayes Error Rate (or Bayes Risk), represented by :math:`\min_{h\in H} L(h)`.


    """

    if 'EMPIRICAL_TRAIN' in self.loss_types and 'EMPIRICAL_TEST' in self.loss_types:
      # estimate bayes error rate using the average of the last loss values computed in the simulation for the empirical loss types
      losses = [ self.report.loss_N[d]['EMPIRICAL_TRAIN'][-1] , self.report.loss_N[d]['EMPIRICAL_TEST'][-1]]
    elif 'EMPIRICAL_TRAIN' in self.loss_types and 'THEORETICAL' in self.loss_types:
      losses = [ self.report.loss_N[d]['EMPIRICAL_TRAIN'][-1] , self.report.loss_N[d]['THEORETICAL'][-1]]
    else:
      losses = [ self.report.loss_N[d][loss_type][-1] for loss_type in self.loss_types]

    return np.mean(losses)

  def generate_dataset(self, n, i):
    """This method generates all simulation required datasets for all dimensionalities :math:`d` set to be simulated, given:
      - covariance matrix :math:`\Sigma` contained in the simulator object
      - class means :math:`\mathbf{\mu_+}` and :math:`\mathbf{\mu_+}` contained in the simulator object
      - random seed :math:`i`
      - cardinality :math:`n`

    Required Simulation Datasets:
      - :math:`D = D_{train} \cup D_{test}` : dataset
      - :math:`D_{train} = D_{(+)train} \cup D_{(-)train}` : training set
      - :math:`D_{test} = D_{(+)test} \cup D_{(-)test}` : test set
      - :math:`D_{(+)train} = (\mathbf{X}_{(+)train}, \mathbf{y}_{(+)train})` : training set for class :math:`(+)`
      - :math:`D_{(-)train} = (\mathbf{X}_{(-)train}, \mathbf{y}_{(-)train})` : training set for class :math:`(-)`
      - :math:`D_{(+)test} = (\mathbf{X}_{(+)test}, \mathbf{y}_{(+)test})` : test set for class :math:`(+)`
      - :math:`D_{(-)test} = (\mathbf{X}_{(-)test}, \mathbf{y}_{(-)test})` : test set for class :math:`(-)`

    Gaussian Distribution Parameters:
      - :math:`\mathbf{\Sigma}` : covariance matrix :math:`d \\times d` for training and test sets for both classes
      - :math:`n` : cardinality for training and test sets for both classes
      - :math:`\mathbf{\mu_+}` : mean vector for class :math:`(+)`
      - :math:`\mathbf{\mu_-}` : mean vector for class :math:`(-)`
      - :math:`\mathbf{\mu_+} = \\bigcup_{i=1}^{d} \mu_{+i}`
      - :math:`\mathbf{\mu_+} = \\bigcup_{i=1}^{d} \mu_{-i}`
      - :math:`\mu_{+i} = 1` and :math:`\mu_{-i} = -1`, for :math:`i \in [1,...,d]`
      - :math:`\mathbf{\mu_-} = [\mu_{-i}]` for :math:`i \in [1,...,d] = [\mu_{-1}, ..., \mu_{-d}] = [1,...,1]`
      - :math:`\mathbf{\mu_+} = [\mu_{+i}]` for :math:`i \in [1,...,d] = [\mu_{+1}, ..., \mu_{+d}] = [-1,...,-1]`


    Parameters:
      self (Simulator): self object of the class Simulator
      n (int): cardinality :math:`n` for training and testing sets for both classes
      i (int): random seed :math:`i`

    Returns:
      dict: A dictionary containing all required simulation datasets for all dimensionalities :math:`d` set to be simulated, represented by :math:`D_{(+)train}`, :math:`D_{(-)train}`, :math:`D_{(+)test}`, :math:`D_{(-)test}`.

    """

    half_n = int(n/2)

    #generate N/2 D-variate normal points for trainning images for each class
    sample_data_class_pos_train = multivariate_normal(self.model.mean_pos, self.model.cov).rvs(size=half_n, random_state=i)
    sample_data_class_neg_train = multivariate_normal(self.model.mean_neg, self.model.cov).rvs(size=half_n, random_state=i+1)

    #generate D-variate normal points for testing images
    self.time_spent_test = time.time()
    sample_data_class_pos_test = multivariate_normal(self.model.mean_pos, self.model.cov).rvs(size=self.test_samples_amt, random_state=i+2)
    sample_data_class_neg_test = multivariate_normal(self.model.mean_neg, self.model.cov).rvs(size=self.test_samples_amt, random_state=i+3)
    self.time_spent_test = time.time() - self.time_spent_test

    #generate target array for 2 classes: pos (class 1), neg (class 0) training images
    sample_target_class_pos_train = np.full((half_n, 1), 1, dtype=int)
    sample_target_class_neg_train = np.full((half_n, 1), 0, dtype=int)

    #generate target array for 2 classes: pos (class 1), neg (class 0) testing images
    self.time_spent_test = time.time() - self.time_spent_test
    sample_target_class_pos_test = np.full((self.test_samples_amt, 1), 1, dtype=int)
    sample_target_class_neg_test = np.full((self.test_samples_amt, 1), 0, dtype=int)
    self.time_spent_test = time.time() - self.time_spent_test

    #concatenate images for both classes into one single array for training and testing images
    sample_data_train = np.append(sample_data_class_pos_train, sample_data_class_neg_train, axis=0) if n > 2 else np.append(sample_data_class_pos_train, sample_data_class_neg_train, axis=0).reshape(2, self.model.dim)
    self.time_spent_test = time.time() - self.time_spent_test
    sample_data_test = np.append(sample_data_class_pos_test, sample_data_class_neg_test, axis=0)
    self.time_spent_test = time.time() - self.time_spent_test

    #concatenate target for both classes into one single array for training and testing images
    sample_target_train = np.append(sample_target_class_pos_train, sample_target_class_neg_train , axis=0)
    self.time_spent_test = time.time() - self.time_spent_test
    sample_target_test = np.append(sample_target_class_pos_test, sample_target_class_neg_test , axis=0)
    self.time_spent_test = time.time() - self.time_spent_test

    dataset = {'train': {'data': sample_data_train, 'target':sample_target_train},
          'test': {'data': sample_data_test, 'target':sample_target_test}}

    return dataset


  def run(self):

    """run the simulation

    :param self: simulation object
    :type self: Simulator
    :return: simulation report object
    :rtype: Report

    """

    # get the start time
    st = time.time()


    # iniciate  counters
    prev_mean_mesure = {dim : {loss_type: 0 for loss_type in self.loss_types} for dim in self.dims}

    # compute min(L(h)) and d for each dimension
    self.report.loss_bayes = {d : loss_bayes_analytical(np.array(self.model.cov[0:d]).T[0:d].T) for d in self.dims}
    self.report.d = {d : distance_from_origin_to_intersection_between_normalized_ellipsoid_and_main_diagonal(np.array(self.model.cov[0:d]).T[0:d].T) for d in self.dims}


    fig = self.model.data_points_plot
    datapoints_plot_png_path = self.model.save_data_points_plot_as_png(verbose=self.verbose)

    N = self.model.N
    N_start_size = len(self.model.N)

    ## for each cardinality n in N  do the simulation for each dimension d in dims
    while True:
      for n in N :

        # set iteration parameters for the current n
        augmentation_factor = math.sqrt(self.augmentation_until_n)/math.sqrt(n)
        iter_per_step = math.floor(self.iters_per_step*augmentation_factor) if n < self.augmentation_until_n else self.iters_per_step
        max_iter = iter_per_step*self.max_steps
        self.report.max_iter_N.append(max_iter)

        # iniciate sum(L) vars for each dimension and loss type
        loss_sum = {d : {loss_type : 0 for loss_type in self.loss_types} for d in self.dims}
        iter_N = {d : {loss_type : 0 for loss_type in self.loss_types} for d in self.dims}

        #iniciate control switches for each dimension and loss type
        switch = {'first_step': True,'train': { d : {loss_type : True if loss_type in self.loss_types else False for loss_type in [loss_type.value for loss_type in LossType ]} for d in self.dims}, 'dims': {d : True for d in self.dims} }

        # terminal output N progress bar
        if self.verbose:
          self.print_N_progress(n, max_iter,iter_per_step,fig,datapoints_plot_png_path)

        # for each iteration i in max_iter do the simulation for each dimension d in dims and estimate L(h) for each loss type
        for i in range(max_iter):

          stopwatch_dataset = time.time()
          dataset = self.generate_dataset(n,i)
          stopwatch_dataset = time.time() - stopwatch_dataset
          self.report.time_spent['total'] += stopwatch_dataset

          dims_on_by_loss_types = {loss_type : sum([switch['train'][dim][loss_type] for dim in self.dims]) for loss_type in self.loss_types}

          ## for each dimension d in dims do the simulation and estimate L(h) for each loss type
          for d in self.dims:

            if switch['dims'][d] :
              stopwatch_train = time.time()
              clf = linear_svm_train(d, dataset['train'])
              stopwatch_train = time.time() - stopwatch_train
              self.report.time_spent['total'] += stopwatch_train
              self.report.time_spent[d] += stopwatch_train
              self.report.time_spent[d] = time.time() - self.report.time_spent[d]

              if switch['train'][d][LossType.THEORETICAL.value]:
                self.report.time_spent[LossType.THEORETICAL.value] = time.time() - self.report.time_spent[LossType.THEORETICAL.value] - stopwatch_dataset/dims_on_by_loss_types[LossType.THEORETICAL.value] - stopwatch_train + self.time_spent_test/dims_on_by_loss_types[LossType.THEORETICAL.value]
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                loss_sum[d][LossType.THEORETICAL.value] += loss_theoretical(clf,np.array(self.model.cov[0:d]).T[0:d].T)
                self.report.time_spent[LossType.THEORETICAL.value] = time.time() - self.report.time_spent[LossType.THEORETICAL.value]
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                iter_N[d][LossType.THEORETICAL.value] += 1

              if switch['train'][d][LossType.EMPIRICALTRAIN.value]:
                self.report.time_spent[LossType.EMPIRICALTRAIN.value] = time.time() - self.report.time_spent[LossType.EMPIRICALTRAIN.value] - stopwatch_dataset/dims_on_by_loss_types[LossType.EMPIRICALTRAIN.value] - stopwatch_train + self.time_spent_test/dims_on_by_loss_types[LossType.EMPIRICALTRAIN.value]
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                loss_sum[d][LossType.EMPIRICALTRAIN.value] += loss_empirical(clf, dataset['train'])
                self.report.time_spent[LossType.EMPIRICALTRAIN.value] = time.time() - self.report.time_spent[LossType.EMPIRICALTRAIN.value]
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                iter_N[d][LossType.EMPIRICALTRAIN.value] += 1

              if switch['train'][d][LossType.EMPIRICALTEST.value]:
                self.report.time_spent[LossType.EMPIRICALTEST.value] = time.time() - self.report.time_spent[LossType.EMPIRICALTEST.value] - stopwatch_dataset/dims_on_by_loss_types[LossType.EMPIRICALTEST.value] - stopwatch_train
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                loss_sum[d][LossType.EMPIRICALTEST.value] += loss_empirical(clf, dataset['test'])
                self.report.time_spent[LossType.EMPIRICALTEST.value] = time.time() - self.report.time_spent[LossType.EMPIRICALTEST.value]
                self.report.time_spent['total'] = time.time() - self.report.time_spent['total']
                iter_N[d][LossType.EMPIRICALTEST.value] += 1

              self.report.time_spent[d] = time.time() - self.report.time_spent[d]

          ## simulate for at least first_step*iter_per_step iterations before checking for convergence
          if (i+1) % iter_per_step == 0:
            if switch['first_step']:

              ## start checking for convergence after turn off first_step switch
              if (i+1) == self.first_step*iter_per_step:
                switch['first_step'] = False
                for d in self.dims:
                  for loss_type in self.loss_types:
                    prev_mean_mesure[d][loss_type] = loss_sum[d][loss_type]/iter_N[d][loss_type]
            ## check for convergence and turn off switch if necessary for each loss type and dimension d
            else:
              for d in self.dims:
                for loss_type in self.loss_types:
                  if switch['train'][d][loss_type]:
                    if abs(loss_sum[d][loss_type]/iter_N[d][loss_type] - prev_mean_mesure[d][loss_type]) < (self.precision if loss_type != LossType.EMPIRICALTRAIN.value else self.precision*1e-2) :
                      switch['train'][d][loss_type] = False
                      if self.verbose:
                        print('SWITCH OFF: (', loss_type, ',', d,')')
                      if (np.unique(list(switch['train'][d].values())).tolist()) == [False] :
                        switch['dims'][d] = False
                        if self.verbose:
                          print('SWITCH OFF: dim (' , d ,')')
                    else:
                      prev_mean_mesure[d][loss_type] = loss_sum[d][loss_type]/iter_N[d][loss_type]

            ## print report for each step
            if self.verbose:
              self.print_step_report(i,iter_per_step,loss_sum,iter_N)

            ## check if all dimensions are turned off and stop simulation if so
            if np.unique(list(switch['dims'].values())).tolist() == [False]:
              break

        ## compute time spent for each cardinality n
        try:
          self.report.time_spent['n'][int(math.log(n,2)-1)] = self.report.time_spent['total'] - sum(self.report.time_spent['n'][:int(math.log(n,2)-1)])
        except IndexError:
          self.report.time_spent['n'].append(self.report.time_spent['total'] - sum(self.report.time_spent['n']))

        ## report loss and iter number for each dim and loss type
        for d in self.dims:
          for loss_type in self.loss_types:
            self.report.loss_N[d][loss_type].append(loss_sum[d][loss_type]/iter_N[d][loss_type])
            self.report.iter_N[d][loss_type].append(iter_N[d][loss_type])

      ## after simulation for each cardinality n in self.dims is finished, some tests are performed to check if simulation should be continued
      finish = True

      if 'EMPIRICAL_TEST' in self.loss_types:

        ## test if intersection point between dim d and dim d-1 is found and continue simulation for one more cardinality n if not
        if self.report.loss_N[self.dims[-1]]['EMPIRICAL_TEST'][-1] > self.report.loss_N[self.dims[-2]]['EMPIRICAL_TEST'][-1]:
          finish = False

        ## test if Loss Functions are not yet converged and continue simulation for one more cardinality n if not in order to estimate the Bayes risk
        if self.report.loss_bayes[self.model.dim] == 0 and abs(self.report.loss_N[self.model.dim]['EMPIRICAL_TEST'][-1] - self.report.loss_N[self.model.dim]['EMPIRICAL_TEST'][-2]) > 0.001:
          finish = False
        if 'EMPIRICAL_TRAIN' in self.loss_types:
          if self.report.loss_bayes[self.model.dim] == 0 and abs(self.report.loss_N[self.model.dim]['EMPIRICAL_TEST'][-1] - self.report.loss_N[self.model.dim]['EMPIRICAL_TRAIN'][-1]) > 0.001:
            finish = False


      if 'THEORETICAL' in self.loss_types:

        ## test if intersection point between dim d and dim d-1 is found and continue simulation for one more cardinality n if not
        if self.report.loss_N[self.dims[-1]]['THEORETICAL'][-1] > self.report.loss_N[self.dims[-2]]['THEORETICAL'][-1]:
          finish = False

        ## test if Loss Functions are not yet converged and continue simulation for one more cardinality n if not in order to estimate the Bayes risk
        if self.report.loss_bayes[self.model.dim] == 0 and abs(self.report.loss_N[self.model.dim]['THEORETICAL'][-1] - self.report.loss_N[self.model.dim]['THEORETICAL'][-2]) > 0.001:
          finish = False
        if 'EMPIRICAL_TRAIN' in self.loss_types:
          if self.report.loss_bayes[self.model.dim] == 0 and abs(self.report.loss_N[self.model.dim]['THEORETICAL'][-1] - self.report.loss_N[self.model.dim]['EMPIRICAL_TRAIN'][-1]) > 0.001:
            finish = False

      ## stop simulation if all Loss Functions are converged and intersection points are found, or if max cardinality model.max_n is reached
      if finish == True or max(N) >= self.model.max_n:
        break
      else:
        if self.model.N[-1]/self.model.N[-2] == 2:
          new_N = 2*max(N)
          self.model.N.append(new_N)
          N = [new_N]
        else:
          new_N = 2 + max(N)
          self.model.N.append(new_N)
          N = [new_N]

    ## estimate Bayes risk for each dimension if not yet estimated by analytical method
    for d in list(self.report.loss_bayes.keys()):
      if not self.report.loss_bayes[d]:
        self.report.loss_bayes[d] = self.loss_bayes_empirical(d)

    plt.close()
    ## set and save the final plot

    self.report.loss_plot = self.report.plot_with_intersection()

    ## print the final report
    self.report.print_report()

    ## save the final plot
    self.report.save_loss_plot_as_png(verbose=self.verbose)
    plt.close()

    # get the end time
    et = time.time()

    # get the execution time
    elapsed_time = et - st

    if self.verbose:
      print('Execution time:', elapsed_time/3600, 'h\n')

    ## transform time spent from seconds to hours
    self.report.duration = elapsed_time/3600
    for key in list(self.report.time_spent.keys()):
      if not isinstance(self.report.time_spent[key], list):
        self.report.time_spent[key] = self.report.time_spent[key]/3600
      else:
        self.report.time_spent[key] = [self.report.time_spent[key][i]/3600 for i in range(len(self.report.time_spent[key]))]



