# -*- coding: utf-8 -*-
"""Hidrometer_Consumption.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FjbyC-ixq-pJ3ugzzp8vH9M7hPlz3H2v

# Time Series Analysis

<h1>Table of Contents<span class="tocSkip"></span></h1>
<div class="toc"><ul class="toc-item"><li><span><a href="#Imports" data-toc-modified-id="Imports-1"><span class="toc-item-num">1&nbsp;&nbsp;</span>Imports</a></span></li><li><span><a href="#Function-Definitions" data-toc-modified-id="Function-Definitions-2"><span class="toc-item-num">2&nbsp;&nbsp;</span>Function Definitions</a></span></li><li><span><a href="#Data-Pre-processing-and-Statistical-Analysis" data-toc-modified-id="Data-Pre-processing-and-Statistical-Analysis-3"><span class="toc-item-num">3&nbsp;&nbsp;</span>Data Pre-processing and Statistical Analysis</a></span><ul class="toc-item"><li><span><a href="#1.-Load-Data-,-drop/input-value-in-NaN." data-toc-modified-id="1.-Load-Data-,-drop/input-value-in-NaN.-3.1"><span class="toc-item-num">3.1&nbsp;&nbsp;</span>1. Load Data , drop/input value in NaN.</a></span></li><li><span><a href="#1.1-Data-Mocking:-Langevin-Integration" data-toc-modified-id="1.1-Data-Mocking:-Langevin-Integration-3.2"><span class="toc-item-num">3.2&nbsp;&nbsp;</span>1.1 Data Mocking: Langevin Integration</a></span></li><li><span><a href="#2.-General-Cleaning:-Log-diff-(for-non-stationary-time-series)-,-0.0-values-interpolation,-Standardization." data-toc-modified-id="2.-General-Cleaning:-Log-diff-(for-non-stationary-time-series)-,-0.0-values-interpolation,-Standardization.-3.3"><span class="toc-item-num">3.3&nbsp;&nbsp;</span>2. General Cleaning: Log-diff (for non-stationary time-series) , 0.0 values interpolation, Standardization.</a></span><ul class="toc-item"><li><span><a href="#2.1-Data-Serialization-(json)" data-toc-modified-id="2.1-Data-Serialization-(json)-3.3.1"><span class="toc-item-num">3.3.1&nbsp;&nbsp;</span>2.1 Data Serialization (json)</a></span></li></ul></li><li><span><a href="#3.-Subtract-sazonalities-by-difference." data-toc-modified-id="3.-Subtract-sazonalities-by-difference.-3.4"><span class="toc-item-num">3.4&nbsp;&nbsp;</span>3. Subtract sazonalities by difference.</a></span></li><li><span><a href="#4.-Bootstrap." data-toc-modified-id="4.-Bootstrap.-3.5"><span class="toc-item-num">3.5&nbsp;&nbsp;</span>4. Bootstrap.</a></span><ul class="toc-item"><li><span><a href="#4.1-Gauss-Kernel." data-toc-modified-id="4.1-Gauss-Kernel.-3.5.1"><span class="toc-item-num">3.5.1&nbsp;&nbsp;</span>4.1 Gauss-Kernel.</a></span></li></ul></li><li><span><a href="#5.-Covariance-matrix,-PCA.-Time-Series-and-Interpolated-versions." data-toc-modified-id="5.-Covariance-matrix,-PCA.-Time-Series-and-Interpolated-versions.-3.6"><span class="toc-item-num">3.6&nbsp;&nbsp;</span>5. Covariance matrix, PCA. Time-Series and Interpolated versions.</a></span></li><li><span><a href="#6.-Null-hypothesis,-normal-(mean,var)--for-time-series-realization.-No-low-dimension." data-toc-modified-id="6.-Null-hypothesis,-normal-(mean,var)--for-time-series-realization.-No-low-dimension.-3.7"><span class="toc-item-num">3.7&nbsp;&nbsp;</span>6. Null hypothesis, normal (mean,var)  for time-series realization. No low dimension.</a></span></li></ul></li><li><span><a href="#Visualizations" data-toc-modified-id="Visualizations-4"><span class="toc-item-num">4&nbsp;&nbsp;</span>Visualizations</a></span><ul class="toc-item"><li><span><a href="#Time-Series" data-toc-modified-id="Time-Series-4.1"><span class="toc-item-num">4.1&nbsp;&nbsp;</span>Time-Series</a></span><ul class="toc-item"><li><span><a href="#Hydrometers,-interpolated,-sazonality-subtracted-versions" data-toc-modified-id="Hydrometers,-interpolated,-sazonality-subtracted-versions-4.1.1"><span class="toc-item-num">4.1.1&nbsp;&nbsp;</span>Hydrometers, interpolated, sazonality subtracted versions</a></span></li><li><span><a href="#Auto-correlation-function-for-time-series" data-toc-modified-id="Auto-correlation-function-for-time-series-4.1.2"><span class="toc-item-num">4.1.2&nbsp;&nbsp;</span>Auto-correlation function for time-series</a></span></li></ul></li><li><span><a href="#Histograms" data-toc-modified-id="Histograms-4.2"><span class="toc-item-num">4.2&nbsp;&nbsp;</span>Histograms</a></span><ul class="toc-item"><li><span><a href="#Eigenvalue-counting-for-covariance-matrix-in-cases-of-time-series,-interpolated-ts-and-null-hypothesis-ts." data-toc-modified-id="Eigenvalue-counting-for-covariance-matrix-in-cases-of-time-series,-interpolated-ts-and-null-hypothesis-ts.-4.2.1"><span class="toc-item-num">4.2.1&nbsp;&nbsp;</span>Eigenvalue counting for covariance matrix in cases of time-series, interpolated ts and null-hypothesis ts.</a></span></li></ul></li><li><span><a href="#Low-Dimensional-Embedding" data-toc-modified-id="Low-Dimensional-Embedding-4.3"><span class="toc-item-num">4.3&nbsp;&nbsp;</span>Low Dimensional Embedding</a></span><ul class="toc-item"><li><span><a href="#T-SNE-non-linear-manifold-embedding." data-toc-modified-id="T-SNE-non-linear-manifold-embedding.-4.3.1"><span class="toc-item-num">4.3.1&nbsp;&nbsp;</span>T-SNE non-linear manifold embedding.</a></span></li></ul></li></ul></li><li><span><a href="#Clustering-Measures" data-toc-modified-id="Clustering-Measures-5"><span class="toc-item-num">5&nbsp;&nbsp;</span>Clustering Measures</a></span><ul class="toc-item"><li><span><a href="#K-Means" data-toc-modified-id="K-Means-5.1"><span class="toc-item-num">5.1&nbsp;&nbsp;</span>K-Means</a></span></li><li><span><a href="#Agglomeration" data-toc-modified-id="Agglomeration-5.2"><span class="toc-item-num">5.2&nbsp;&nbsp;</span>Agglomeration</a></span></li><li><span><a href="#Mutual-Information/TODO:-Time-Delayed-Mutual-Information" data-toc-modified-id="Mutual-Information/TODO:-Time-Delayed-Mutual-Information-5.3"><span class="toc-item-num">5.3&nbsp;&nbsp;</span>Mutual Information/TODO: Time Delayed Mutual Information</a></span></li></ul></li></ul></div>

## Imports
"""

####
## Load libs
#
# Module installs through pip to use this notebook with any virtual machine.
#!pip list 
#!pip uninstall -y Bregman
!pip install --upgrade-strategy=only-if-needed git+https://github.com/Uiuran/BregmanToolkit
!pip install scikit-image
#!pip install scikits.audiolab

## python Executable and imports
import sys
print("python exe {}".format(sys.executable))
print("python caminhos import {}".format(sys.path))

# %matplotlib notebook
# %matplotlib inline


## Connect DB
#import pyodbc

## Numeric,statiscs,sazonality and tabular data libs
from pylab import *
from bregman.suite import *
import aspa # 2D spectrogram patch sparse approximation library
#reload(A) # for debugging
# get_ipython().magic('matplotlib inline')
# get_ipython().magic('matplotlib inline')

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.patches as mpatches
from matplotlib.ticker import NullFormatter
rcParams['figure.figsize'] = (12.0, 8.0) # Make larger in-line figures
#from sklearn import model_selection
#from sklearn.model_selection import train_test_split
import scipy as scp
from scipy.stats import genextreme as gev
from scipy import interpolate as interp
from scipy import fft
from scipy import signal
from scipy.integrate import odeint
from sklearn.neighbors import KernelDensity

## Code debug, serialization and IO
import traceback
import json
import io
import csv
from time import time

## Audio IO from np array
import scikits.audiolab as audio

## Machine Learn/Statistical Modelling and Dimensionality Reduction Analysis
from sklearn.cluster import KMeans, SpectralClustering, AgglomerativeClustering
from sklearn.manifold import TSNE



pd.options.mode.use_inf_as_na = True

"""## Function Definitions"""

####
## Definicoes para Interpolação
#

def initialization_interpolation(hydrometer_ts):
  '''
  Return interpolated dict time-series and the auxiliary dictionaries,
  x axis points to generate interpolated points (xinterp), x axis points to fit spline(x_ts), interpolated time series points (itp).
  '''
  pop(hydrometer_ts, thresh = 10)
  #print('hydro ', hydrometer_ts)
  xinterp = {k:np.array([i for i in range( np.size(hydrometer_ts[k]) ) if np.array(hydrometer_ts[k])[i] == 0.0  ]) for k in np.array(hydrometer_ts.keys())}
  pop(xinterp, thresh = 2)
  #print('interpolacao ', xinterp)
  x_ts = { k : np.setxor1d(np.linspace(0,np.size(hydrometer_ts[k])-1,np.size(hydrometer_ts[k]) ),xinterp[k]) for k in np.array(xinterp.keys())}  
  pop(x_ts, thresh = 4)
  #print('x_ts ', x_ts)

  itp_ts = {k:fit_spline(xinterp[k],x_ts[k],hydrometer_ts[k][x_ts[k].astype(np.int16)]) for k in np.array(x_ts.keys())}
  interpolated_ts = {k:interpolation(xinterp[k],itp_ts[k],hydrometer_ts[k].copy()) for k in np.array(x_ts.keys())}
  
  return interpolated_ts,itp_ts,xinterp,x_ts


def interpolation(xinterp,interp,hydrometer_ts):
   
  l = np.size(hydrometer_ts)

  # Energia 
  e = np.zeros(np.size(hydrometer_ts)-1)
  for i in range(0,np.size(hydrometer_ts)-1):
    e[i] = np.abs(hydrometer_ts[i+1] - hydrometer_ts[i])
  e_m = np.mean(e)
  e_std = e.std() + 1e-7
  i = 0 
  for x in xinterp:
    if x == 0 or x == (l-1):
      
      
      if np.abs(interp[i]) < 0.5:
        hydrometer_ts[x] = interp[i]
      else:
        if x == 0:
          hydrometer_ts[x] = hydrometer_ts[x+1]/3.0
        else:
          hydrometer_ts[x] = hydrometer_ts[x-1]/3.0
    else:        
        
        j = 1
        k = 1
        
        while ((hydrometer_ts[x-j] == 0.0) or (hydrometer_ts[x+k] == 0.0)) and (((x-j) > 0) and ((x+k) < (l-1)) ):
          j += 1
          while (hydrometer_ts[x+k] == 0.0) and ((x+k) < (l-1)):
            k += 1
          media = (hydrometer_ts[x-j]+hydrometer_ts[x+k]+1e-7)/2.0
          eps = ((np.abs(interp[i] - hydrometer_ts[x-j]) + np.abs( - interp[i] + hydrometer_ts[x+k]))/2.0 - e_m)/e_std
          if  np.abs(eps) > 2.0:            
            hydrometer_ts[x] = media
          else:
            hydrometer_ts[x] = interp[i]
        media = (hydrometer_ts[x-j]+hydrometer_ts[x+k]+1e-7)/2.0
    i += 1
  return hydrometer_ts


def fit_spline(x,x_points,y_points):

    #print('x and y ',len(x_points),len(y_points))
    #print('x eval ',x)

    
    
    tck = interp.splrep(x_points, y_points)
    #print('tck ',tck)
    return interp.splev(x, tck)

####
## Standardization and Log differentiation (exclusion of possible non-stationarity between time series samples)
#

def normalize_dict(d,std=False, log_diff=False, demean = False):
  for k,v in d.items():
    if log_diff:
      d[k] = np.array([ np.log(v[i+1]+1e-7) - np.log(v[i]+1e-7) for i in range(np.size(v)-1)])      
    if std == True:
      d[k] = (v - v.mean())/(v.std()+1e-6)
    if demean:
      d[k] = v - v.mean()    
    
def pop(hydrometer_ts, thresh = 40):
  '''
  Receive dict or list of time-series and pop the one who has less than thresh elements
  '''
  count_out = 0;
  if type(hydrometer_ts) == type([]):
    l = len(hydrometer_ts)
    i = 0
    while i < l:
      
      nsize = np.size(hydrometer_ts[i])
  
      if nsize > 1 and nsize < thresh:    
        
        hydrometer_ts.pop(i)
        count_out += 1
        l = len(hydrometer_ts)  
      elif nsize == 1:
        hydrometer_ts.pop(i)
        l = len(hydrometer_ts)
        count_out += 1
      else:        
        i += 1
    print('numero de registros restantes: ',len(hydrometer_ts))
      

  else:
    for k in hydrometer_ts.keys():
      nsize = np.size(hydrometer_ts[k])
      
      if nsize > 1:    
        if nsize < thresh:          
          hydrometer_ts.pop(k)
          count_out += 1          
      else:        
        hydrometer_ts.pop(k)  
        count_out += 1
    print('numero de registros restantes: ',len(hydrometer_ts.keys()))  
  print('numero de registros retirados: ',count_out)
  
####
## NaN number filtering and imputation  of 0.0s or what-ever function of dataframe/np.array you want
#   
    
def isnan(m, fillna = True, fillwith = lambda x: 0.0, verbose = True):
  '''
  Detect NaN and fill with provided fillwith function, standard is 0.0
  Input: np.array or DataFrame
  
  '''  
  if type(m) == type(np.array([])):
    for i in range(len(m[:,0])):
      for j in range(len(m[0,:])):
        if np.isnan(m[i,j]) == True:
          if verbose:
            print("{}".format((i,j)),m[i,j])
          if fillna:
            m[i,j] = fillwith(m)
  elif type(m) == type(pd.DataFrame([])):
    for k in np.array(m.index.unique()):
      v = np.array(m.loc[k,'volume_lido'])
      if len(v.shape) == 0:
        v = np.array([v])
      if np.isnan(v).any() == True:        
        for j in range(v.size):          
          if np.isnan(v[j]) == True:
            if verbose:
              print("{}".format((k,j)),v[j])
            if fillna and len(v.shape) == 1:
              m.loc[k,'volume_lido'] = fillwith(m)
            elif fillna:
              m.loc[k,'volume_lido'][j] = fillwith(m)
  print('-')

####
## Transform list or dict into matrix by thresholding the time-series by thresh.
#

def to_tensor(hydrometer_ts, thresh = 45): 
  '''
  Turn a list or dictionary of time series into a matrix by discarding the elements above thresh and excluding time series with less elemets than thresh.
  '''
  m = None
  pop(hydrometer_ts, thresh = thresh)
  # list-> np.array 2D
  if type(hydrometer_ts) == type([]):
    m = np.array([hydrometer_ts[i][0:thresh] for i in range(len(hydrometer_ts))])
    nomes = np.linspace(0,np.size(hydrometer_ts)-1,np.size(hydrometer_ts))
  # dict-> np.array 2D    
  if type(hydrometer_ts) == type({'':0.0}):
    m = np.array([hydrometer_ts[k][0:thresh] for k in hydrometer_ts.keys()])    
    nomes = hydrometer_ts.keys()
    
  return m,nomes

####
## Subtrair sazonalidade, não está muito funcional, talvez melhor com fourier.
#


def subtrai_sazonalidade(hydrometer_ts, medianizar = 15):
  '''
  Recebe dicionario de series temporais, e retorna lista de séries diferenciadas em relaçao a maior pico de autocorrelação
  TODO: testar graficamente. Setando temporariamente para 12.
  '''
  ac,n = acf(hydrometer_ts,medianizar)

  for j in range(len(n)):
    season = 0
    for i in range(1,len(ac[j])-medianizar):      
      count = 0
      argma = np.argmax(ac[j][count:count+medianizar])
      argmi = np.argmin(ac[j][count:count+medianizar])
      while count < len(ac[j])-1 and argmi > argma:
        if argma > argmi:       
          season = argma + count
          break;
        else:
          count += 1        
        argma = np.argmax(ac[j][count:count+medianizar])
        argmi = np.argmin(ac[j][count:count+medianizar])
        
        z = 1
        while z < count:
          argma = np.argmax(ac[j][count-z:count+medianizar])
          argmi = np.argmin(ac[j][count-z:count+medianizar])          
          if argma > argmi:       
            season = argma + count - z
            break
          else:
            z += 1                       
        
        
    #print('Season',season)    
    ts = [np.array(hydrometer_ts[i].copy()) for i in hydrometer_ts.keys()]
    for i in range(len(ts[j])-season):
      #ts[j][i] = ts[j][i]-ts[j][np.mod(i+season,np.size(ts[j][:]))]
      ts[j][i] = ts[j][i]-ts[j][np.mod(i+12,np.size(ts[j][:]))]
  
  return ts

def acf(hydrometer_ts,medianizar):
  '''
  Return list of autocorrelation function from dict of time-series and an array of size n of the acf.
  '''
  autocorr = lambda ts: scp.correlate(ts,ts, mode="full")/(scp.var(ts, ddof = 1)+1e-6)
  pop(hydrometer_ts, thresh = 2)
  ts = [np.array(hydrometer_ts[i].copy()) for i in hydrometer_ts.keys()]
  
  n = np.array([ int(np.size(scp.correlate(i,i, mode="full"))/2.0) for i in ts])
  acf = [ np.array([ np.mean(autocorr(ts[j])[i:i+medianizar]) for i in range(n[j],2*n[j]-medianizar) ] ) for j in range(len(n))]
  return acf,n

####
## Kernel Density Estimation
#

def getKernelDensityEstimation(values, x, bandwidth = 0.2, kernel = 'gaussian'):
    model = KernelDensity(kernel = kernel, bandwidth=bandwidth)
    model.fit(values[:, np.newaxis])
    log_density = model.score_samples(x[:, np.newaxis])
    return np.exp(log_density)

####
## Mutual Information Measures
#

def mutual_information(X,Y,k=5,base=np.exp(1)):
  '''
  pyMIestimator is a function for estimating Mutual Information

  Inputs function:
  X,Y         : an N x M matrix (samples x features)
  k           : the number of nearest neighbour
  unit base   : base=2 -> Shannon (bits), base=exp(1)=nat
 
  Outputs function:
  I1, I2      : mutual information estimates
 
  Martha Arbayani Zaidan, PhD
  Postdoctoral Research Fellow
  Aalto University, Finland
 
  References:
 
     Alexander Kraskov Harald Stogbauer, and Peter Grassberger,
     Estimating mutual information,
     Physical review E 69, no. 6 (2004): 066138.
 
     Jorge Numata, Oliver Ebenhoh and Ernst-Walter Knapp,
     Measuring correlations in metabolomic networks with mutual information,
     Genome Informatics 20 (2008): 112-122.
  '''
  [N,M] = X.shape #  The number of samples

  # Calculate the distance between each data point (sample)
  # and its k-th nearest neighbour:
  dx = np.zeros((N,N))
  dy = np.zeros((N,N))
  dz = np.zeros((N,N))

  for i in range(0,N):
    for j in range(0,N):
      dx[i,j] = np.sqrt(np.sum((X[i,:] - X[j,:])**2))
      dy[i,j] = np.sqrt(sum((Y[i,:] - Y[j,:])**2))
      dz[i,j] = max([dx[i,j], dy[i,j]])


  # Count nx(i) and ny(i) using the criteria defined in Kraskov et al. (2004)
  Eps = np.zeros((N,1))
  nx1 = np.zeros((N,1))
  ny1 = np.zeros((N,1))
  nx2 = np.zeros((N,1))
  ny2 = np.zeros((N,1))

  for i in range(0,N):

  # the first cols of {dx,dy,dz} are zeros because the first cols are
  # the distance between themselves, so we remove them here:
    dxS = dx[i,:]
    dxS[i] = np.nan
    idx=np.logical_not(np.isnan(dxS))
    dxS=dxS[idx]

    dyS = dy[i,:]
    dyS[i] = np.nan
    idx=np.logical_not(np.isnan(dyS))
    dyS=dyS[idx]

    dzS = dz[i,:]
    dzS[i] = np.nan
    idx=np.logical_not(np.isnan(dzS))
    dzS=dzS[idx]

    dzSort = np.sort(dzS)
  # Eps(i) is epsilon(i)/2
  # Eps(i) is the distance from sample z(i) to its k-th neighbor
    Eps[i] = dzSort[k-1]

  # For algorithm 1:
  # count the number nx(i) of points x(j) whose distance from x(i) is
  # strictly less than Eps, and similarly for y instead of x:
    nx1[i] = sum(dxS < Eps[i])
    ny1[i] = sum(dyS < Eps[i])

  # For algorithm 2:
  # we replace nx(i) and ny(i) by the number of points with
  # ||x(i)-x(j)|| <= Eps(i) and ||y(i)-y(j)|| <= Eps(i)
    nx2[i] = sum(dxS <= Eps[i])
    ny2[i] = sum(dyS <= Eps[i])

  # Estimating Mutual Information:
  I1 = (digamma(k) - (sum(digamma(nx1 + 1) + digamma(ny1 + 1)) / N) + digamma(N) ) / np.log(base)
  I2 = (digamma(k) - 1.0/k - (sum(digamma(nx2) + digamma(ny2)) / N) + digamma(N) ) / np.log(base)

  I1=np.sign(I1)*np.sqrt(1-np.exp(-2*abs(I1)))
  I2=np.sign(I2)*np.sqrt(1-np.exp(-2*abs(I2)))

  # No negative MI:
  if I1 < 0: I1 = 0
  elif I2 < 0: I2 = 0

  return I1, I2

def digamma ( x ):
  '''
   digamma function calculates digamma(x) = d(log(gamma(x)))/dX
   Authors:

    Original FORTRAN77 version by Jose Bernardo.
    Original Python version by John Burkardt.
    Modified Python version by Martha Arbayani Zaidan.
    This modified version takes numpy array input.

   Reference:

    Jose Bernardo,
    Algorithm AS 103:
    Psi ( Digamma ) Function,
    Applied Statistics,
    Volume 25, Number 3, 1976, pages 315-317.
  '''
  x=np.float64(x) # convert to be double precision

  #  Check the input.
  if ( x.any <= 0.0 ):
    value = 0.0
    return value

  #  Initialize.
  value = 0.0

  #  Use approximation for small argument.

  if ( x.any <= 0.000001 ):
    # Euler-Mascheroni constant:
    euler_mascheroni = 0.57721566490153286060651209008240243104215933593992
    value = - euler_mascheroni - 1.0 / x + 1.6449340668482264365 * x
    return value

  #  Reduce to DIGAMA(X + N).

  while ( x.any < 8.5 ):
    value = value - 1.0 / x
    x = x + 1.0

#  Use Stirling's (actually de Moivre's) expansion.

  r = 1.0 / x
  value = value + np.log ( x ) - 0.5 * r
  r = r * r
  value = value \
    - r * ( 1.0 / 12.0
    - r * ( 1.0 / 120.0
    - r * ( 1.0 / 252.0
    - r * ( 1.0 / 240.0
    - r * ( 1.0 / 132.0 ) ) ) ) )

  return value

####
## Mock Noise Harmonic Oscillator with Levy random noise
#
# TODO: 1. Adapt lingevin to any map and noise, TODO: 2. 'Runge-Kutta' for StochasticDE

# first a python version found elsewere, TODO - fix the langevin equation

def R(t): #R is the "random force" term
  return np.random.normal(0,1) 

def LangevinOscillator(x,t):
  w,g = 5.0,3.5
  z=x[0]+1j*x[1]
  #print("z=",z)
  dz=1j*(w+g*R(t))*z
  #print("dz=",dz)
  return np.real(dz),np.imag(dz)

def langevin(x, t, func, seed= 1002384610730):

    n = len(x);
    
    np.random.seed(seed);
    noise = np.random.normal(size=n);
    y = np.zeros(n);
    
    y = func(x, t) + noise;

    return y;

def stochastic_int(func, x, t, param = ()):

  N = len(t);
  n = len(x);  
  dt = t[1] - t[0];
  y = np.zeros((N,n));
  #tau = 1.21
  

  ####
  ## Levy
  #
  if param[2] == 'Levy':
    lv = scp.stats.levy_stable(1.8,-0.5)
    noise = lv.rvs(N, random_state = np.random.randint(10,high=1751)) 
  else:
    np.random.set_state = np.random.randint(10,high=1751)
    noise = np.random.normal(0.0,0.2,N)

  #noise = np.random.normal(0.0,1.0,N)
  y[0,:] = x + dt*func(x, param[0], param[1]) + np.sqrt(dt)*noise[0]
  y[0,-1] = y[0,-1]# + np.sqrt(dt)*noise[0]

  for i in range(0,N-1):                
    y[i+1,:] = y[i,:] + dt*func(y[i,:], param[0], param[1])# + np.sqrt(dt)*noise[i]
    y[i+1,-1] = y[i,-1] + np.sqrt(dt)*noise[i]
  return y.T;  

# Gaussian/Levy Mixture Mock/Sampling
# TODO: Levy, iterative choosen mixture, Expectation Maximization Mixture
def sample_gaussmix(data=None,N=500, mean_base = 0.0,var_base = 0.15, num_mods = 30, random_mods=False)
  '''
    Generate sample noise from num_mods different gaussian with data or only noise.
    
    TODO: select mod from a list of tuples with (mean,variance)
  '''
  num_mods = int(num_mods)
  if type(data) != None:
    shape = np.shape(data)
  c = np.int(shape[0]/num_mods)
  np.random.set_state = 1440*shape[0]    

  if type(data) != None:
    if random_mods:
      if len(shape) > 1:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),c) for i in range(num_mods)]).flatten() )    
    else:
      if len(shape) > 1:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),c) for i in range(num_mods)]).flatten() )      
  else:
    if random_mods:
      if len(shape) > 1:
        mix = lambda data: (np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),c) for i in range(num_mods)]).flatten() )    
    else:
      if len(shape) > 1:
        mix = lambda data: (np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),c) for i in range(num_mods)]).flatten() )      
  return mix(data)


ma: np.array([ x[1] , -k*x[0] -gamma*x[1]])

def mock(x0, num_samples):
  '''
  Harmonic Oscillators with Levy Distributed noise for mocking time series. The time series are truncated to positive values.
  '''
    
  t = np.linspace(0.0,10.0,np.int32(num_samples) )
  y0 = np.ndarray((np.shape(x0)[0],num_samples))
  for i in range(np.shape(x0)[0]):
    #y0[i] = stochastic_int(harmonic, x0[i,:], t, param = (0.4+np.random.normal(0.0,0.05),0.3,'random') )[1] 
    y0[i] = odeint(LangevinOscillator,x0[i,:],t).T[1]

    #if np.min(y0[i]) < 0.0:
      #y0[i] = (np.abs(np.min(y0[i])) + y0[i])/np.abs(np.min(y0[i]))
  return y0,t

####
## DB Resources (RDB and so on)
#

def flat_for_sql(mocked, **kwargs):
  '''
   Flat matrix N x M  (Samples,Fatures) or (Num_Var,Time Series Samples) and add another columns through fields.
  '''
    
  n,m = np.shape(mocked)
  flatted = []  
  j = 0
  while j < n:
    
    for a in mocked[j].flat:
      flatted.append([j,'Pressure','Harmonic Levy',a])      
    j += 1
  return flatted

####
##Plotting
#

def grid_plot(hydrometer_ts, shape, hidronum = 10):
  '''
  Dict/List grid-plot
  '''
  
  fig, axs = plt.subplots(shape[0], shape[1], sharex=True, sharey=False)    

  for i in np.arange(hidronum,hidronum+shape[0]):
    for j in np.arange(hidronum+shape[0],hidronum+shape[0]+shape[1]):
      if type(hydrometer_ts) == type({'':0.0}):        
        axs[i-hidronum][j-(hidronum+shape[0])].plot(hydrometer_ts[hydrometer_ts.keys()[i+j]], '-')
        axs[i-hidronum][j-(hidronum+shape[0])].set_title('{}'.format(hydrometer_ts.keys()[i+j]))
      else:
        axs[i-hidronum][j-(hidronum+shape[0])].plot(hydrometer_ts[i+j], '-')
        #axs[i-hidronum][j-(hidronum+shape[0])].set_title('{}'.format(hydrometer_ts.keys()[i+j]))
      
def grid_plot_2array(a, b, shape, hidronum = 10):
  '''
  Dict/List grid-plot
  '''
  
  fig, axs = plt.subplots(shape[0], shape[1], sharex=True, sharey=False)    

  for i in np.arange(hidronum,hidronum+shape[0]):
    for j in np.arange(hidronum+shape[0],hidronum+shape[0]+shape[1]):
      if type(b) == type({'':0.0}):        
        axs[i-hidronum][j-(hidronum+shape[0])].plot(b[i+j], '-')
        axs[i-hidronum][j-(hidronum+shape[0])].set_title('{}'.format(b.keys()[i+j]))
      else:
        axs[i-hidronum][j-(hidronum+shape[0])].plot(a[i+j], b[i+j], '-')
        #axs[i-hidronum][j-(hidronum+shape[0])].set_title('{}'.format(hydrometer_ts.keys()[i+j]))

"""## Data Pre-processing and Statistical Analysis 
### 1. Load Data , drop/input value in NaN.
###  1.1 Data Mocking: Langevin Integration
### 2. General Cleaning: Log-diff (for non-stationary time-series) , 0.0 values interpolation, Standardization.  
#### 2.1 Data Serialization (json)
### 4. Bootstrap.
#### 4.1 Gauss-Kernel.
### 5. Covariance matrix, PCA. Time-Series and Interpolated versions.
### 6. Null hypothesis, normal (mean,var)  for time-series realization. No low dimension.
"""

df = pd.read_csv('/content/SQL - Stattus4V2.csv',sep=';')
df = df.dropna(axis=0)
#isnan(df,verbose = True)
#/home/humannoise/Desktop/Data Sample

df.head(40)

'''
####
## Generating Matrix of Mocked Harmonic Oscillator with Levy Noise Random Noise
#

x0 = np.ndarray((50,2))
mocked_data = np.ndarray((50,4000))
for i in range(50):
  el0 = 3.0-0.01*np.mod(np.random.randint(i,high=i+4000),400)  
  el1 = -3.0+0.01*np.mod(np.random.randint(i,high=i+4000),400)
  sign0 = np.random.choice([np.sign(el0),np.sign(el1)])
  x0[i,:] = np.array([sign0*el0,-sign0*el1])


mocked_data,t = mock(x0,4000)
grid_plot(mocked_data[:,0:4000],(3,3),hidronum=10)
'''

####
## Interpolation
#
df0 = df.copy().set_index('numero_hidrometro')
hydrometer_ts = {k:np.array(df0.loc[k,'volume_lido']) for k in np.array(df0.index.unique())}
#normalize_dict(hydrometer_ts, std=False, log_diff = True)
interpolated_ts,itp_ts,xinterp,_ = initialization_interpolation(hydrometer_ts)

####
## Standardization
#
normalize_dict(interpolated_ts, std=True, log_diff = False)
normalize_dict(hydrometer_ts, std=True, log_diff = False)

####
## Serialize to json and load. It also includes 4 int numbers to predict the monthly data size.
#
d = {hydrometer_ts.items()[i][0]:list(hydrometer_ts.items()[i][1]) for i in range(np.shape(hydrometer_ts.items())[0])}
d['num_dias'] = 10
d['num_horas'] = 24
d['sample_interval'] = 15
d['resolution'] = 60 # 60 for minutes 3600 for seconds in a hour

# register
with open('/home/stattus4dpenalva/stattus4/dados/sample.json',"w") as fp:
  json.dump(d,fp)

#load
with open('/home/stattus4dpenalva/stattus4/dados/sample.json',"r") as fp:
  l = json.load(fp)

####
## Interpolacao inspecao visual
#


'''
for i in range(0,4):

  df0 = df.copy().set_index('numero_hidrometro')
  hydrometer_ts = {k:np.array(df0.loc[k,'volume_lido']) for k in np.array(df0.index.unique())}
  normalize_dict(hydrometer_ts, std=False, log_diff = True)

  interpolated_ts,itp_ts,xinterp,_ = initialization_interpolation(hydrometer_ts)


  plt.figure()
  plt.plot(hydrometer_ts[hydrometer_ts.keys()[i]],c='g')
  plt.plot( xinterp[hydrometer_ts.keys()[i]], itp_ts[hydrometer_ts.keys()[i]], marker='o', color='c', ls='None')
  plt.plot(interpolated_ts[hydrometer_ts.keys()[i]],c='b')
'''

####
##Retirar Sazonalidades por diferencas
#
ts = subtrai_sazonalidade(interpolated_ts.copy(),medianizar=15)

### 3. Subtract sazonalities by difference.
ac,_ = acf(interpolated_ts,5)

a = []
for i in range(0,9):
  a.append(fft(ac[i]))

len(interpolated_ts.values()[12])

a = np.array([signal.welch(mocked_data[k], detrend = lambda x: x - np.mean(x)) for k in range(0,50)])
grid_plot_2array(a[:,0],a[:,1],(6,6),hidronum=1)

#filtered = signal.wiener(mocked_data[0,0:2000])
f, t, Sxx = signal.spectrogram(mocked_data[3,:], detrend = lambda x: x - np.mean(x))
plt.figure()
plt.pcolormesh(t, f, Sxx)
plt.ylabel('Frequency [Hz]')
plt.xlabel('Time [sec]')
grid_plot(mocked_data[:,0:300],(3,3),hidronum=1)
#grid_plot(filtered,(3,3),hidronum=1)

####
## Bootstrap
#

mresample,_ = to_tensor(hydrometer_ts,thresh=45)
x = np.mgrid[mresample[5,:].min():mresample[5,:].max():30j]
kernel = KernelDensity(kernel = "gaussian", bandwidth= 45**(-1.0/5.0) )
kernel.fit( mresample[5,:].reshape(-1, 1) )
log_density = kernel.score_samples(x.reshape(-1, 1))
z = np.exp(log_density)

####
## Example
#

####
## Original Data
#
plt.figure()
plt.hist(mresample[5,:], bins=8, density = True)

####
## Gaussian Kernel Interpolation Data PDF
#
plt.figure()
plt.plot(x,z)

####
## Gaussian Kernel Resampled Data Histogram
#
resampled = kernel.sample(n_samples = 10000)[:,0]
plt.figure()
plt.hist(resampled, bins=100, density = True)

####
## Notas
#
'''
Interpolar Kernel Gaussianos para estimar PDF e Resamplear gera pequenas gaussianas. Comparar com GEV.
'''

####
## Resample from Gaussian Kernel Full Matrix
#

mresample,_ = to_tensor(hydrometer_ts,thresh=45)
resampled = np.ndarray((mresample.shape[0], 10000))
#mresample.shape[0]
for i in range(2):
    
  kernel = KernelDensity(kernel = "gaussian", bandwidth=45**(-1.0/5.0))
  kernel.fit( mresample[i, :].reshape(-1, 1) )
  resampled[i,:] = kernel.sample(n_samples = 10000)[:,0]

####
## Gaussian Mixture Mock, Multi - Modal Mock.
#
mock_ts = sample_mixture(resampled[0])
plt.figure()
_ = plt.hist(mock_ts, bins=100, density = True)

LogFrenquencySpectrum?

#### 
## Constant Q-Transform, similar to FFT but is log spaced and center hamming windows with N(k) points. Suitable for music and audio.
# https://en.wikipedia.org/wiki/Constant-Q_transform
#
#Analyze 10s of a natural scene (field recording) using a log frequency scale (constant-Q transform) 
#F = LogFrequencySpectrum(resampled[211,:],nhop=1024, nfft=8192, wfft=4096, npo=24) # constant-Q transform
#F = LogFrequencySpectrum("/content/birds.wav",nhop=1024, nfft=8192, wfft=4096, npo=24) # constant-Q transform
#TODO - Study implementation
data2,fs2,enc = audio.wavread('/content/ID 1.477705.wav')
F = LogFrequencySpectrum(data2,nhop=1, nfft=fs2/4, wfft=500, npo=5) # constant-Q transform 

#Plot the spectrum series and the time-averaged constant-Q spectrum.
pargs = {'normalize':True, 'dbscale':True, 'cmap':cm.hot, 'vmax':0, 'vmin':-45} # plot arguments
subplot(211); F.feature_plot(nofig=True, **pargs) # plot the transform
title('Constant-Q Spectrum (Log Amplitude)', fontsize=14)
subplot(212); plot(20*log10(1+F.X).sum(1)) # and the time-averaged constant-Q spectrum
title('Mean Constant-Q Spectrum: Log Amplitude ', fontsize=14)
xticks(arange(0,100,8), F._logfrqs[0:-1:8].round())
xlabel('Frequency (Hz)'); ylabel('Log Amplitude')
grid();axis('tight');ax=colorbar();ax.ax.set_visible(False)

#Invert the constant-Q transform to an audio signal, using inverse constant-Q transform
#xh = F.inverse()
#audio.play(balance_signal(xh))
audio.wavwrite(balance_signal(xh),'/content/qspect_trimodsparse.wav',fs=10000)
#play(balance_signal(xh)) # Check the resynthesis without sparse coding

# Learn sparse codes from data using dictionary learnng on n x m patches of the constant-Q transform.
s1 = aspa.SparseApproxSpectrum(patch_size=(8,8)) # Set spectrum 2D patch size here (n,m) 
s1.extract_codes(F.X, n_components=9, alpha=1, zscore=True, log_amplitude=True) # Learn a dictionary of patches for spectrogram
# show the learned codes
s1.plot_codes(cbar=True, cmap=cm.hot) 

s1.reconstruct_individual_spectra(plotting=True, **pargs)
plt.figure()
plt.subplot(211) 
feature_plot(F.X, nofig=True, **pargs)
title('Original Spectrogram', fontsize=14)
plt.subplot(212)
feature_plot(s1.X_hat, nofig=True, **pargs)
title('Sparse Approximation Spectrogram', fontsize=14)

# Apply 2D Gabor field to n x m patches of the constant-Q transform.
s2 = aspa.SparseApproxSpectrum(patch_size=(12,12))
s2.make_gabor_field(F.X, thetas=arange(0,4), sigmas=(2,3.5), frequencies=(0.05,0.15), zscore=True, log_amplitude=True) # Generate a dictionary of Gabor patches for spectrogram
s2.plot_codes(cbar=True, cmap=cm.hot) # show the learned codes

# Reconstruct the constant-Q spectrogram using each learned patch basis
s2.reconstruct_individual_spectra(plotting=True, **pargs)
figure()
subplot(211); feature_plot(F.X, nofig=True, **pargs); title('Original Spectrogram', fontsize=14)
subplot(212); feature_plot(s2.X_hat, nofig=True, **pargs); title('Sparse Approximation', fontsize=14)

X_hat = s2.X_hat_l[13] # <- change the patch to reconstruct here
X_hat_ = F.inverse(X_hat) #, Phi_hat=(rand(*F.STFT.shape)*2-1)*pi) 
audio.wavwrite(balance_signal(X_hat_),'/content/qspect_trimodgabor.wav',fs=1000)
#feature_plot(X_hat_, **pargs); title('1 Component Reconstruction', fontsize=14)

####
## Covariance and PCA
# Interpolated and Time-Series dict

m0,nomes0 = to_tensor(interpolated_ts,thresh=45)
m1,_ = to_tensor(hydrometer_ts,thresh=45)

mcov0 = np.cov(m0)
mcov1 = np.cov(m1)
mcov_resampled = np.cov(resampled)

eigv0 = np.linalg.eigvals(mcov0)
eigv1 = np.linalg.eigvals(mcov1)
eigv_resampled = np.linalg.eigvals(mcov_resampled)

print("Autovalores da covariancia 'PCA' para series interpoladas", np.int32(eigv0[0:80]))
print("Autovalores da covariancia 'PCA' para series", np.int32(eigv1[0:80]))
print("Autovalores da covariancia 'PCA' para series reamostrada", np.int32(eigv_resampled))
print("Numero de autovalores nao nulos para series interpoladas", len(np.nonzero(np.int32(eigv0[0:80]))[0]))
print("Numero de autovalores nao nulos para series interpoladas", len(np.nonzero(np.int32(eigv1[0:80]))[0]))
print("Numero de autovalores nao nulos para series reamostrada", len(np.nonzero(np.int32(eigv_resampled))[0]))

####
## Random Normal Data of (Mean,Std) of the sampled variable (hydrometer). For hypothesis of random normal time-series
#

random_dict = {}
for i in range(np.shape(mresample)[0]):
  np.random.seed(i*i-i+127)
  random_dict[str(i)] = np.random.normal(mresample[i,:].mean(),mresample[i,:].std(),10000)

####
## Covariance and PCA
# Null Hypothesis Time-Series

m_null,_ = to_tensor(random_dict,thresh=10000)
mcov_null = np.cov(m_null)
eigv_null = np.linalg.eigvals(mcov_null)

print("Autovalores da covariancia 'PCA' Hipotese Nula", np.int32(eigv_null[0:80]))
print("Numero de autovalores nao nulos Hipotese Nula ", len(np.nonzero(np.int32(eigv_null[0:80]))[0]))

"""## Visualizations

### Time-Series
#### Hydrometers, interpolated, sazonality subtracted versions
#### Auto-correlation function for time-series

###  Histograms
#### Eigenvalue counting for covariance matrix in cases of time-series, interpolated ts and null-hypothesis ts.

### Low Dimensional Embedding
#### T-SNE non-linear manifold embedding.
"""

grid_plot(hydrometer_ts,(3,3))
grid_plot(mocked_data[:,0:50],(4,4))
grid_plot(interpolated_ts,(3,3))

ac,_ = acf(interpolated_ts,15)
grid_plot(ac,(3,4), hidronum = 10)

####
## Plot histogram of eigenvals for covariance matrix of time series.
#

bins = np.linspace(np.sort(eigv0)[0],np.sort(eigv0)[-1]+50,20)
print(bins.astype(np.float16))

plt.figure()
plt.hist(eigv0[0:44],bins=bins.astype(np.float16), density = True)
plt.figure()
plt.hist(eigv1[0:44],bins=bins.astype(np.float16), density = True)
plt.figure()
plt.hist(eigv_resampled,bins=bins.astype(np.float16), density = True)
plt.figure()
plt.hist(eigv_null[0:44],bins=bins.astype(np.float16), density = True)

'''
####
## T-SNE: Non-linear manifold embedding
#
# Notes: 
# - Tricky, and studies on small portions of dataset may be useful to understand the visualization.
# - Non-convex KL divergence loss, many initializations may lead to diverse visualizations.
# - Cluster size dosnt matter, distances are distorted for better view depending on densities.
# - Cluster-cluster distance also dosnt matter.
# - Small non-random like pattern may be random data and effect of the densities.
# - Different perplexities may improve the visualization insights.

 For small noisy time-series:
 - Seems to produce random viz for all 2000.
 For resampled (from Gaussian Kernel Estimator):
 - Low perplexity seems to have a single data cluster and some points, close to what the PCA exhibited.
 For Random Null Hypothesis:
 - Seems to be identical to the Resampled version with small number of points not in the center, in what follows that resample from Gauss Kernel for these quantity of time series may not be enough for extract quality low dim information.
'''

for i in range(0,20,4):
  print(np.float32(10+5*i))
  viz = TSNE(n_components=2, perplexity= np.float32(10+5*i), early_exaggeration=12.0, learning_rate=10.0, n_iter=10000, n_iter_without_progress=300, min_grad_norm=1e-07, metric='euclidean', init='random', verbose=0, random_state=None, method='barnes_hut', angle=0.5)

  t0   = time()
  Y = viz.fit_transform(mocked_data)
  t1 = time()
  print("t-SNE: %.2g sec" % (t1 - t0))
  fig = plt.figure()
  plt.scatter(Y[:, 0], Y[:, 1], c=np.linspace(0,np.size(Y[:, 1]),np.size(Y[:, 1])), cmap=plt.cm.Spectral)
  plt.title("t-SNE (%.2g sec)" % (t1 - t0))
  #ax.xaxis.set_major_formatter(NullFormatter())
  #ax.yaxis.set_major_formatter(NullFormatter())
  plt.axis('tight')
  '''
  count0,n0,_ = plt.hist(Y[:,0])
  count1,n1,_ = plt.hist(Y[:,1])
  print("Acumulação nos eixos 1 ",count0)
  print(n0)
  print("Acumulação nos eixos 2 ",count1)
  print(n1)
  '''

"""## Clustering Measures
### K-Means
### Agglomeration
### Mutual Information/TODO: Time Delayed Mutual Information
"""

####
## Clustering
#

# KMeans
plt.figure()
kclassifier = KMeans(n_clusters = 9, n_init = 40, random_state = 137)
y = kclassifier.fit(invcov0).predict(invcov0)
countm0,_,_ = plt.hist(y,bins=9)

kclassifier = KMeans(n_clusters = 9, n_init = 30, random_state = 451)
y = kclassifier.fit(invcov1).predict(invcov1)
countm1,_,_ = plt.hist(y,bins=9)

kclassifier = KMeans(n_clusters = 9, n_init = 17, random_state = 1337)
y = kclassifier.fit(invcov2).predict(invcov2)
countm2,_,_ = plt.hist(y,bins=9)

kclassifier = KMeans(n_clusters = 9, n_init = 70, random_state = 337)
y = kclassifier.fit(invcov_resampled).predict(invcov_resampled)
countm3,_,_ = plt.hist(y,bins=9)

print('Interp ',countm0)
print('Hydro ',countm1)
print('Null ',countm2)
print('Resampled ',countm3)

####
## Clustering
#

# Agglomeration
plt.figure()
aggclassifier = AgglomerativeClustering(n_clusters=9, affinity='euclidean', linkage='ward')  
y = aggclassifier.fit_predict(invcov0)
countm0,_,_ = plt.hist(y,bins=9)

aggclassifier = AgglomerativeClustering(n_clusters=9, affinity='euclidean', linkage='ward')  
y = aggclassifier.fit_predict(invcov1)
countm1,_,_ = plt.hist(y,bins=9)

aggclassifier = AgglomerativeClustering(n_clusters=9, affinity='euclidean', linkage='ward')  
y = aggclassifier.fit_predict(invcov2)
countm2,_,_ = plt.hist(y,bins=9)


print('Interp ',countm0)
print('Hydro ',countm1)
print('Null ',countm2)

####
## Connect to DB 
#
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=smartwaterhomolog.database.windows.net;DATABASE=SmartWaterHomolog;UID=usr_stattus;PWD=PYIbzXI!')

####
## Flat matrix with time series for SQL allocation
#

flatted = flat_for_sql(mocked_data[:,0:1000])

####
## Using pyodbc to create, add column, fill table with SQL commands to the DB (RDB).
#

cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=smartwaterhomolog.database.windows.net;DATABASE=SmartWaterHomolog;UID=usr_stattus;PWD=PYIbzXI!')
with cnxn.cursor() as cursor:
  cursor = cursor.execute("create table SmartWaterHomolog.guest.mock ( WhichMock varchar (255), MockType varchar (255), DataPoint float(14))")
  cursor = cursor.execute("alter table SmartWaterHomolog.guest.mock add Id int")
  t0 = time()
  cursor.fast_executemany = True    
  for i in range(0,200):
    cursor.executemany("insert into SmartWaterHomolog.guest.mock (Id,WhichMock,MockType,DataPoint) values (?,?,?,?)",flatted[i*500:(i+1)*500])
  cnxn.commit()
  t1 = time()
  print(t1-t0)

####
## Query rows from a table with SQL
#

cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=smartwaterhomolog.database.windows.net;DATABASE=SmartWaterHomolog;UID=usr_stattus;PWD=PYIbzXI!')
with cnxn.cursor() as cursor:
  rows_with_count = cursor.execute(" select * from SmartWaterHomolog.guest.mock where Id in (select Id from SmartWaterHomolog.guest.mock group by Id having count(*) > 1) ").fetchall()
  time_series = cursor.execute(" select * from SmartWaterHomolog.guest.mock where Id = 4; ").fetchall()

####
## Bulk Insert SQL Syntax
#
with open("/home/stattus4dpenalva/output.csv", "wb") as f:
  writer = csv.writer(f)
  writer.writerows(flatted)
'''
BULK INSERT   
   { database_name.schema_name.table_or_view_name | schema_name.table_or_view_name | table_or_view_name }
      FROM 'data_file'   
     [ WITH   
    (   
   [ [ , ] BATCHSIZE = batch_size ]   
   [ [ , ] CHECK_CONSTRAINTS ]   
   [ [ , ] CODEPAGE = { 'ACP' | 'OEM' | 'RAW' | 'code_page' } ]   
   [ [ , ] DATAFILETYPE =   
      { 'char' | 'native'| 'widechar' | 'widenative' } ]   
   [ [ , ] DATASOURCE = 'data_source_name' ]
   [ [ , ] ERRORFILE = 'file_name' ]
   [ [ , ] ERRORFILE_DATA_SOURCE = 'data_source_name' ]   
   [ [ , ] FIRSTROW = first_row ]   
   [ [ , ] FIRE_TRIGGERS ]   
   [ [ , ] FORMATFILE_DATASOURCE = 'data_source_name' ]
   [ [ , ] KEEPIDENTITY ]   
   [ [ , ] KEEPNULLS ]   
   [ [ , ] KILOBYTES_PER_BATCH = kilobytes_per_batch ]   
   [ [ , ] LASTROW = last_row ]   
   [ [ , ] MAXERRORS = max_errors ]   
   [ [ , ] ORDER ( { column [ ASC | DESC ] } [ ,...n ] ) ]   
   [ [ , ] ROWS_PER_BATCH = rows_per_batch ]   
   [ [ , ] ROWTERMINATOR = 'row_terminator' ]   
   [ [ , ] TABLOCK ]   

   -- input file format options
   [ [ , ] FORMAT = 'CSV' ]
   [ [ , ] FIELDQUOTE = 'quote_characters']
   [ [ , ] FORMATFILE = 'format_file_path' ]   
   [ [ , ] FIELDTERMINATOR = 'field_terminator' ]   
   [ [ , ] ROWTERMINATOR = 'row_terminator' ]   
    )]   
    
import pypyodbc
conn_str = "DSN=myDb_SQLEXPRESS;"
cnxn = pypyodbc.connect(conn_str)
crsr = cnxn.cursor()
sql = """
BULK INSERT myDb.dbo.SpikeData123
FROM 'C:\\__tmp\\biTest.txt' WITH (
    FIELDTERMINATOR='\\t',
    ROWTERMINATOR='\\n'
    );
"""

sql = """
   bulk insert
       SmartWaterHomolog.guest.mock
       from '/home/stattus4dpenalva/output.csv' 
       with
       (
        format = 'CSV',
        fieldterminator = ',',   
        rowterminator = '\\n' 
       );
       
       """
crsr.execute(sql)
cnxn.commit()
crsr.close()
cnxn.close()
'''

np.flatiter?
####
## Dumping Metrics in the lasts cells ...
#

####
## Inverse Covariance
#

invcov0 = np.array([np.array([ 1.0/(mcov0[i,j]+1e-7) for j in range(np.size(mcov0[0,:]) ) ]) for i in range( np.size(mcov0[0,:]))])
invcov1 = np.array([np.array([ 1.0/(mcov1[i,j]+1e-7) for j in range(np.size(mcov1[0,:]) ) ]) for i in range( np.size(mcov1[0,:]))])
invcov_resampled = np.array([np.array([ 1.0/(mcov_resampled[i,j]+1e-7) for j in range(np.size(mcov_resampled[0,:]) ) ]) for i in range( np.size(mcov_resampled[0,:]))])
invcov2 = np.array([np.array([ 1.0/(mcov_null[i,j]+1e-7) for j in range(np.size(mcov_null[0,:]) ) ]) for i in range( np.size(mcov_null[0,:]))])

func = lambda x,t,k : np.array([[0.0,1.0],[-k,-0.0]]).dot(x.T)+np.array([0.0,0.00])
x0 = np.array([5.0,-1.0])
k = 0.005
N = 11280
t = np.linspace(0,(N-1),N)
y = scp.integrate.odeint(func,x0,t,args=(k,))
y = y + 25.0
  
plt.plot(t[0:200],y[0:200,0])

def sample_gaussmix(data=None,N=500, mean_base = 0.0,var_base = 0.15, num_mods = 30, random_mods=False)
  '''
    Generate sample noise from num_mods different gaussian with data or only noise.
    
    TODO: select mod from a list of tuples with (mean,variance)
  '''
  num_mods = int(num_mods)
  if type(data) != None:
    shape = np.shape(data)
  c = np.int(shape[0]/num_mods)
  np.random.set_state = 1440*shape[0]    

  if type(data) != None:
    if random_mods:
      if len(shape) > 1:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),c) for i in range(num_mods)]).flatten() )    
    else:
      if len(shape) > 1:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (data + np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),c) for i in range(num_mods)]).flatten() )      
  else:
    if random_mods:
      if len(shape) > 1:
        mix = lambda data: (np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (np.array([np.random.normal(mean_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),var_base*np.mod(np.random.randint((i+1),high=1234*i+4),num_mods),c) for i in range(num_mods)]).flatten() )    
    else:
      if len(shape) > 1:
        mix = lambda data: (np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),(c,shape[1:]) ) for i in range(num_mods)]).flatten() )
      else:
        mix = lambda data: (np.array([np.random.normal(mean_base*(i+1),var_base*(i+1),c) for i in range(num_mods)]).flatten() )      
  return mix(data)

vazao0 = np.abs((sample-np.mean(sample))/(np.std(sample,ddof=1)+1e-8))

vazao = np.zeros((len(vazao0),))
for i in np.arange(1,len(vazao0)):
  vazao[i] = vazao[i-1]+vazao0[i-1]

table = pd.DataFrame(data=zip(vazao,sample),columns=['volume','pressão'])
table.head(60)

with open('/content/mock.csv','w') as f:
  table.to_csv(f)