#+OPTIONS: toc:nil tex:t H:6 date:t author:nil tags:nil num:nil
#+OPTIONS: html5-fancy:t
#+OPTIONS: html-link-use-abs-url:nil html-postamble:auto
#+OPTIONS: html-preamble:t html-scripts:t html-style:t
#+STARTUP: hideblocks
#+SELECT_TAGS: export
#+EXCLUDE_TAGS: noexport deprecated
#+PROPERTY: header-args :session dewakss :results silent :exports both :eval never-export :comments link
#+PROPERTY: header-args:ipython :shebang "#!/usr/bin/env python" :session dewakss
#+LATEX_HEADER: \usepackage{natbib}
#+LATEX_HEADER: \usepackage[nomarkers,figuresonly]{endfloat}
#+title: DEWAKSS EMT MAGIC data set


* Apply DEWAKSS to EMT data from MAGIC publication

Initialize a reasonable python session.
#+name: initiate-sc-session
#+begin_src ipython :exports code :results silent :noweb yes
%matplotlib tk
%load_ext autoreload
%autoreload 2
<<initialize-ob-ipython-session>>
<<load-ob-ipython-libraries>>
<<set-ob-ipython-default-plot-configs>>
<<get-branch-in-git>>
import scanpy as sc

gitbranch = os.path.join(gitbranch, 'EMT_MAGIC')

figdir = os.path.join("..", "img", gitbranch)
datadir = os.path.join("..", "data", gitbranch)

sc.settings.figdir = figdir
sc.settings.file_format_figs = "svg"

import scprocessing.pipeline as scpipe
import scprocessing.preprocessing as scpp
import scprocessing.preprocessing.Svensson2019 as Svensson2019
import scprocessing.plotting.anndata as scpl
import dewakss.decomposition as dede
import dewakss.denoise as dewakss
#+end_src

** On MAGIC example

The epithelial-to-mesenchymal transition (EMT) data from cite:MAGIC and [[https://nbviewer.jupyter.org/github/KrishnaswamyLab/magic/blob/master/python/tutorial_notebooks/emt_tutorial.ipynb][here]].

#+name: load-data-EMT_MAGIC
#+begin_src ipython
import scprep
from scipy.sparse import csr

emt_data = scprep.io.load_tsv("https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE114397&format=file&file=GSE114397_HMLE_TGFb.tsv.gz")

# Convert to scanpy
adata = sc.AnnData(csr.csr_matrix(emt_data.values), obs=emt_data.index.astype(str), var=emt_data.columns.astype(str))
adata.var.columns = ['ID']

# adata.obs['n_counts'] = adata.X.sum(axis=1).A1
# adata.obs['n_genes'] = adata.X.astype(bool).sum(axis=1).A1
sc.pp.filter_cells(adata, min_counts=1500)
sc.pp.filter_genes(adata, min_cells=10)

adata.layers['counts'] = adata.X.copy()
sc.pp.normalize_per_cell(adata)
# sc.pp.normalize_per_cell(adata, counts_per_cell_after=np.median(adata.obs['n_counts']))
scpp.ftt(adata)
adata.raw = adata
del emt_data

sc.pp.highly_variable_genes(adata, min_mean=0.25)
adata.write(os.path.join(datadir, "loaded_data.h5ad"))
#+end_src

#+name: reload-data-EMT
#+begin_src ipython
adata = sc.read(os.path.join(datadir, "loaded_data.h5ad"))
#+end_src

First lets check if the dataset is highly enriched for zero counts given our assumptions of negative binomial distribution. I'm using the approach of cite:Svensson2019 where red line indicate our predicted distribution and with fitting of the model of over dispersion. The parameter \(\phi\) is fitted to the data. The first plot (left) plots the variance versus mean of the observed data compared to the predicted and the second plot (right) plots the probability of zeros given for our data compared to the predicted.
#+name: EMT_MAGIC-check-basic-stats
#+begin_src ipython :results output drawer replace
adata = Svensson2019.add_statistics(adata, use_layer='counts', copy=True)
fig, ax = Svensson2019.stats_vs_mean(scatter_data = adata.var[['mean_', 'var_', 'frac_zero']], phi_vec=adata.uns['phi_hat'], logrange=(-3,3), rho_var=adata.uns['var_corr'], rho_zero=adata.uns['zero_corr'])

fdir = os.path.join(figdir, 'statistics')
fname = f"EMT_MAGIC_NB_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: EMT_MAGIC-check-basic-stats
:results:
[[file:../img/master/EMT_MAGIC/statistics/EMT_MAGIC_NB_figure.png]]
:end:

#+name: find-optimal-n-pcs
#+begin_src ipython
from sklearn.decomposition import TruncatedSVD

DeTSVD = dede.decomposition_wrapper(TruncatedSVD)
rescaler = {sc.pp.normalize_per_cell: {"copy": True}, scpp.ftt: {'copy': True}}
# rescaler = {sc.pp.normalize_per_cell: {"counts_per_cell_after": np.median(adata.obs['n_counts']), "copy": True}}
dpca = DeTSVD(strategy='binomial', rescaler=rescaler, n_components=50, subsample=None, test_size=None)

dpca.fit(adata.layers['counts'].copy())
#+end_src

#+name: select-optimal-n_pca-EMT_MAGIC
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(10,4), constrained_layout=True)
ax = fig.subplots(1, 2, sharex=True).flatten()

optimalpcs = dpca.optimal_
ax[0].plot(dpca.rank_range, dpca.mse_)
ax[0].grid()
ax[0].axvline(optimalpcs, color='k', linestyle='--')
ax[0].set_ylabel('MSE')

ax[1].plot(dpca.rank_range, dpca.explained_variance_ratio_[dpca.rank_range])
ax[1].grid()
ax[1].axvline(optimalpcs, color='k', linestyle='--')
ax[1].set_xlabel('n PCs')
ax[1].set_ylabel('EV')

sns.despine()
fig.suptitle(f"Selecting n PCs = {optimalpcs}, EMT MAGIC")

fdir = figdir
fname = f"EMT_MAGIC_selecting_npcs_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
#+end_src

#+RESULTS: select-optimal-n_pca-EMT_MAGIC
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_selecting_npcs_figure.png]]
:end:

Note that fewer knn seem to genereate higher number of iterations for smoothings as optimal.
#+name: run-pipeline-with-optimal-npcs
#+begin_src ipython
scpipe.base_computations(adata, npcs=dpca.optimal_, nneighbors=30, min_dist=0.5)
scpipe.rank_genes_groups(adata, groupby='leiden')

g1 = adata.var[adata.var['ID'] == 'CDH1'].index[0]
g2 = adata.var[adata.var['ID'] == 'VIM'].index[0]
g3 = adata.var[adata.var['ID'] == 'ZEB1'].index[0]

gene_mapping = {'CDH1': g1, 'VIM': g2, 'ZEB1': g3}
#+end_src

#+name: plot-EMT_MAGIC-marker-genes-raw
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(6,5), constrained_layout=True)
ax = fig.subplots(2, 2).flatten()

for i, (g, v) in enumerate(gene_mapping.items()):
    sc.pl.umap(adata, color=v, cmap='inferno', ax=ax[i])
    ax[i].set_title(g)
    # for i, j in enumerate([0, 2, 4]):
    #     ax[j].set_title(list(gene_mapping.keys())[i])

__ = list(gene_mapping.values())

cax2 = ax[3].scatter(adata[:, __[1]].X, adata[:, __[0]].X, c=adata[:, __[2]].X, cmap='inferno', s=1)
__ = list(gene_mapping.keys())
ax[3].set_xlabel(__[1])
ax[3].set_ylabel(__[0])
ax[3].set_title('Raw Data', fontsize=14)
cbar = fig.colorbar(cax2, ax=ax[3], label=__[2], aspect=30)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_raw_umap_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")


fig = plt.figure(figsize=(6,5), constrained_layout=True)
ax = fig.subplots(2, 2).flatten()

for i, (g, v) in enumerate(gene_mapping.items()):
    sc.pl.pca(adata, color=v, cmap='inferno', ax=ax[i])
    ax[i].set_title(g)
    # for i, j in enumerate([0, 2, 4]):
    #     ax[j].set_title(list(gene_mapping.keys())[i])

__ = list(gene_mapping.values())

cax2 = ax[3].scatter(adata[:, __[1]].X, adata[:, __[0]].X, c=adata[:, __[2]].X, cmap='inferno', s=1)
__ = list(gene_mapping.keys())
ax[3].set_xlabel(__[1])
ax[3].set_ylabel(__[0])
ax[3].set_title('Raw Data', fontsize=14)
cbar = fig.colorbar(cax2, ax=ax[3], label=__[2], aspect=30)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_raw_pca_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-EMT_MAGIC-marker-genes-raw
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_raw_umap_figure.png]]
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_raw_pca_figure.png]]
:end:

** Vizsualize optimal denoising

#+name: load-performance-data
#+begin_src ipython
performance_data = pd.read_csv(os.path.join(datadir.replace('simplify-dewakss', 'master'), "emt_results", "performance_dewakss_emt.csv"), sep='\t', index_col=0)
del performance_data['Unnamed: 0.1']
#+end_src

#+name: plot-performance-hyper-parameters
#+begin_src ipython :results output drawer replace
dosave = False
# pdata = performance_data[performance_data['symmetrize'] == False]
pdata = performance_data.copy().sort_values(['neighbors', 'iteration'])
for (mode, dt), df in pdata.groupby(['mode', 'denoisetype']):

    metric = 'MSE'
    combos = df[['neighbors']].drop_duplicates()

    fig = plt.figure(figsize=(14, 3.5), constrained_layout=True)

    fold = 1
    ax = fig.subplots(fold, combos.shape[0]//fold, sharex=True, sharey='row').flatten(order='F')

    combos['axes'] = ax
    combos = combos.set_index(['neighbors'])

    max_xticks = 0
    for (neighbors, pcs), subdf in df.groupby(['neighbors', 'pcs']):
        axes = combos.loc[neighbors][0]
        subdf = subdf[~(subdf['iteration'] == 0)]
        axes.plot(subdf['iteration'].values, subdf[metric].values, label=pcs, zorder=-pcs+1000, linewidth=2)
        axes.legend().set_visible(False)
        axes.set_xlabel('iteration')
        axes.set_ylabel(f"{metric}")

        if subdf['iteration'].values.max() > max_xticks:
            axes.set_xticks(subdf['iteration'].values)
            max_xticks = subdf['iteration'].values.max()

        axes.set_title(f"k={neighbors}")
        axes.grid(linewidth=0.5, linestyle='--')
        axes.label_outer()

    ax[-1].legend(title='PCs', loc='center right')

    if metric == 'MSE':
        optind = df.groupby(['neighbors'])[metric].min()
    elif metric == 'R2':
        optind = df.groupby(['neighbors'])[metric].max()
        
    optit = df.set_index(['neighbors'])
    for (neighbors), value in combos.iterrows():
        axes = value[0]
        minmse = optind.loc[neighbors]
        opts = (optit.loc[neighbors][metric] == minmse).values
        its = optit.loc[neighbors][opts]['iteration'][neighbors]
        optpcs = optit.loc[neighbors][opts]['pcs'][neighbors]
        sns.despine()
        ylims = np.array(axes.get_ylim())
        axes.vlines([its, its], *(ylims), zorder=500, linestyle=':')
        hl = 'left' if its < 10 else 'right'
        xl = its+1 if its < 10 else its-1

        axes.text(xl, ylims[1], f"MSE={minmse:.4f}\nPCs={optpcs}", ha=hl, va='top')
        axes.set_ylim(*ylims)

    if metric == 'MSE':
        opte = optit[optit[metric] == optind.min()]
    elif metric == 'R2':
        opte = optit[optit[metric] == optind.max()]
    
    fig.suptitle(f"Denoise type={dt}, {mode}\nOptimal: MSE={opte['MSE'].iloc[0]:.4f}, it={opte['iteration'].iloc[0]}, PCs={opte['pcs'].iloc[0]}, k={opte.reset_index()['neighbors'][0]}")

    if dosave:
        fdir = figdir
        fname = f"denoise_type_{dt}_{mode}_{metric}_hyper_paramters_"
        fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
        print_file = "[[file:" + fnames[0] + "]]"
        print(print_file, sep=",", end="")
        print("")
#+end_src

#+RESULTS: plot-performance-hyper-parameters
:results:
[[file:../img/master/EMT_MAGIC/denoise_type_mean_connectivities_MSE_hyper_paramters_figure.png]]
[[file:../img/master/EMT_MAGIC/denoise_type_mean_distances_MSE_hyper_paramters_figure.png]]
:end:

#+name: performance-trends
#+begin_src ipython :results output drawer replace
doplot = True
metric = 'MSE'
# pdata = performance_data[performance_data['symmetrize'] == False]
pdata = performance_data.copy()
pdata = pdata.groupby(['mode', 'denoisetype', 'pcs', 'neighbors'])[metric].min().reset_index()
style_label = ('seaborn-poster')

for dt, df in pdata.groupby(['denoisetype']):

    with plt.style.context(style_label):

        g = sns.lmplot(hue="pcs", y="MSE", x="neighbors", col='mode', truncate=True, data=df, ci=None, fit_reg=False, height=6, aspect=0.6)

        for ax in g.axes.flatten():
            ax.grid()
            ax.set_ylim([df['MSE'].min()-df['MSE'].min()/500, df['MSE'].max()+df['MSE'].min()/500])
            ax.set_xlim([4, 500])
            ax.set_xscale('log')

        fig = g.fig
        # fig.suptitle(f"Denoise type={dt}")

    if doplot:
        fdir = figdir
        fname = f"denoise_type_{dt}_{metric}_minimal_trend_hyper_paramters_"
        fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
        print_file = "[[file:" + fnames[0] + "]]"
        print(print_file, sep=",", end="")
        print("")
#+end_src

#+RESULTS: performance-trends
:results:
[[file:../img/simplify-dewakss/EMT_MAGIC/denoise_type_mean_MSE_minimal_trend_hyper_paramters_figure.png]]
:end:

#+name: get-optimal-parameters
#+begin_src ipython :results output drawer replace
print(performance_data.loc[performance_data['MSE'].argmin()])
#+end_src

#+RESULTS: get-optimal-parameters
:results:
iteration              1
MSE                0.222
R2               0.00738
mode           distances
neighbors            100
pcs                  100
denoisetype         mean
Name: 392, dtype: object
:end:

#+name: plot-diffuse-trend
#+begin_src ipython :results output drawer replace
doplot = False
# style_list = ['default', 'classic'] + sorted(style for style in plt.style.available if style != 'classic')
# style_label = 'fivethirtyeight'
style_label = 'seaborn-deep'
colors = ["windows blue", "faded green", 'rose', "amber", "greyish", "pale red", "dusty purple", "denim blue", "medium green", 'olive']
pal = sns.xkcd_palette(colors)

pcs = [5, 100, 400]
for npcs in pcs:
    opt_pcs = performance_data[performance_data['pcs'] == npcs]
    # opt_pcs = performance_data.copy()
    opt_pcs = opt_pcs[opt_pcs['mode'] == 'distances']

    with plt.style.context(style_label):
        fig = plt.figure(figsize=(5, 4), constrained_layout=True)
        axes = fig.subplots(1, 1)

        i = 0
        for (neighbors, pcs), subdf in opt_pcs.groupby(['neighbors', 'pcs']):
            subdf = subdf[~(subdf['iteration'] == 0)]

            axes.plot(subdf['iteration'].values, subdf[metric].values, zorder=neighbors-1000, linewidth=2, color=pal[i]) # , color=pal[i]
            mini = subdf[metric].values.argmin()
            axes.scatter(subdf['iteration'].values[mini], subdf[metric].values[mini], s=100, zorder=neighbors-1000, label=neighbors, color=pal[i]) # , color=pal[i]

            i=i+1

        axes.set_xlabel('diffusion step')
        axes.set_ylabel(f"{metric}")

        axes.set_xticks(np.arange(1, opt_pcs['iteration'].max()+1))
        axes.set_title(f'# PCs = {npcs}')

        # axes.set_xlim([0.5,19.5])
        axes.set_ylim([0.221, 0.23])
        axes.legend(title='neighbors', loc='center left', bbox_to_anchor=(1, 0.5))

        sns.despine(offset=10)
        axes.grid(linewidth=0.5, linestyle='--', zorder=10000)
        # axes.label_outer()

    if doplot:
        fdir = figdir
        fname = f"EMT_MAGIC_denoise_type_{dt}_{metric}_npcs_{npcs}_neighbours_diffuse_trend_"
        fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
        print_file = "[[file:" + fnames[0] + "]]"
        print(print_file, sep=",", end="")
        print("")
#+end_src

#+RESULTS: plot-diffuse-trend
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_denoise_type_mean_MSE_npcs_5_neighbours_diffuse_trend_figure.png]]
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_denoise_type_mean_MSE_npcs_100_neighbours_diffuse_trend_figure.png]]
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_denoise_type_mean_MSE_npcs_400_neighbours_diffuse_trend_figure.png]]
:end:

** Optimal Denoising

#+name: apply-optimal-dewakss
#+begin_src ipython
import scipy as sp
import dewakss.denoise as dewakss
tmpadata = adata.copy()
pcs = 100
N = 100
sc.pp.pca(tmpadata, n_comps=pcs, use_highly_variable=False)
sc.pp.neighbors(tmpadata, n_neighbors=N, n_pcs=pcs)

denoiseer = dewakss.DEWAKSS(tmpadata, mode='distances')
denoiseer.fit(tmpadata)
denoiseer.transform(tmpadata, copy=False)
adata.layers['norm'] = adata.X.A.copy()
adata.X = tmpadata.layers['Ms'].toarray() if sp.sparse.issparse(tmpadata.layers['Ms']) else tmpadata.layers['Ms']

del tmpadata
#+end_src

#+name: save-denoised-computed-subset
#+begin_src ipython
adata.write(os.path.join(datadir, f"EMT_MAGIC_denoised_data_.h5ad"))
#+end_src

** Apply self supervised denoising

#+name: load-data
#+begin_src ipython
adata = sc.read(os.path.join(datadir, f"EMT_MAGIC_denoised_data_.h5ad"))
#+end_src

#+name: plot-EMT_MAGIC-marker-genes-denoised
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(6,5), constrained_layout=True)
ax = fig.subplots(2, 2).flatten()

adata_dn = adata.copy()
adata_dn.layers['Ms'] = adata.X.copy()
sc.pp.pca(adata_dn)

for i, (g, v) in enumerate(gene_mapping.items()):
    # cax2 = sc.pl.umap(adata, color=v, cmap='inferno', ax=ax[i], layer='Ms', use_raw=False)
    cax2 = sc.pl.pca(adata_dn, color=v, cmap='inferno', ax=ax[i], layer='Ms', use_raw=False)
    ax[i].set_title(g)

__ = list(gene_mapping.values())

# cax2 = ax[3].scatter(adata[:, __[1]].layers["Ms"], adata[:, __[0]].layers["Ms"], c=adata[:, __[2]].layers["MS"], cmap='inferno', s=1)
cax2 = ax[3].scatter(adata[:, __[1]].X, adata[:, __[0]].X, c=adata[:, __[2]].X, cmap='inferno', s=1)

__ = list(gene_mapping.keys())
ax[3].set_xlabel(__[1])
ax[3].set_ylabel(__[0])
ax[3].set_title('Denoise Data', fontsize=14)
cbar = fig.colorbar(cax2, ax=ax[3], label=__[2], aspect=30)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_denoised_pca_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-EMT_MAGIC-marker-genes-denoised
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_denoised_pca_figure.png]]
:end:

#+name: plot-single-color-pca
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(6,6), constrained_layout=True)
ax = fig.subplots(2, 2).flatten()

adata_dn = adata.copy()
adata_dn.X = adata.X.copy()
sc.pp.pca(adata_dn)

cax2 = sc.pl.scatter(adata_dn, basis='pca', color='#3778bf', ax=ax[0], use_raw=False)
ax[0].set_title('Denoised')
cax2 = sc.pl.scatter(adata, basis='pca', color='#3778bf', ax=ax[1], use_raw=False)
ax[1].set_title('Initial')

cax2 = sc.pl.scatter(adata_dn, basis='pca', color='#3778bf', ax=ax[2], use_raw=False, components='3,4')
# ax[2].set_title('Denoised')
cax2 = sc.pl.scatter(adata, basis='pca', color='#3778bf', ax=ax[3], use_raw=False, components='3,4')
# ax[3].set_title('Initial')

fdir = figdir
fname = f"EMT_MAGIC_denoised_pca_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-single-color-pca
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_denoised_pca_figure.png]]
:end:

Note that fewer knn seem to genereate higher number of iterations for smoothings as optimal.
#+name: recompute-pipeline-with-denoised
#+begin_src ipython
scpipe.base_computations(adata, npcs=23, nneighbors=30, min_dist=0.5)
scpipe.rank_genes_groups(adata, groupby='leiden')

g1 = adata.var[adata.var['ID'] == 'CDH1'].index[0]
g2 = adata.var[adata.var['ID'] == 'VIM'].index[0]
g3 = adata.var[adata.var['ID'] == 'ZEB1'].index[0]
gene_mapping = {'CDH1': g1, 'VIM': g2, 'ZEB1': g3}
#+end_src

#+name: plot-EMT_MAGIC-marker-genes-denoised-umap
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(6,5), constrained_layout=True)
ax = fig.subplots(2, 2).flatten()

adata_dn = adata.copy()
adata_dn.layers['Ms'] = adata.X.copy()
sc.pp.pca(adata_dn)

for i, (g, v) in enumerate(gene_mapping.items()):
    # # cax2 = sc.pl.umap(adata, color=v, cmap='inferno', ax=ax[i], layer='Ms', use_raw=False)
    cax2 = sc.pl.umap(adata_dn, color=v, cmap='inferno', ax=ax[i], layer='Ms', use_raw=False)
    ax[i].set_title(g)

__ = list(gene_mapping.values())

# cax2 = ax[3].scatter(adata[:, __[1]].layers["Ms"], adata[:, __[0]].layers["Ms"], c=adata[:, __[2]].layers["MS"], cmap='inferno', s=1)
cax2 = ax[3].scatter(adata[:, __[1]].X, adata[:, __[0]].X, c=adata[:, __[2]].X, cmap='inferno', s=1)

__ = list(gene_mapping.keys())
ax[3].set_xlabel(__[1])
ax[3].set_ylabel(__[0])
ax[3].set_title('Denoise Data', fontsize=14)
cbar = fig.colorbar(cax2, ax=ax[3], label=__[2], aspect=30)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_denoised_umap_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-EMT_MAGIC-marker-genes-denoised-umap
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_denoised_umap_figure.png]]
:end:


*** COMMENT DEPRECATED Oversmooth analysis

#+name: apply-optimal-dewakss
#+begin_src ipython
import scipy as sp
import dewakss.denoise as dewakss
tmpadata = adata.copy()
pcs = 100
N = 100
sc.pp.pca(tmpadata, n_comps=pcs, use_highly_variable=False)
sc.pp.neighbors(tmpadata, n_neighbors=N, n_pcs=pcs)

denoiseer = dewakss.DEWAKSS(tmpadata, mode='distances')
denoiseer.fit(tmpadata)
denoiseer.transform(tmpadata, copy=False)
adata.layers['norm'] = adata.X.A.copy()
adata.X = tmpadata.layers['Ms_over'].toarray() if sp.sparse.issparse(tmpadata.layers['Ms_over']) else tmpadata.layers['Ms_over']

del tmpadata
#+end_src


#+name: plot-EMT_MAGIC-marker-genes-denoised-over-smoothing-100pcs
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(12,4), constrained_layout=True)
ax = fig.subplots(1, 3).flatten(order='F')

__ = list(gene_mapping.values())

cax1 = ax[1].scatter(tmpadata[:, __[1]].layers["Ms_over"], tmpadata[:, __[0]].layers["Ms_over"], c=tmpadata[:, __[2]].layers["Ms_over"], cmap='inferno', s=1)

cax0 = ax[0].scatter(tmpadata[:, __[1]].layers["Ms"], tmpadata[:, __[0]].layers["Ms"], c=tmpadata[:, __[2]].layers["Ms"], cmap='inferno', s=1)
__ = list(gene_mapping.keys())
ax[0].set_xlabel(__[1])
ax[0].set_ylabel(__[0])
ax[0].set_title('Optimal denoised data', fontsize=14)

ax[1].set_xlabel(__[1])
ax[1].set_ylabel(__[0])
ax[1].set_title('Over denoised data', fontsize=14)
cbar = fig.colorbar(cax1, ax=ax[1], label=__[2], aspect=30)
cbar = fig.colorbar(cax0, ax=ax[0], label=__[2], aspect=30)

dewaxer.plot(ax=ax[2], verbose=False)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_denoised_vs_over_smoothing_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=300)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-EMT_MAGIC-marker-genes-denoised-over-smoothing-100pcs
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_denoised_vs_over_smoothing_figure.png]]
:end:

#+name: plot-EMT_MAGIC-marker-genes-denoised-over-smoothing-umap-100pcs
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(8,5), constrained_layout=True)
ax = fig.subplots(2, 3).flatten(order='F')

for i, (g, v) in enumerate(gene_mapping.items()):
    cax2 = sc.pl.umap(tmpadata, color=v, cmap='inferno', ax=ax[i], layer='Ms', use_raw=False)
    ax[i].set_title(g)

__ = list(gene_mapping.values())

cax2 = ax[3].scatter(tmpadata[:, __[1]].layers["Ms"], tmpadata[:, __[0]].layers["Ms"], c=tmpadata[:, __[2]].layers["Ms"], cmap='inferno', s=1)
__ = list(gene_mapping.keys())
ax[3].set_xlabel(__[1])
ax[3].set_ylabel(__[0])
ax[3].set_title('Denoise Data', fontsize=14)
cbar = fig.colorbar(cax2, ax=ax[3], label=__[2], aspect=30)

dewaxer.plot(ax=ax[4], verbose=False)
ax[4].set_title(f"Run time: {dewaxer._extime:.2f}s")
dewaxer.plot(ax=ax[5], metric='r2', verbose=False)

fdir = figdir
fname = f"EMT_MAGIC_markergenes_denoised_umap_over_smoothing_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-EMT_MAGIC-marker-genes-denoised-over-smoothing-umap-100pcs
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_markergenes_denoised_umap_over_smoothing_figure.png]]
:end:

** COMMENT DEPRECATED Robustness of preprocessing

It is vital that the method is robust against choices of imput paramters that we do not select optimally. To this end we run DEWAKSS with a varying number of imput PCs and a varying number of neighbours for the KNN-G. We would expect that an optimal number of PCs would make the selection of neighbours less sensitive and allow for more iterations to be performed while the suseptability to over smooth would be larger for more PCs and therefore generate fewer iterations to arrive at an optimal smoothing as more iterations would alow for to much smoothing to be applied. The same argument would hold for the number of neighbours. Fewer neighbours need to gather information from a larger area to find an optimal smoothing while a large number of neiughbours should more easily cause an over smoothing leading to a worse fit.

To test how the method cope with varying input parameters from the preprocessing we pick the optimal number of PCs and vary the number of neighbours in the network.
#+name: run-robustness-to-neighbours-with-optimal-pcs
#+begin_src ipython
npcs = dpca.optimal_
sc.pp.pca(adata, n_comps=npcs)

neighbours = [3, 4, 5, 10, 20, 50]

hyperp = []
for n in neighbours:
    sc.pp.neighbors(adata, n_neighbors=n, n_pcs=npcs, random_state=42)
    # sc.tl.umap(adata, random_state=42)
    dewaksser = dewakss.DEWAKSS(adata, init_diag=0, set_diag=0,  iterations=15, decay=1, init_thresholding=False, symmetrize=False, run2best=False, mode='connectivities', subset=0.1)
    dewaksser.fit(adata, iterations=15)
    df = pd.DataFrame(dewaksser.prediction_).T
    
    df = df.reset_index()
    df.columns = ['iteration', 'MSE', 'R2']
    df['neighbours'] = n

    hyperp.append(df)

hyperp = pd.concat(hyperp).reset_index(drop=True)
#+end_src

#+name: plot-EMT_MAGIC-optimal-pca-hyper-parameter-neighbours
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(7,5), constrained_layout=True)
ax = fig.subplots(1, 1)

for i, (name, df) in enumerate(hyperp.groupby('neighbours')):
    pax = df.plot(x='iteration', y='MSE', label=name, ax=ax, zorder=100-int(name), linewidth=2, markersize=10, marker='.')
    optval = df['iteration'].values[np.argmin(df['MSE'].values)]
    ax.axvline(optval + i%2/50, color=pax.get_lines()[-1].get_color(), linestyle='--')

ax.set_ylabel('MSE')
ax.set_xlabel('iteration')
ax.legend(title='# neighbours', loc=(0.7, 0.6))
ax.set_xticks(hyperp['iteration'].drop_duplicates().values)
ax.set_title(f"{dpca.optimal_} PCs")
ax.grid()
sns.despine(offset=10)

fdir = figdir
fname = f"EMT_MAGIC_opt_pca_hyper_parameter_neighbours_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

For the number of neighbours our intuition hold. Namely that fewever neighbours allow for more fine grained tuning of the number iterations. With 50 neighbours being optimized already at 1 iteration followed by and 20 and 10 sharing 2 iterations and 5, 4 and 3 neighbors at 4, 6, and 12 iterations respectively.
#+RESULTS: plot-EMT_MAGIC-optimal-pca-hyper-parameter-neighbours
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_opt_pca_hyper_parameter_neighbours_figure.png]]
:end:

Next we increase the number of PCs used for calculating the KNN-G. Here to 100 which is the MAGIC default setting. This would allow more variation to be incorporated in the KNN-G and as a consequence our method should want to make fewer iterations to attenuate the influence of noise. 
#+name: run-robustness-to-neighbours-with-100-pcs
#+begin_src ipython
npcs = 100
sc.pp.pca(adata, n_comps=npcs)

neighbours = [3, 4, 5, 10, 20, 50]

hyperp = []
for n in neighbours:
    sc.pp.neighbors(adata, n_neighbors=n, n_pcs=npcs, random_state=42)
    # sc.tl.umap(adata, random_state=42)
    dewaksser = dewakss.DEWAKSS(adata, init_diag=0, set_diag=0,  iterations=15, decay=1, init_thresholding=False, symmetrize=False, run2best=False, mode='connectivities', subset=0.1)
    dewaksser.fit(adata, iterations=10)
    df = pd.DataFrame(dewaksser.prediction_).T
    
    df = df.reset_index()
    df.columns = ['iteration', 'MSE', 'R2']
    df['neighbours'] = n

    hyperp.append(df)

hyperp = pd.concat(hyperp).reset_index(drop=True)
#+end_src

#+name: plot-EMT_MAGIC-100-pca-hyper-parameter-neighbours
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(7,5), constrained_layout=True)
ax = fig.subplots(1, 1)

for i, (name, df) in enumerate(hyperp.groupby('neighbours')):
    pax = df.plot(x='iteration', y='MSE', label=name, ax=ax, zorder=100-int(name), linewidth=2, markersize=10, marker='.')
    optval = df['iteration'].values[np.argmin(df['MSE'].values)]
    ax.axvline(optval + i%2/50, color=pax.get_lines()[-1].get_color(), linestyle='--')

ax.set_ylabel('MSE')
ax.set_xlabel('iteration')
ax.legend(title='# neighbours', loc=(0.7, 0.6))
ax.set_xticks(hyperp['iteration'].drop_duplicates().values)
ax.set_title('100 PCs')
ax.grid()
sns.despine(offset=10)

fdir = figdir
fname = f"EMT_MAGIC_100_pcs_hyper_parameter_neighbours_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

When increasing the number of PCs we see that the number of iterations decrease. 50 and 20 now share optimum at 1 iteration, 10, 5, and 4 neighbours are optimized at 2, 3and 4 iterations respectively and 3 neighbours have a long convergence and is optimized at 8 iterations.
#+RESULTS: plot-EMT_MAGIC-100-pca-hyper-parameter-neighbours
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_100_pcs_hyper_parameter_neighbours_figure.png]]
:end:
This falls in line with our intuition of what we would expect to observe.

** Compare dimensionality reduction on data, DEWAX to MAGIC

#+name: load-data
#+begin_src ipython
adata = sc.read(os.path.join(datadir, f"EMT_MAGIC_denoised_data_.h5ad"))
#+end_src

#+name: calculate-dewakss-singular-values
#+begin_src ipython
from sklearn.preprocessing import StandardScaler
StS = StandardScaler()

mode = 'distances'
# dpca.optimal_ = 23
opt_pca = 23 # dpca.optimal_
# [Forced iterations, decay, Run2best, n_pcs, n_neighbours]
names = {'optimal_pca': [None, 1, True, 100, 100], 'oversmoothed': [5, 1, False, 100, 100], 'robust_pca': [None, 1, True, opt_pca, 100], 'KNN50_50pcs': [None, 1, True, 50, 50], 'KNN10': [None, 1, True, 100, 10], 'high_decay': [None, 5, True, 100, 100], 'X_base_opt': [None, 1, True, 0, 100]}

Sx = np.linalg.svd(StS.fit_transform(adata.layers['norm']), compute_uv=False)
# sc.pp.neighbors(adata, n_neighbors=50, n_pcs=dpca.optimal_)

SV = []
SV.append(Sx)
MSE = []
MSE.append(np.var(adata.layers['norm']))
VAR = []
VAR.append(np.var(adata.layers['norm']))
sc.pp.pca(adata, n_comps=100)
for n, v in names.items():
    
    tmpadata = adata.copy()
    tmpadata.X = tmpadata.layers['norm'].copy()

    # if n == 'X_base_opt':
    #     sc.pp.neighbors(tmpadata, n_neighbors=v[4], n_pcs=[3])
    # else:
    sc.pp.neighbors(tmpadata, n_neighbors=v[4], n_pcs=v[3])

    dewaksser = dewakss.DEWAKSS(tmpadata, init_diag=0, set_diag=0, iterations=10, run2best=v[2], decay=v[1], mode=mode)
    dewaksser.fit(tmpadata, v[0])
    tmpadata = dewaksser.transform(tmpadata)
    tmpadata.layers['Ms'] = tmpadata.layers['Ms'].toarray() if sp.sparse.issparse(tmpadata.layers['Ms']) else tmpadata.layers['Ms']
    Sx = np.linalg.svd(StS.fit_transform(tmpadata.layers['Ms']), compute_uv=False)

    SV.append(Sx)

    l = list(dewaksser.prediction_.values())
    mse = min(list(map(list, zip(*l)))[0])
    MSE.append(mse)
    VAR.append(tmpadata.uns['variance'])


singular_vals = pd.DataFrame(SV, index=['X'] + list(names.keys())).T
MSE = pd.DataFrame(MSE, index=['X'] + list(names.keys())).T
VAR = pd.DataFrame(VAR, index=['X'] + list(names.keys())).T

MSEVAR = pd.concat([MSE, VAR]).T
MSEVAR.columns = ['MSE', 'VAR']

singular_vals.to_csv(os.path.join(datadir, 'DEWAKSS_EMT_MAGIC_normalized_singular_vals_V3.tsv.gz'), compression='gzip', sep='\t')
MSEVAR.to_csv(os.path.join(datadir, 'DEWAKSS_EMT_MAGIC_MSE_VAR_V3.tsv.gz'), compression='gzip', sep='\t')
#+end_src

#+name: load-magic-singular-values
#+begin_src ipython
singular_vals = pd.read_csv(os.path.join(datadir.replace('simplify-dewakss', 'master'), 'DEWAKSS_EMT_MAGIC_normalized_singular_vals_V3.tsv.gz'), sep='\t', index_col=0)

svset = ['X', 'optimal_pca', 'oversmoothed', 'robust_pca', 'X_base_opt']

singular_vals = singular_vals[svset]

magic_singular_values = pd.read_csv(os.path.join('..', 'data', 'temporary', 'MAGIC', 'MAGIC_EMT_normalized_singular_vals_V2.tsv.gz'), sep='\t', index_col=0)

msvset = ['MAGIC', 'MAGIC_t1', 'MAGIC_dewakss']

magic_singular_values = magic_singular_values[msvset]
#+end_src

#+name: calc-explained-variance-spectra
#+begin_src ipython
rank_thresholds = singular_vals.max() * singular_vals.shape[0] * np.finfo(singular_vals.values.dtype).eps
singular_vals[singular_vals <= rank_thresholds[2]] = np.nan

rank_thresholds = magic_singular_values.max() * magic_singular_values.shape[0] * np.finfo(magic_singular_values.values.dtype).eps
magic_singular_values[magic_singular_values <= rank_thresholds[2]] = np.nan

deltaS_dewax = singular_vals**2/ (singular_vals**2).sum()
deltaS_magic = magic_singular_values**2 / (magic_singular_values**2).sum()
#+end_src

#+name: plot-explained-variance-spectra
#+begin_src ipython :results output drawer replace
doasave = True
style_label = ('seaborn-poster','bmh')
with plt.style.context(style_label):

    fig = plt.figure(figsize=(16,8), constrained_layout=True)
    ax = fig.subplots(2, 2, sharex='col').flatten()

    colormap = mpl.cm.tab20.colors

    linelist = np.unique(deltaS_dewax.columns.tolist() + deltaS_magic.columns.tolist())

    linelist = {l: colormap[i] for i, l in enumerate(linelist)}

    nev_dewax = (deltaS_dewax.cumsum() < 0.99).sum()

    nev_magic = (deltaS_magic.cumsum() < 0.99).sum()
    # del nev_magic['raw']

    nev = pd.concat([nev_dewax, nev_magic])

    n = -1

    for i in deltaS_dewax:
        data = deltaS_dewax[i]
        ax[0].semilogy(data[:n].values, label=i, c=linelist[i])

    for i in deltaS_magic:
        if i == 'raw':
            continue

        data = deltaS_magic[i]
        ax[0].semilogy(data[:n].values, label=i, c=linelist[i])

    for i in deltaS_dewax:
        data = deltaS_dewax[i].cumsum()
        ax[2].plot(data[:n].values, label=i, c=linelist[i])

    for i in deltaS_magic:
        if i == 'raw':
            continue

        data = deltaS_magic[i].cumsum()
        ax[2].plot(data[:n].values, label=i, c=linelist[i])


    n = 100
    for i in deltaS_dewax:
        data = deltaS_dewax[i]
        ax[1].semilogy(data[:n].values, label=i, c=linelist[i])

    for i in deltaS_magic:
        if i == 'MAGIC_d1':
            continue
        if i == 'raw':
            continue
        data = deltaS_magic[i]
        ax[1].semilogy(data[:n].values, label=i, c=linelist[i])

    for i in deltaS_dewax:
        if i == 'raw':
            continue
        data = deltaS_dewax[i].cumsum()
        ax[3].plot(data[:n].values, label=f"{i}, {nev[i]}", c=linelist[i])

    for i in deltaS_magic:
        if i == 'raw':
            continue
        data = deltaS_magic[i].cumsum()
        ax[3].plot(data[:n].values, label=f"{i}, {nev[i]}", c=linelist[i])

    ax[0].set_ylabel('$\eta^2$', fontsize=24)
    ax[2].set_xlabel('$\Sigma$', fontsize=24)

    ax[1].set_ylabel('$\eta^2$', fontsize=24)
    ax[3].set_xlabel('$\Sigma$', fontsize=24)

    # ax[2].legend(title='algorithm, n $\Sigma$s | cumsum($\eta^2$) < 0.99')
    L = ax[3].legend(title='algorithm, n $\Sigma$s | cumsum($\eta^2$) < 0.99', loc='center left', bbox_to_anchor=(1, 0.5), fontsize=20)
    L.get_title().set_fontsize(20)

    ax[2].set_ylabel('cumsum($\eta^2$)', fontsize=24)
    ax[3].set_ylabel('cumsum($\eta^2$)', fontsize=24)

    # for x in ax:
    #     x.grid()

if doasave:
    fdir = figdir
    fname = f"EMT_MAGIC_DEWAX_vs_MAGIC_explained_variance_"
    fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
    print_file = "[[file:" + fnames[0] + "]]"
    print(print_file, sep=",", end="")
    print("")

#+end_src

#+RESULTS: plot-explained-variance-spectra
:results:
[[file:../img/simplify-dewakss/EMT_MAGIC/EMT_MAGIC_DEWAX_vs_MAGIC_explained_variance_figure.png]]
:end:

#+name: plot-bar-components-with-variance
#+begin_src ipython :results output drawer replace
doasave = True
nev_dewax = (deltaS_dewax.cumsum() < 0.99).sum()
nev_magic = (deltaS_magic.cumsum() < 0.99).sum()
nev = pd.DataFrame(pd.concat([nev_dewax, nev_magic]), columns=['components'])
nev['EV'] = '0.99'
nev = nev.drop(['X'])

nc = deltaS_dewax.shape[0]

nev_dewax = (deltaS_dewax.cumsum() < 0.9).sum()
nev_magic = (deltaS_magic.cumsum() < 0.9).sum()
__ = pd.DataFrame(pd.concat([nev_dewax, nev_magic]), columns=['components'])
__['EV'] = '0.9'
__ = __.drop(['X'])

nev = pd.concat([nev, __]).reset_index()

nev['Denoise setting'] = nev['index'].str.replace('_', '\n')
nev['log(components)'] = np.log10(nev['components']+1)
nev[f'components of total ({nc})'] = nev['components']/nc

style_label = ('seaborn-poster')

with plt.style.context(style_label):

    g = sns.catplot(x="Denoise setting", y=f"components of total ({nc})", hue="EV", data=nev, height=4, aspect=2.5, kind="bar", palette="muted", legend_out=False, hue_order=['0.99', '0.9'])
    g.despine()
    ax = g.fig.gca()
    ax.grid(axis='y')

    rects = ax.patches

    # Make some labels.
    labels = ["%d" % nev['components'].values[i] for i in range(len(rects))]

    for rect, label in zip(rects, labels):
        height = rect.get_height()
        # print(height)
        # ax.text(rect.get_x() + rect.get_width() / 2, height + 5, label, ha='center', va='bottom')
        ax.text(rect.get_x() + rect.get_width() / 2.0, height, label, ha='center', va='bottom', fontsize=16)

if doasave:
    fdir = figdir
    fname = f"EMT_MAGIC_DEWAX_vs_MAGIC_EV_nComponents_"
    fnames = scpl.save_figure(g.fig, fdir, fname=fname, dpi=300)
    print_file = "[[file:" + fnames[0] + "]]"
    print(print_file, sep=",", end="")
    print("")
#+end_src

#+RESULTS: plot-bar-components-with-variance
:results:
[[file:../img/simplify-dewakss/EMT_MAGIC/EMT_MAGIC_DEWAX_vs_MAGIC_EV_nComponents_figure.png]]
:end:

#+name: plot-rank
#+begin_src ipython :results output drawer replace
doasave = True
style_label = ('seaborn-poster')
with plt.style.context(style_label):

    nc = deltaS_dewax.shape[0]
    dewax_rank = (~deltaS_dewax.isna()).sum()
    magic_rank = (~deltaS_magic.isna()).sum()
    ranks = pd.DataFrame(pd.concat([dewax_rank, magic_rank]), columns=['rank'])

    colormap = mpl.cm.tab20.colors
    linelist = np.unique(ranks.index)
    linelist = {l: colormap[i] for i, l in enumerate(linelist)}

    ranks['color'] = 1
    ranks['color'] = ranks['color'].astype(object)
    for k, v in linelist.items():

        ranks.at[k, 'color'] = v

    ranks = ranks.reset_index()
    ranks.columns = ['algorithm', 'rank', 'color']
    ranks['Denoise setting'] = ranks['algorithm'].str.replace('_', '\n')
    ranks['log(rank)'] = np.log10(ranks['rank'])

    g = sns.catplot(x='Denoise setting', y='log(rank)', data=ranks, height=4, aspect=2.5, kind="bar", palette=[sns.color_palette('muted')[0]], legend_out=False)
    # g = sns.barplot(x='Denoise setting', y='log(rank)', data=ranks)

    g.despine()
    ax = g.fig.gca()
    ax.grid(axis='y')

if doasave:
    fdir = figdir
    fname = f"EMT_MAGIC_DEWAX_vs_MAGIC_Rank_"
    fnames = scpl.save_figure(g.fig, fdir, fname=fname, dpi=300)
    print_file = "[[file:" + fnames[0] + "]]"
    print(print_file, sep=",", end="")
    print("")
#+end_src

#+RESULTS: plot-rank
:results:
[[file:../img/simplify-dewakss/EMT_MAGIC/EMT_MAGIC_DEWAX_vs_MAGIC_Rank_figure.png]]
:end:

#+name: calc-condition-number-spectra
#+begin_src ipython
deltaS_dewax = singular_vals.loc[0]/ singular_vals / singular_vals.shape[0]
deltaS_magic = magic_singular_values.loc[0] / magic_singular_values / magic_singular_values.shape[0]

N = 10
S = []
for i in range(0, N):
    Sx = np.linalg.svd(np.random.randn(deltaS_dewax.shape[0], deltaS_dewax.shape[0]), compute_uv=False)
    # K.append([np.linalg.cond(np.random.randn(n, n)) for n in sizes])
    S.append(Sx)

S = np.array(S)
SR = (S[:,0].T / S.T / deltaS_dewax.shape[0]).mean(1)
# plt.plot(S.mean(1))

# N = 100
# sizes = np.round(np.exp(np.arange(2,9,0.5))).astype(int)
# sizes[sizes > deltaS_dewax.shape[0]] = deltaS_dewax.shape[0]
# sizes = np.unique(sizes)
# k = np.zeros(sizes.shape)
# K = []
# for i in range(0, N):
#     K.append([np.linalg.cond(np.random.randn(n, n)) for n in sizes])
#+end_src

#+name: plot-condition-number-spectra
#+begin_src ipython :results output drawer replace
fig = plt.figure(figsize=(12,5), constrained_layout=True)
ax = fig.subplots(1, 2, sharex='col').flatten()

colormap = mpl.cm.tab20.colors

linelist = np.unique(deltaS_dewax.columns.tolist() + deltaS_magic.columns.tolist())

linelist = {l: colormap[i] for i, l in enumerate(linelist)}

nev_dewax = (deltaS_dewax < 1).sum()

nev_magic = (deltaS_magic < 1).sum()
del nev_magic['raw']

nev = pd.concat([nev_dewax, nev_magic])

n = -1

for i in deltaS_dewax:
    data = deltaS_dewax[i]
    # ax[0].semilogy(data[:n].values, label=i, c=linelist[i])
    ax[0].semilogy(data[:n].values, label=f"{i}, {nev[i]}", c=linelist[i])


for i in deltaS_magic:
    if i == 'raw':
        continue

    data = deltaS_magic[i]
    # ax[0].semilogy(data[:n].values, label=i, c=linelist[i])
    ax[0].semilogy(data[:n].values, label=f"{i}, {nev[i]}", c=linelist[i])

# ax[0].semilogy(SR, label='random, @size($\Sigma$)', c='r', linestyle='--', linewidth=0.5)
# ax[0].semilogy(sizes, K/sizes, label='random, @size(N)', c='k', linestyle='--', linewidth=0.5)

n = 100
for i in deltaS_dewax:
    data = deltaS_dewax[i]
    ax[1].semilogy(data[:n].values, label=i, c=linelist[i])

for i in deltaS_magic:
    if i == 'raw':
        continue
    data = deltaS_magic[i]
    ax[1].semilogy(data[:n].values, label=i, c=linelist[i])

ax[1].semilogy(SR[:n], c='r', linestyle='--', linewidth=0.5)
# ax[1].semilogy(sizes[sizes < n] , (K/sizes)[sizes < n], c='k', linestyle='--', linewidth=0.5)

ax[0].set_ylabel('$|\kappa_i|/N$')
ax[0].set_xlabel('$\Sigma$')

ax[1].set_ylabel('$|\kappa_i|/N$')
ax[1].set_xlabel('$\Sigma$')

ax[0].legend(title="algorithm, n $\kappa_i$'s | $|\kappa_i/N$ < 1", loc='lower right')

for x in ax:
    x.grid()

fdir = figdir
fname = f"EMT_MAGIC_DEWAX_vs_MAGIC_condition_number_"
fnames = scpl.save_figure(fig, fdir, fname=fname, dpi=150)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-condition-number-spectra
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_DEWAX_vs_MAGIC_condition_number_figure.png]]
:end:

#+name: plot-kappa-bar
#+begin_src ipython :results output drawer replace
nev_dewax = (deltaS_dewax < 1).sum()
nev_magic = (deltaS_magic < 1).sum()
del nev_magic['raw']

# nev = pd.concat([nev_dewax, nev_magic])

nev = pd.DataFrame(pd.concat([nev_dewax, nev_magic]), columns=['Condition number'])

colormap = mpl.cm.tab20.colors
linelist = np.unique(nev.index)
linelist = {l: colormap[i] for i, l in enumerate(linelist)}

nev['color'] = 1
nev['color'] = nev['color'].astype(object)
for k, v in linelist.items():
    
    nev.at[k, 'color'] = v

nev = nev.reset_index()
nev.columns = ['algorithm', 'Components', 'color']
nev['Denoise setting'] = nev['algorithm'].str.replace('_', '\n')
nev['log(Components)'] = np.log10(nev['Components'])

g = sns.catplot(x='Denoise setting', y='Components', data=nev, height=4, aspect=2.5, kind="bar", palette=[sns.color_palette('muted')[0]], legend_out=False)
# nev['color'].tolist()

g.despine()
ax = g.fig.gca()
ax.grid(axis='y')
ax.set_ylabel('Components($|\kappa_i|/N$)')

fdir = figdir
fname = f"EMT_MAGIC_DEWAX_vs_MAGIC_normalized_kappa_"
fnames = scpl.save_figure(g.fig, fdir, fname=fname, dpi=300)
print_file = "[[file:" + fnames[0] + "]]"
print(print_file, sep=",", end="")
print("")
#+end_src

#+RESULTS: plot-kappa-bar
:results:
[[file:../img/master/EMT_MAGIC/EMT_MAGIC_DEWAX_vs_MAGIC_normalized_kappa_figure.png]]
:end:

