"""
Contains functions for applying and testing single and dual node network detection. The two main functions are
- network_detection_algorithm_application(): to apply the NDA to experimental TMS data, and
- network_detection_algorithm_testing(): to generate artificial data and apply the NDA for testing reasons.
Details to all settings and options can be found in data/network mapping configuration/configuration guide.md.
"""
import os
import csv
import time
import h5py
import scipy
import lmfit
import trimesh
import numpy as np
import pandas as pd
from tqdm import tqdm
import multiprocessing
from sklearn import tree
import matplotlib.pyplot as plt
from numpy.random import default_rng
from scipy.signal import argrelextrema
from sklearn.neighbors._kde import KernelDensity
import sklearn.feature_selection as fs
import pynibs

VERBOSE = False


def determine_coil_position_idcs(num_idcs, all_efields, rn_seed):
    """
    Select coil positions from the set of all positions.

    Parameters
    ----------
    num_idcs : int
        How many coil positions to select (sample size).
    all_efields : ndarray of float
       (n_zaps_available, n_elms) Efields generated by all available coil positions.
    rn_seed : int
       Random seed used in selection.
    Returns
    -------
    selection : ndarray of int
        (n_zaps_used) Indices of chosen coil positions.
   """
    selection = np.array([],dtype=int)

    all_idcs = list(range(all_efields.shape[0]))

    rn_generator = default_rng(seed=rn_seed)
    selection = rn_generator.choice(all_idcs,size=num_idcs,replace=False)

    return selection


def determine_detectable_hotspots(e_subset, rn_seed):
    """
    From a given e-field matrix, two possible hotspots are picked, with the requirements that both e-fields (their
    maximum value) reach values higher than 1.0 and that they are not too close to each other (corr_coef < 0.5)
    Returns the indices of the hotspots to be.

    Parameters
    ----------
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    rn_seed : int
        Random_seed used for hotspot selection.
    Returns
    -------
    idx0 : int
        index of chosen hotspot element 0.
    idx1 : int
        index of chosen hotspot element 1.
    """
    finished = False
    e_max_thr = 1.0  # lower bound for maximum e-field value of a chosen hotspot (0: not limited) # hhh hardcoded values
    corr_thr = 0.5  # upper bound for correlation between the hotspots (1: not limited) # hhh
    np.random.seed(rn_seed)
    while not finished:
        idx0 = np.random.choice(e_subset.shape[1])
        idx1 = np.random.choice(e_subset.shape[1])

        if pynibs.compute_correlation_with_all_elements(e_subset, idx0)[idx1] < corr_thr:
            if np.max(e_subset[:, idx0]) > e_max_thr and np.max(e_subset[:, idx1]) > e_max_thr:
                finished = True

    return idx0, idx1


def create_response_data(efields, config):
    """
    Create response data mimicking a chosen network.
    The mean of the used distribution is computed based on the e-field values of the chosen network, the deviation
    depends on the chosen noise level.

    Parameters
    ----------
    efields : ndarray of float
        (n_zaps, n_elms) Efield generated by used coil positions.
    config: dict
        YAML-configuration file content as dictionary

    Returns
    -------
    data : ndarray of float
        (n_zaps) Artificially generated response data array.
    """
    data = np.zeros(efields.shape[0])

    rn_seed = config['rn_seed']
    network_type = config['network_type']
    distribution_type = config['distribution_type']
    jitter_ratio = config['jitter_ratio']
    jitter_scale = config['jitter_scale']
    effect_full = config['effect_full']
    effect_saturation = config['effect_saturation']
    hotspot_idcs = (config['hotspot_elm0'], config['hotspot_elm1'])
    efields_hot = efields[:, hotspot_idcs]
    rn_generator = default_rng(seed=rn_seed)
    np.random.seed(rn_seed)

    # choose class to pull distribution from
    if distribution_type == 'normal':
        dist = scipy.stats._continuous_distns.norm_gen
        gen_dist = dist(name='gen_dist')
        gen_dist.random_state = rn_generator
        # adjust response magnitude:
        response_shift = 1  # expected mean of N_0
        response_multiplier = 1  # e-mag * response_multiplier = expected effect (nominal values)
    elif distribution_type == 'logistic':
        dist = scipy.stats._continuous_distns.logistic_gen
        gen_dist = dist(name='gen_dist')
        gen_dist.random_state = rn_generator
        response_shift = 0
        response_multiplier = 1
    elif distribution_type == 'ExGaussian':  # not fully implemented! ! Parameter K needed next to loc and scale
        dist = scipy.stats._continuous_distns.exponnorm_gen
        gen_dist = dist(name='gen_dist')
        gen_dist.random_state = rn_generator
        response_shift = 400
        response_multiplier = 10

    else:
        raise NotImplementedError(f"Distribution type {distribution_type} not supported. "
                                  f"Please choose one of the following:"
                                  f" 'normal', 'logistic' ")

    # assign an effect indicator to each trial based on the hotspot e_mag values
    effect_indicator = np.zeros((efields.shape[0], 2))
    for efield_idx in range(0, efields.shape[0]):
        effect_indicator[efield_idx, 0] = min(efields_hot[efield_idx, 0] / effect_full, effect_saturation)
        effect_indicator[efield_idx, 1] = min(efields_hot[efield_idx, 1] / effect_full, effect_saturation)

    # radical noise: jitter_ratio determines the percentage of data that shows a random effect in [0, effect_saturation]
    rd_size = int(np.round(efields.shape[0] * jitter_ratio))
    rd_idcs = np.random.choice(effect_indicator.shape[0], size=rd_size)  # Get random indices
    effect_indicator[rd_idcs, 0] = np.random.uniform(0, effect_saturation, size=rd_size)
    effect_indicator[rd_idcs, 1] = np.random.uniform(0, effect_saturation, size=rd_size)

    # generate response data for every manifestation, taking the network type into consideration
    if distribution_type == 'ExGaussian':
        if network_type == 'NO':
            for efield_idx in range(0, efields.shape[0]):
                effect = 0

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'AND':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0] * effect_indicator[efield_idx, 1])**2  # \in [0, 1.44]

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=50)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == '1_INH_0':
            for efield_idx in range(0, efields.shape[0]):
                effect = max((effect_indicator[efield_idx, 0]**2 - effect_indicator[efield_idx, 1]**2), 0)

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'SH_0':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0])**4

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == '0_INH_1':
            for efield_idx in range(0, efields.shape[0]):
                effect = max((effect_indicator[efield_idx, 1]**2 - effect_indicator[efield_idx, 0]**2), 0)

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'SH_1':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 1]) ** 4

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'XOR':
            for efield_idx in range(0, efields.shape[0]):
                x = effect_indicator[efield_idx, 0]
                y = effect_indicator[efield_idx, 1]
                if x > y:
                    effect = x**2 - y**2
                else:
                    effect = y**2 - x**2

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'OR':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0]**4 + effect_indicator[efield_idx, 1]**4)/2

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/4) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale, K=100)
                data[efield_idx] = idx_prob_dist.rvs()

        else:
            print("Selected network type not supported!")
            raise NotImplementedError()

    else:  # other distributions (merge after solved distribution parameter problem ?)

        if network_type == 'NO':
            for efield_idx in range(0, efields.shape[0]):
                effect = 0

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'AND':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0] * effect_indicator[efield_idx, 1]) ** 2  # \in [0, 1.44]

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == '1_INH_0':
            for efield_idx in range(0, efields.shape[0]):
                # effect = max((effect_indicator[efield_idx, 0]**2 - effect_indicator[efield_idx, 1]**2), 0)
                effect = max((effect_indicator[efield_idx, 0] ** 2 - effect_indicator[efield_idx, 1] ** 2), 0)

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'SH_0':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0]) ** 4

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == '0_INH_1':
            for efield_idx in range(0, efields.shape[0]):
                effect = max((effect_indicator[efield_idx, 1] ** 2 - effect_indicator[efield_idx, 0] ** 2), 0)

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'SH_1':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 1]) ** 4

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'XOR':
            for efield_idx in range(0, efields.shape[0]):
                x = effect_indicator[efield_idx, 0]
                y = effect_indicator[efield_idx, 1]
                if x > y:
                    effect = x ** 2 - y ** 2
                else:
                    effect = y ** 2 - x ** 2

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        elif network_type == 'OR':
            for efield_idx in range(0, efields.shape[0]):
                effect = (effect_indicator[efield_idx, 0] ** 4 + effect_indicator[efield_idx, 1] ** 4) / 2

                loc = response_shift + response_multiplier * effect
                scale = (1+effect/2) * jitter_scale
                idx_prob_dist = gen_dist(loc=loc, scale=scale)
                data[efield_idx] = idx_prob_dist.rvs()

        else:
            print("Selected network type not supported!")
            raise NotImplementedError()

    return data


def binarize_response_data(response_values, method, bin_factor):
    """
    Binarizes the response data according to the selected method.

    Parameters
    ----------
    response_values : numpy.ndarray
        (n_zaps) Response data.
    method : str
        Method of calculating the binarization threshold. ("mean", "slope", "median")
    bin_factor : float
        Factor multiplied with threshold for splitting the data.
        (bin_factor * method(response)) is the threshold used for categorizing the data.

    Returns
    -------
    response_bin : numpy.ndarray
        (n_zaps) Binarized response data, where values greater than the threshold are set to 1, and values
        less than or equal to the threshold are set to 0.
    """
    # compute binarization threshold according to chosen method
    if method == 'mean':
        bin_thr = np.mean(response_values) * bin_factor
    elif method == 'median':
        bin_thr = np.median(response_values) * bin_factor
    elif method == 'slope':
        meps_sorted = np.sort(response_values)

        # sliding window averaging with window size N = 100
        # This will result in a shorter array as the boundary values are
        # discarded to avoid boundary effects (sliding window is reaching
        # 'over' the boundaries of the array)
        meps_sorted_smoothed = np.convolve(meps_sorted, np.ones((100,)) / 100, mode='valid')
        num_discarded_values = meps_sorted.shape[0] - meps_sorted_smoothed.shape[0]

        meps_sorted_smoothed_gradient = np.gradient(meps_sorted_smoothed)

        # ignore the first and last 5 % of the values when searching for the maximum
        #   (there is commonly a maximum towards the end)
        search_area = (
            int(meps_sorted_smoothed.shape[0] * 0.1),
            int(meps_sorted_smoothed.shape[0] - meps_sorted_smoothed.shape[0] * 0.1)
        )

        max_slope_idx = np.argmax(meps_sorted_smoothed_gradient[search_area[0]:search_area[1]])
        max_slope_idx += int(num_discarded_values / 2) + search_area[0]

        bin_thr = meps_sorted[max_slope_idx] * bin_factor

    # binarize every data point
    response_bin = np.zeros(response_values.shape)
    for i in range(0, response_values.shape[0]):
        if response_values[i] > bin_thr:
            response_bin[i] = 1
        else:
            response_bin[i] = 0

    return response_bin


def binarize_real_meps(meps, method='kde'):
    """
    Binarizes real MEP measurements.

    Parameters
    ----------
    meps : numpy.ndarray
        (n_zaps) The MEP measurements.
    method : str
        The method for determining the binarization threshold. Valid options are:
        - 'kde': Determine the threshold based on the density of the measurement points.
                 The point density is low when the value of the MEPs is rapidly changing.
        - 'slope': Determine the threshold based on the derivative/slope.

    Returns
    -------
    numpy.ndarray
        An array of the same shape as `meps` with 1 for data points below the determined threshold,
        and 0 otherwise.
    """
    split_at = 0

    if method == 'kde':

        # estimate optimal bandwidth
        bandwidth = None
        q25, q75 = np.percentile(meps, [25, 75])
        bandwidth = .9 * min(np.std(meps), q75 - q25) * pow(meps.shape[0], -0.2)

        kde = KernelDensity(
            kernel='gaussian',
            bandwidth=bandwidth
        ).fit(np.reshape(meps, (-1, 1)))

        x_grid = np.linspace(0, max(meps))
        values_on_grid = kde.score_samples(x_grid.reshape(-1, 1))

        # use the first occurrence (smallest/minimum index) of a minimum as threshold
        split_at = min(x_grid[argrelextrema(values_on_grid, np.less)[0]])

    elif method == 'slope':
        meps_sorted = np.sort(meps)

        # sliding window averaging with window size N = 100
        # This will result in a shorter array as the boundary values are
        # discarded to avoid boundary effects (sliding window is reaching
        # 'over' the boundaries of the array)
        meps_sorted_smoothed = np.convolve(meps_sorted, np.ones((100,)) / 100, mode='valid')
        num_discarded_values = meps_sorted.shape[0] - meps_sorted_smoothed.shape[0]

        meps_sorted_smoothed_gradient = np.gradient(meps_sorted_smoothed)

        # ignore the first and last 5 % of the values when searching for the maximum
        #   (there is commonly a maximum towards the end)
        search_area = (
            int(meps_sorted_smoothed.shape[0] * 0.1),
            int(meps_sorted_smoothed.shape[0] - meps_sorted_smoothed.shape[0] * 0.1)
        )

        max_slope_idx = np.argmax(meps_sorted_smoothed_gradient[search_area[0]:search_area[1]])
        max_slope_idx += int(num_discarded_values / 2) + search_area[0]

        split_at = meps_sorted[max_slope_idx]

        # debug; remove later
        plt.figure()
        plt.scatter(x=range(0, meps_sorted.shape[0]), y=meps_sorted)
        plt.scatter(
            x=range(meps_sorted_smoothed.shape[0] + int(num_discarded_values / 2)),
            y=np.pad(meps_sorted_smoothed, (int(num_discarded_values / 2), 0))
        )
        plt.figure()
        plt.scatter(x=range(0, meps_sorted_smoothed_gradient.shape[0]), y=meps_sorted_smoothed_gradient)
        plt.show()

    else:
        print("Invalid 'method' selected.")

    print(f"Split at: {split_at}")

    # to be consistent with the function "create_binary_MEP", we use
    #   - 0 for values below the threshold (no response)
    #   - 1 for values above the threshold (response)
    return np.where(meps > split_at, 1, 0)


def plot_data_std(response_data, e_field0, e_field1):
    """
    Plot the response data within axes representing the efields of the respective hotspots.

    Parameters
    ----------
    response_data : np.ndarray
        (n_zaps) The response data values.
    e_field0 : np.ndarray
        The efields of hotspot 0.
    e_field1 : np.ndarray
        The efields of hotspot 1.

    Returns
    -------
    plt.pyplot
        The matplotlib.pyplot object containing the plot.
    """
    plt.close('all')
    x = e_field0
    y = e_field1
    z = response_data

    # create plot_std
    plt.scatter(x, y, c=z, cmap='coolwarm', linewidth=0.5, edgecolor='grey')
    plt.xlabel(f'E-field $h_0$', fontsize=12, labelpad=4)
    plt.ylabel(f'E-field $h_1$', fontsize=12, labelpad=4)
    plt.gca().spines['right'].set_visible(False)
    plt.gca().spines['top'].set_visible(False)
    plt.gca().spines['left'].set_visible(True)
    plt.gca().spines['bottom'].set_visible(True)
    plt.colorbar()
    #plt.title(f'{response_data.shape[0]} data points with color indicating response value')
    corr = scipy.stats.pearsonr(x, y)
    plt.title(f'{response_data.shape[0]} data points -  '
              f'efield correlation: {np.round(corr[0], 2)}')
    return plt


def plot_data_bin(response_data_bin, e_field0, e_field1):
    """
    Plot the binarized response data within axes representing the efields of the respective hotspots.

    Parameters
    ----------
    response_data_bin : np.ndarray
        (n_zaps) The binarized response data.
    e_field0 : np.ndarray
        The efields of hotspot 0.
    e_field1 : np.ndarray
        The efields of hotspot 1.

    Returns
    -------
    plt.pyplot
        The matplotlib.pyplot object containing the plot.
    """
    plt.close('all')
    x = e_field0
    y = e_field1
    z = response_data_bin

    # create plot_bin
    plt.scatter(x, y, c=z, cmap='RdYlBu_r', vmin=-0.25, vmax=1.25, linewidth=0.5, alpha=0.95, edgecolors='grey')
    plt.xlabel(f'E-field $h_0$', fontsize=12, labelpad=4)
    plt.ylabel(f'E-field $h_1$', fontsize=12, labelpad=4)
    plt.gca().spines['right'].set_visible(False)
    plt.gca().spines['top'].set_visible(False)
    plt.gca().spines['left'].set_visible(True)
    plt.gca().spines['bottom'].set_visible(True)
    #plt.colorbar(ticks=[0,1])
    #plt.title(f'{response_data_bin.shape[0]} data points with color indicating category of response value ')

    return plt


def plot_data_clf(response_data_bin, e_field0, e_field1):
    """
    Plot the binarized response data within axes representing the efields of the respective hotspots.
    The background of the scatterplot will be colored according to the trained classifier's predictions for the data.
    Also plots the structure of the decision tree used to classify the data.

    Parameters
    ----------
    response_data_bin : np.ndarray
        (n_zaps) The binarized response data.
    e_field0 : np.ndarray
        The efields of hotspot 0.
    e_field1 : np.ndarray
        The efields of hotspot 1.

    Returns
    -------
    Tuple[plt.pyplot, plt.pyplot]
        A tuple containing the matplotlib.pyplot objects for the decision tree and the for the scatterplot.
    """
    plt.close('all')
    x = e_field0
    y = e_field1
    z = response_data_bin

    # set up classifier
    num_coil_samples = response_data_bin.shape[0]
    min_samples_leaf = int(0.05 * num_coil_samples)
    print(f'min_samples_leaf for plotting: {min_samples_leaf}')

    clf = tree.DecisionTreeClassifier(max_depth=2, min_samples_leaf=min_samples_leaf)
    samples = np.vstack((e_field0, e_field1)).transpose()
    res = clf.fit(samples, response_data_bin)
    score = clf.score(samples, response_data_bin)

    print(f'accuracy: {score}')
    print(f'feature_importances_: {res.feature_importances_}')
    print(f'tree_.feature: {clf.tree_.feature}')

    # plot decision tree
    tree_plt = plt.figure()
    tree.plot_tree(clf)
    plt.close('all')

    # determine boundaries of the classification overlay
    grid_x_min, grid_x_max = np.min(e_field0), np.max(e_field0)
    grid_y_min, grid_y_max = np.min(e_field1), np.max(e_field1)
    # define a regular grid between the boundaries of the data
    grid_x_coords, grid_y_coords = \
        np.meshgrid(np.arange(grid_x_min, grid_x_max+0.005, .005),
                    np.arange(grid_y_min, grid_y_max+0.005, .005))
    # have the classifier determine the class of the grid points
    grid_points_predicted = clf.predict(
        np.c_[grid_x_coords.ravel(), grid_y_coords.ravel()]
    )
    # Put the result into a color plot
    grid_points_predicted = grid_points_predicted.reshape(grid_x_coords.shape)

    xmin, xmax = np.min(x), np.max(x)
    ymin, ymax = np.min(y), np.max(y)

    # create plot_clf
    plt.scatter(x, y, c=z, cmap='RdYlBu_r', vmin=-0.25, vmax=1.25, linewidth=0.5, alpha=0.95, edgecolors='grey')
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()
    plt.contourf(grid_x_coords, grid_y_coords, grid_points_predicted, cmap='RdYlBu_r', alpha=0.15)
    plt.xlim(xmin, xmax)
    plt.ylim(ymin, ymax)

    plt.xlabel(f'E-field $h_0$', fontsize=12, labelpad=4)
    plt.ylabel(f'E-field $h_0$', fontsize=12, labelpad=4)
    plt.gca().spines['right'].set_visible(False)
    plt.gca().spines['top'].set_visible(False)
    plt.gca().spines['left'].set_visible(True)
    plt.gca().spines['bottom'].set_visible(True)
    # plt.colorbar(ticks=[0,1])
    #plt.title(f'{response_data_bin.shape[0]} data points with color indicating category of response value')
    plt.title(f'Classification Accuracy: {np.round(score, 2)}')

    return tree_plt, plt


def write_effect_map_hdf5(datatype, e_matrix, roi_surf, detection_result, base_path, config):
    """
    Create effect maps for result validation. The idea is that given a detection result, especially the hotspot_idcs and
     network_type, a prediction can be made about where the response is supposed to be affected most.
    Comparing these predictions and real response measurements is a possibility for result validation.
    See output_documentation.md for more.

    Parameters
    ----------
    datatype : str
        Whether the response data is 'real' or 'artificial'.
    e_matrix : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of all available coil positions across all ROI elements.
    roi_surf : ROI obj
       ROI surface object.
    detection_result : np.ndarray
        (7) contains the result of the detection, consisting of
        (found_network_type, found_idcs, found_acc, found_distance, found_scores, network_type_certainty, shape_vector).
    base_path : str
        Path to the folder where results should end up.
    config : dict
        YAML configuration file content as a dictionary.
    """

    print("Effect values are being calculated.")

    effect_full = config['effect_full']
    effect_saturation = config['effect_saturation']

    # calculate effect for each element based on the hotspot e_mag values
    network_type = detection_result[0]
    found_idcs = detection_result[1]
    effect = np.zeros(e_matrix.shape[1])
    for elm in range(0, e_matrix.shape[1]):
        # 'selection' represents the specific sample of the available samples where element 'elm' is stimulated highest
        # (a loose approximation of what would happen if this element was stimulated)
        selection = np.argmax(e_matrix[:, elm])
        e_subset = e_matrix[selection]

        # can also go with different stimulation position from here, instead of the proxy
        effect_vals = [np.zeros(e_matrix.shape[1]), np.zeros(e_matrix.shape[1])]
        for idx in [0,1]:
            if not np.isnan(found_idcs[idx]):
                effect_vals[idx] = min(e_subset[found_idcs[idx]] / effect_full, effect_saturation)

        if network_type == 1:
            effect[elm] = 0
        elif network_type == 2:
            effect[elm] = (effect_vals[0] * effect_vals[1]) ** 2  # \in [0, 1.44]
        elif network_type == 3:
            effect[elm] = max((effect_vals[0] ** 2 - effect_vals[1] ** 2), 0)
        elif network_type == 4:
            effect[elm] = effect_vals[0] ** 4
        elif network_type == 5:
            effect[elm] = max((effect_vals[1] ** 2 - effect_vals[0] ** 2), 0)
        elif network_type == 6:
            effect[elm] = effect_vals[1] ** 4
        elif network_type == 7:
            if effect_vals[0] > effect_vals[1]:
                effect[elm] = effect_vals[0] ** 2 - effect_vals[1] ** 2
            else:
                effect[elm] = effect_vals[1] ** 2 - effect_vals[0] ** 2
        elif network_type == 8:
            effect[elm] = (effect_vals[0] ** 4 + effect_vals[1] ** 4)/2
        else:
            raise NotImplementedError()

    if datatype == 'artificial':
        # output naming
        jitter = (config['jitter_ratio'], config['jitter_scale'])
        rn_seed = config['rn_seed']
        flag = f"jitter_{jitter}_seed_{rn_seed}"
    else:
        flag = config['fn_flag']
    # output naming
    fn_out_roi_effectmap = f"{base_path}/effectmap_{flag}.hdf5"
    fn_out_roi_geo = f"{base_path}/effectmap_{flag}_geo.hdf5"

    # save data as hdf5 _geo file (mapped)
    print(" > Creating .hdf5 geo files (mapped brain and roi) ...")
    while True:
        try:
            pynibs.write_geo_hdf5_surf(out_fn=fn_out_roi_geo,
                                                points=roi_surf.node_coord_mid,
                                                con=roi_surf.node_number_list,
                                                replace=True,
                                                hdf5_path='/mesh')
            pynibs.write_data_hdf5_surf(data=effect,
                                                 data_names=['effect_map'],
                                                 data_hdf_fn_out=fn_out_roi_effectmap,
                                                 geo_hdf_fn=fn_out_roi_geo,
                                                 replace=True)
            break
        except:
            print('problem accessing effectmap hdf5')
            time.sleep(1)

    print(f'Saved in folder: {base_path} \n **** \n ')


def write_network_detection_data_hdf5(datatype, e_matrix, response_values, base_path, config):
    """
    Preprocessing experimental data for the network detection algorithm. Either artificial response data is generated or
    real response data used. In the YAML files in the bin_configuration folder, the parameters
    or real measurements are used. In either case, the data is binarized (classified into 1: affected, and 0: not
    affected).
    Different data plotting options are included.

    Parameters
    ----------
    datatype : str
        Whether response data needs to be generated ('artificial') or real experimental data is available ('real').
    e_matrix : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of all available coil positions across all ROI elements.
    response_values : np.ndarray
        Real response values if available, or 'None' in artificial case.
    base_path : str
        Path to the folder where results should end up.
    config : dict
        YAML configuration file content as a dictionary.

    Returns
    -------
    runtime_gen : float
       Running time of data generation.
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    response : np.ndarray
        (2, n_zaps) Response values and binarized values in the form [response_values, response_binarized].
    """
    start = time.time()

    if datatype == 'artificial':
        print('Artificial response data will now be generated and binarized.')
        num_coil_samples = config['sample_size']
        jitter = (config['jitter_ratio'], config['jitter_scale'])
        rn_seed = config['rn_seed']
        hotspot_idcs = (config['hotspot_elm0'], config['hotspot_elm1'])
        fn_out_data = f"{base_path}/data_jitter_{jitter}_seed_{rn_seed}.hdf5"  # nnn

        # (1.1) Pick e-fields as random samples from coil positions and orientations
        idcs = pynibs.determine_coil_position_idcs(num_idcs=num_coil_samples, all_efields=e_matrix, rn_seed=rn_seed)
        e_subset = e_matrix[idcs]

        # (1.2) generate  response data
        response_values = pynibs.create_response_data(efields=e_subset, config=config)

    if datatype == 'real':
        print('Real data will now be binarized.')
        flag = config['fn_flag']
        fn_out_data = f"{base_path}/data_{flag}.hdf5"  # nnn
        e_subset = e_matrix

    response_bin = pynibs.binarize_response_data(response_values, config['bin_method'],
                                                 bin_factor=config['bin_factor'])
    response = np.vstack((response_values, response_bin))
    stop = time.time()
    runtime_gen = np.round(stop - start, 2)

    # (3) save data hdf5
    if config['save_files']:
        os.makedirs(base_path, exist_ok=True)
        while True:
            try:
                with h5py.File(fn_out_data, 'w') as f:
                    f.create_dataset(
                        'response_data',
                        data=response
                    )
                    f.create_dataset(
                        'e_subset',
                        data=e_subset
                    )
                break
            except:
                print('problem accessing data hdf5')
                time.sleep(1)
        print(f'Saved data in {runtime_gen} s under:{fn_out_data} \n **** \n ')

    # (4) plot artificial response data on plain spanned by both hotspot efields
    if datatype == 'artificial':
        flag = f'jitter_{jitter}_seed_{rn_seed}_hotspots'
        if config['plot_std']:
            std_plt = pynibs.plot_data_std(response[0], e_subset[:, hotspot_idcs[0]], e_subset[:, hotspot_idcs[1]])
            fn_std_plt = os.path.join(base_path,
                                      f'plot_std_{flag}.png')  # nnn
            std_plt.savefig(fn_std_plt, dpi=600)
            std_plt.close()
        if config['plot_bin']:
            bin_plt = pynibs.plot_data_bin(response[1], e_subset[:, hotspot_idcs[0]], e_subset[:, hotspot_idcs[1]])
            fn_bin_plt = os.path.join(base_path,
                                      f'plot_bin_{flag}.png')  # nnn
            bin_plt.savefig(fn_bin_plt, dpi=600)
            bin_plt.close()
        if config['plot_curves']:
            plot_idx0 = hotspot_idcs[0]

            plt_curve = pynibs.plot_data_bin(np.zeros(response[1].shape), e_subset[:, plot_idx0], response[0])
            plt_curve.ylabel('response')
            plt_curve.savefig(f'{base_path}/plot_{plot_idx0}_curve_{flag}.png', dpi=600)
            plt_curve.close()

            plot_idx1 = hotspot_idcs[1]

            plt_curve1 = pynibs.plot_data_bin(np.zeros(response[1].shape), e_subset[:, plot_idx1], response[0])
            plt_curve1.ylabel('response')
            plt_curve1.xlabel('E-field $h_1')
            plt_curve1.savefig(f'{base_path}/plot_{plot_idx1}_curve_{flag}.png', dpi=600)
            plt_curve1.close()

    return e_subset, response


def determine_scoring_idcs(e_subset, scoring_emag_thr=0, scoring_interval=22, method='optimized', fn_geo=None,
                           required_idcs=None):
    """
    Computes which elements to consider in the score calculations.

    Parameters
    ----------
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    scoring_emag_thr : float, default: 0
        Threshold to control stimulation accessibility of the scored elements. Elements with a stimulation peak
         below this threshold are ignored.(high thr -> only gyral elements are scored).
         (method 'optimized' not possible with an active threshold in place.)
    scoring_interval : int, default: 22
        The resolution for selecting elements. Only every scoring_interval-th element is considered.
    method : str, default: 'optimized'
        Specify method for selecting scoring indices.

        ** Random elements across the ROI are picked ('random'),
        ** Optimal elements are chosen by maximizing the distances between them. ('optimized') (Only applicable without
        scoring emag threshold. Requires fn_geo.)

    fn_geo : str, optional
        Required to compute element distances if method='optimized'.
    required_idcs : np.ndarray of int, optional
        Any indices that should definitely be included in scoring process, regardless of subsampling. (e.g. hotspots)

    Returns
    -------
    np.ndarray
        Array of indices representing the elements to consider in the score calculations.
    """
    if scoring_interval == 1:
        return np.arange(e_subset.shape[1])

    else:
        num_subsample = int(e_subset.shape[1] / scoring_interval)

        if method == 'random':
            # get maximum e-value over all data manifestations per element to apply scoring e-field threshold.
            # Where max > thr: 1, where max < thr: 0
            idcs = np.where(np.max(e_subset[:, :], 0) > scoring_emag_thr, 1, 0)
            scoring_idcs = np.nonzero(idcs)
            scoring_idcs = scoring_idcs[0]

            # now apply scoring interval: select only every scoring_interval-th element
            scoring_idcs = scoring_idcs[scoring_idcs % scoring_interval == 0]

        if method == 'optimized':
            with h5py.File(fn_geo, 'r') as f:
                mesh = trimesh.Trimesh(
                    vertices=f['mesh/nodes/node_coord'][:],
                    faces=f['mesh/elm/triangle_number_list'][:]
                )

                pts, scoring_idcs = trimesh.sample.sample_surface_even(
                    mesh=mesh,
                    count=num_subsample)
        # add additional desired index
        if required_idcs is not None:
            scoring_idcs = np.append(scoring_idcs, required_idcs)
        scoring_idcs = np.sort(scoring_idcs)

        return scoring_idcs


def compute_scores_with_single_element_clf(element_idx, efields, data, weights=None, scoring_idcs=None):
    """
    Computes the classifier (DecisionTree) score of element 'element_idx' with all other elements within the ROI.

    Parameters
    ----------
    element_idx : int
        The index of the element whose score with all other elements should be computed.
    efields : np.ndarray
        (n_zaps, n_elms) The efields in the ROI of all investigated coil positions.
    data : np.ndarray
        (n_zaps) The response corresponding to each coil position.
    weights : np.ndarray, optional
        (n_zaps) Weights to weight the data points.
    scoring_idcs : np.ndarray, optional
        Indices of ROI elements to compute scores for. Decreasing the number of elements significantly enhances runtime.
        If None, scores will be computed for all ROI elements.

    Returns
    -------
    tuple
        A tuple with the index of the investigated element and an array containing the scores of this element with each
        other array element.
    """
    dim    = efields.shape[1]
    scores = np.zeros(dim)

    num_coil_samples = data.shape[0]
    min_samples_leaf = max(int(0.05*num_coil_samples), 1)

    clf = tree.DecisionTreeClassifier(max_depth=2, min_samples_leaf=min_samples_leaf)   # ccc
    # tried different parameter settings and adding max_leaf_nodes=3, didn't improve performance

    elmts_to_iterate_over = scoring_idcs if scoring_idcs is not None else range(dim)

    for i in elmts_to_iterate_over:
        if i > element_idx:
            stacked_efields = np.vstack((efields[:, element_idx], efields[:, i])).transpose()
            clf.fit(stacked_efields, data, sample_weight=weights)
            scores[i] = clf.score(stacked_efields, data)

    return element_idx, scores
    # return scores


def compute_scores_with_single_element_regression(element_idx, efields, data, weights=None, scoring_idcs=None):
    """
    Computes the multivariate regression R2-score of element 'element_idx' with all other elements within the ROI.

    Parameters
    ----------
    element_idx : int
        The index of the element whose score with all other elements should be computed.
    efields : np.ndarray
        (n_zaps, n_elms) The efields in the ROI of all investigated coil positions.
    data : np.ndarray
        (n_zaps) The response corresponding to each coil position.
    weights : np.ndarray, optional
        (n_zaps) Weights to weight the data points.
    scoring_idcs : np.ndarray, optional
        Indices of ROI elements to compute scores for. Decreasing the number of elements significantly enhances runtime.
        If None, scores will be computed for all ROI elements.

    Returns
    -------
    tuple
        A tuple with the index of the investigated element and an array containing the scores of this element with each
        other array element.
    """
    dim    = efields.shape[1]
    scores = np.zeros(dim)
    num_coil_samples = data.shape[0]
    min_samples_leaf = max(int(0.05*num_coil_samples), 1)
    if VERBOSE:
        print(f'min_samples_leaf for scoring: {min_samples_leaf}')

    elmts_to_iterate_over = scoring_idcs if scoring_idcs is not None else range(dim)

    for i in elmts_to_iterate_over:
        if i > element_idx:

            data_var = np.var(data)
            df = pd.DataFrame({
                'a': pd.Series(efields[:, element_idx]),
                'b': pd.Series(efields[:, i]),
                'target': pd.Series(data)
            })

            #def sigmoid_multi(x, y, x0=1, r=1, amp=3):
            #   return amp / (1 + np.exp(-r * (x+y - x0)))
            #model = lmfit.Model(sigmoid_multi, independent_vars=['x', 'y'])
            #fit = model.fit(df['target'], x=df['a'], y=df['b'])
            #scores[i] = 1 - np.var(fit.residual) / data_var

            def gaussian_multi(x,y,amp=1,x0=0,y0=0,sigma_x=1,sigma_y=1):
                return amp * np.exp(-0.5* (((x-x0) / sigma_x)**2 + ((y-y0)/sigma_y)**2))

            model = lmfit.Model(gaussian_multi, independent_vars=['x','y'])
            fit = model.fit(df['target'],x=df['a'], y=df['b'])

            scores[i] = 1 - np.var(fit.residual) / data_var

    return element_idx, scores


def compute_scores_with_all_elements(efields, data, weights=None, scoring_idcs=None, scoring_method='clf'):
    """
    Computes the classifier (DecisionTree) score of each combination of ROI elements
    (upper triangle matrix of the score matrix).

    Parameters
    ----------
    efields : np.ndarray of float
        (n_zaps, n_elms) The efields in the ROI of all investigated coil positions.
    data : np.ndarray
        (n_zaps) The response corresponding to each zap.
    weights : np.ndarray, optional
        (n_zaps) Weights to weight the data points.
    scoring_idcs : np.ndarray, optional
        Indices of ROI elements to compute scores for. Decreasing the number of elements significantly enhances runtime.
        If None, scores will be computed for all ROI elements.
    scoring_method : str, optional
        Select 'clf' for Decision Tree Classifier, 'regression' for multivariable regression method,
        'regress_data' for one-dimensional regression method.

    Returns
    -------
    np.ndarray of float
        (n_elms, n_elms) An upper triangle matrix containing the scores of each ROI element with all others.
    """
    dim    = efields.shape[1]
    scores = np.zeros((dim,dim))

    if scoring_method == 'clf':
        for j in range(dim):
            element_idx, scores_row = pynibs.compute_scores_with_single_element_clf(j,efields, data[1], weights, scoring_idcs)
            scores[element_idx] = scores_row
    elif scoring_method == 'regression':
        for j in range(dim):
            element_idx, scores_row = pynibs.compute_scores_with_single_element_regression(j,efields, data[0], weights, scoring_idcs)
            scores[element_idx] = scores_row

    return scores


def compute_scores_with_all_elements_MP(efields, data, weights=None, scoring_idcs=None, scoring_method='clf'):
    """
    Computes the classifier (DecisionTree) score of each combination of ROI elements
    (multi-core enabled, upper triangle matrix of the score matrix).

    Parameters
    ----------
    efields : np.ndarray of float
        (n_zaps, n_elms) The efields in the ROI of all investigated coil positions.
    data : np.ndarray of float
        (n_zaps) The response corresponding to each coil position.
    weights : np.ndarray, optional
        (n_zaps) Weights to weight the data points.
    scoring_idcs : np.ndarray, optional
        Indices of ROI elements to compute scores for. Decreasing the number of elements significantly enhances runtime.
        If None, scores will be computed for all ROI elements.
    scoring_method : str, optional
        Select 'clf' for Decision Tree Classifier, 'regression' for multivariable regression method,
        'regress_data' for one-dimensional regression method.


    Returns
    -------
    np.ndarray of float
        An upper triangle matrix containing the scores of each ROI element with all others.
    """
    dim    = efields.shape[1]
    scores = np.zeros((dim,dim))

    elmts_to_iterate_over = scoring_idcs if scoring_idcs is not None else range(dim)

    if scoring_method == 'clf':  # clf version: data[1] is used (binarized)
        num_processes=multiprocessing.cpu_count()
        with multiprocessing.Pool(processes=num_processes) as pool:
            # @TODO: monitor if chunksize=1 impairs performance; it was set to 1 to have a smoother tdqm-progress bar.
            mp_res = pool.starmap(
                pynibs.compute_scores_with_single_element_clf,
                tqdm(
                    [(j,efields,data[1],weights,scoring_idcs) for j in elmts_to_iterate_over],
                    total=len(elmts_to_iterate_over)
                ),
                chunksize=1
            )
            pool.close()
            pool.join()

    elif scoring_method == 'regression':  # regression version: data[0] is used (not yet binarized)
        num_processes = multiprocessing.cpu_count()
        with multiprocessing.Pool(processes=num_processes) as pool:
            # @TODO: monitor if chunksize=1 impairs performance; it was set to 1 to have a smoother tdqm-progress bar.
            mp_res = pool.starmap(
                pynibs.compute_scores_with_single_element_regression,
                tqdm(
                    [(j, efields, data[0], weights, scoring_idcs) for j in elmts_to_iterate_over],
                    total=len(elmts_to_iterate_over)
                ),
                chunksize=1
            )
            pool.close()
            pool.join()

    for res in mp_res:
        scores[res[0]] = res[1]
    np.argmax(scores, 1)
    #iterate_idcs = np.argpartition(scores, -10)[-10:]
    return scores


def hotspots_by_score_cumulative(scores):
    """
    Determine the hotspot elements within the ROI based on the cumulative scores.
    Mark the elements with the highest cumulative scores as hotspots.

    Parameters
    ----------
    scores : np.ndarray
        (n_elms, n_elms) Upper triangle matrix of the scores.

    Returns
    -------
    hotspots: np.ndarray
        Array with dimensions equal to the number of ROI elements, where elements are marked as hotspots.
    """
    # Choose number of ROI elements that should be marked as hotspots
    num_desired_elements = 100
    # sum the scores across each row/column
    cumulative_scores = np.sum(scores, axis=0)
    # find the top num_desired_elements cumulative scores
    hotspot_idcs = np.argpartition(-cumulative_scores, num_desired_elements)[:num_desired_elements]
    hotspots = np.zeros(scores.shape[0])
    hotspots[hotspot_idcs] = cumulative_scores[hotspot_idcs]

    return hotspots, hotspot_idcs


def hotspots_by_score_percentiles(scores, accumulated=False):
    """
    Determine the hotspot elements within the ROI based on the upper percentile of scores.
    Mark the top elements as hotspot elements.

    Parameters
    ----------
    scores : np.ndarray
        Upper triangle matrix of the scores.
    accumulated : bool, optional
        Whether to accumulate the number of interacting hotspot elements.
        If True, the number in the hotspot array at location i denotes the number of other hotspot elements
        this element is associated with. If False, hotspots are marked with a value of 1. Default is False.

    Returns
    -------
    hotspots: np.ndarray
        Array with dimensions equal to the number of ROI elements, where elements are marked as hotspots.
    """
    # choose number of score accuracies that should contribute to the computed percentile
    num_desired_elements  = 100
    # each score is associated with two elements - so if we want n-elements as hotspots,
    # we must threshold n/2 scores.                 # n x n matrix of scores
    fraction_from_all_elements = (num_desired_elements/(scores.shape[0]**2))*100
    # somehow num_desired_element doesn't have the desired effect, but threshold still works well
    percentile = np.percentile(scores,100-fraction_from_all_elements)

    hotspot_idcs = np.array(np.where(scores >= percentile))
    hotspots = np.zeros(scores.shape[0])

    if accumulated:
        np.add.at(hotspots, hotspot_idcs.flatten(), 1)
    else:
        hotspots[ hotspot_idcs.flatten() ] = 1

    return hotspots, hotspot_idcs


def write_hotspot_scoremap_hdf5(datatype, e_subset, data, roi_surf, fn_geo, base_path, config, required_idcs=None):
    """
    Calculate the hotspot scores for given TMS experimental data and save scoremap and hotspot scores in hdf5.
    The hotspot scores indicate which ROI elements are most likely to be involved in the effect on the response variable.
    How they are computed depends on the scoring_method specified in the configuration file.

    Parameters
    ----------
    datatype : str
        Whether the response data is 'real' or 'artificial'.
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    data : np.ndarray of float
        (2, n_zaps) Two arrays, data[0] contains the response corresponding to each coil position, data[1] contains its
        binarization (1: response affected, 0: response not affected).
    roi_surf : ROI obj
       ROI surface object
    fn_geo : str
        Path to the geo.hdf5-file.
    base_path : str
        Path to the folder where results should end up.
    config : dict
        YAML configuration file content as a dictionary.
    required_idcs : np.ndarray of int, optional
        Any indices that should definitely be included in scoring process, regardless of subsampling. (e.g. hotspots)

    Returns
    -------
    runtime_gen : float
       Running time of data generation.
   """
    print("Hotspot scores are being computed.")
    start = time.time()

    if datatype == 'artificial':
        jitter = (config['jitter_ratio'], config['jitter_scale'])
        rn_seed = config['rn_seed']
        hotspot_idcs = (config['hotspot_elm0'], config['hotspot_elm1'])
        flag = f'jitter_{jitter}_seed_{rn_seed}'
    elif datatype == 'real':
        flag = config['fn_flag']
    else:
        raise NotImplementedError()

    if config['save_files']:
        # output file naming
        fn_out_roi_geo = f"{base_path}/res_hotspots_{flag}_geo.hdf5"
        fn_out_roi_hotpot_data = f"{base_path}/res_hotspots_{flag}.hdf5"
        fn_out_roi_scoremap = f"{base_path}/res_scoremap_{flag}.hdf5"

    # load e-fields and response (already binarized)
    #fn_data = f'data_{flag}.hdf5'  # nnn
    #with h5py.File(f'{base_path}/{fn_data}', "r") as f:
    #    e_subset = np.array(f['e_subset'])
    #    data = np.array(f['response_data'])  # data[0] is response, data[1] is binarized response

    # select the relevant idcs
    scoring_emag_thr = config['scoring_emag_thr']
    scoring_interval = config['scoring_interval']
    scoring_idcs = pynibs.determine_scoring_idcs(e_subset, scoring_emag_thr, scoring_interval,
                                                 method='optimized', fn_geo=fn_geo, required_idcs=required_idcs)

    # calculate scores and potential hotspots
    # SINGLE NODE METHODS: regress_data and mutual information score
    if config['scoring_method'] == 'regress_data':  # only for single node case
        con = roi_surf.node_number_list
        score_map = np.zeros(e_subset.shape[1])
        score_map[scoring_idcs] = pynibs.regress_data(e_matrix=e_subset,
                                        mep=data[0],
                                        elm_idx_list=scoring_idcs,
                                        # fun='linear',
                                        n_cpu=160,
                                        con=con,
                                        n_refit=0,
                                        return_fits=False,
                                        score_type='R2',
                                        verbose=True,
                                        pool=None,
                                        refit_discontinuities=False,
                                        select_signed_data=False)
        scores = score_map
        hotspots = score_map # only because in dual node approaches, this variable is needed

    elif config['scoring_method'] == 'mi':  # only for single node case
        # Compute Mutual Information for each feature
        mi_scores = fs.mutual_info_regression(e_subset[:, scoring_idcs], data[0])
        # subsampling
        scores = np.zeros(e_subset.shape[1])
        scores[scoring_idcs] = mi_scores
        hotspots = scores # only because in dual node approaches, this variable is needed
        score_map = scores

    # DUAL NODE METHODS: classifier and regression based scoring
    else: # dual nodes expected
        scores = pynibs.compute_scores_with_all_elements_MP(e_subset, data, scoring_idcs=scoring_idcs,
                                                            scoring_method=config['scoring_method'])

        score_map = np.max(scores, axis=0)

        hotspots = pynibs.hotspots_by_score_percentiles(scores, accumulated=True)[0]

    if config['save_files']:
        # save data as hdf5 _geo file (mapped)
        print(" > Creating .hdf5 geo files (mapped brain and roi) ...")
        while True:
            try:
                pynibs.write_geo_hdf5_surf(out_fn=fn_out_roi_geo,
                                                    points=roi_surf.node_coord_mid,
                                                    con=roi_surf.node_number_list,
                                                    replace=True,
                                                    hdf5_path='/mesh')
                pynibs.write_data_hdf5_surf(data=hotspots,
                                                     data_names='res_hotspots',
                                                     data_hdf_fn_out=fn_out_roi_hotpot_data,
                                                     geo_hdf_fn=fn_out_roi_geo,
                                                     replace=True)
                pynibs.write_data_hdf5_surf(data=score_map,
                                                     data_names='res_scoremap',
                                                     data_hdf_fn_out=fn_out_roi_scoremap,
                                                     geo_hdf_fn=fn_out_roi_geo,
                                                     replace=True)
                break
            except:
                print('problem writing score hdf5')
                time.sleep(1)

    stop = time.time()
    runtime_scores = np.round(stop - start, 2)
    print(f'Calculated scores in in {runtime_scores} s and saved in folder: {base_path} \n **** \n ')
    return runtime_scores, scores, hotspots


def compute_correlation_with_all_elements(e_subset, elm_idx):
    """
    Calculate the Pearson correlation coefficient of elm_idx with every other element.

    Parameters
    ----------
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    elm_idx : int
        The index of the element to compute the correlation with.

    Returns
    -------
    corr_coeff : np.ndarray
        (n_elms) Array containing the absolute Pearson correlation coefficients of elm_idx with each other element.
    """

    corr_coeff = np.zeros(e_subset.shape[1])
    x = e_subset[:, elm_idx]
    for idx in range(0, e_subset.shape[1]):
        y = e_subset[:, idx]
        if y.shape[0] > 1:
            pearson_res = scipy.stats.pearsonr(x, y)
            corr_coeff[idx] = abs(pearson_res[0])
        else:
            corr_coeff[idx] = (x==y)
    return corr_coeff


def find_distinct_hotspots(scorematrix, hotspot_mask, e_subset, acc_thr, corr_thr):
    """
    Returns the potential hotspots according to the scores on the ROI. The two elements with the highest scores whose
    e-fields are not too correlated to each other are chosen.
    If one score is very high, a single-hotspot case is assumed.

    Parameters
    ----------
    scorematrix : np.ndarray
        (n_elms, n_elms) Upper triangle score matrix containing the scores of each ROI element with all others.
    hotspot_mask : np.ndarray
        (n_elms) Hotspot score of every element, derived from the scorematrix.
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    acc_thr : float
        Threshold for accuracy criterion.
    corr_thr : float
        Threshold for correlation criterion.

    Returns
    -------
    hotspot_idcs : list
        List containing the indices of the potential hotspots.
    hotspot_scores : list
        List containing the scores of the potential hotspots.
    hotspot_acc : list
        List containing the accuracy values of the potential hotspots.
    found_bool : list
        List indicating whether each potential hotspot was found.
    """
    hotspot_idcs = [np.nan, np.nan]
    hotspot_scores = [np.nan, np.nan]
    hotspot_acc = [np.nan, np.nan]
    found_bool = [False, False]

    # first hotspot: take the max score, if multiple maximums the one with the max accuracy
    
    # Find the indices of the maximum values in hotspot_mask
    max_indices = np.argwhere(hotspot_mask == np.max(hotspot_mask))
    # Create a list of (max_indices, accuracy) pairs
    hotspot_info = [(idx, np.nanmax(scorematrix[:,idx])) for idx in max_indices]
    # Find the entry with the highest accuracy
    max_entry = max(hotspot_info, key=lambda x: x[1])
    # Extract the max_index
    hotspot_idcs[0] = int(max_entry[0])
    hotspot_acc[0] = max_entry[1]
    hotspot_scores[0] = hotspot_mask[hotspot_idcs[0]]
    # check whether hotspot 0 meets hotspot criteria: accuracy threshold and score threshold.
    # The score threshold is hardcoded because it is mostly to keep very strong single hotspot cases from running too
    # long: they often have one hotspot with a very high score and every other element has score 1.
    found_bool[0] = bool((hotspot_scores[0]>=2 and hotspot_acc[0]>=acc_thr))

    # second hotspot: next maximum score checking hotspot and correlation criteria
    if found_bool[0]:
        hotspot_mask[hotspot_idcs[0]] = 0
        correlations = pynibs.compute_correlation_with_all_elements(e_subset, hotspot_idcs[0])

        while np.max(hotspot_mask) > 1 and not found_bool[1]:
            # Find the indices of the maximum values in hotspot_mask
            max_indices = np.argwhere(hotspot_mask == np.max(hotspot_mask))
            # Create a list of (max_indices, accuracy) pairs
            hotspot_info = [(idx, np.max(scorematrix[idx])) for idx in max_indices]
            # Find the entry with the highest accuracy
            max_entry = max(hotspot_info, key=lambda x: x[1])
            # Extract the max_index, this is the second hotspot candidate
            hot_candi = int(max_entry[0])
            hot_candi_acc = max_entry[1]
            hot_candi_scores = hotspot_mask[hot_candi]

            # check whether hotspot candidate meets hotspot criteria, if not: ignore this one and keep looking
            if correlations[hot_candi] < corr_thr and bool((hot_candi_scores>=2 and hot_candi_acc>=acc_thr)):
                hotspot_idcs[1] = hot_candi
                hotspot_scores[1] = hot_candi_scores
                hotspot_acc[1] = hot_candi_acc
                found_bool[1] = True
            else:
                hotspot_mask[hot_candi] = 0 # ignore this one

    return hotspot_idcs, hotspot_scores, hotspot_acc, found_bool


def find_distinct_single_hotspot(hotspot_mask, acc_thr):
    """
    Returns the potential hotspot according to the scores on the ROI. The element with the highest score is chosen.
    If one score is very high, a single-hotspot case is assumed.

    Parameters
    ----------
    hotspot_mask : np.ndarray
        (n_elms) Hotspot score of every element, derived from the scorematrix.
    acc_thr : float
        Threshold for accuracy criteria.

    Returns
    -------
    hotspot_idcs : list
        List containing the indices of the potential hotspots.
    hotspot_scores : list
        List containing the scores of the potential hotspots.
    hotspot_acc : list
        List containing the accuracy values of the potential hotspots.
    found_bool : list
        List indicating whether each potential hotspot was found.
    """
    hotspot_idcs = [np.nan, np.nan]
    hotspot_scores = [np.nan, np.nan]
    hotspot_acc = [np.nan, np.nan]
    found_bool = [False, False]

    # hotspot: take the max hotspot score (highest R2)
    max_entry = np.argmax(hotspot_mask)
    hotspot_idcs[0] = int(max_entry)

    hotspot_acc[0] = hotspot_mask[hotspot_idcs[0]]
    hotspot_scores[0] = hotspot_mask[hotspot_idcs[0]]
    found_bool[0] = bool(hotspot_acc[0]>=acc_thr)

    return hotspot_idcs, hotspot_scores, hotspot_acc, found_bool


def calc_dist_pairwise(fn_geo, idx0, idx1):
    """
    Returns the geodesic distance from ROI element idx0 to idx1. In some rare cases at the border of the ROI, the
    geodesic distance cannot be measured or rather is infinite. In this case, the euclidean measure is used as an
    approximation.

    Parameters
    ----------
    fn_geo : str
        Path to the geo.hdf5-file.
    idx0 : int
        Index of the first ROI element.
    idx1 : int
        Index of the second ROI element.

    Returns
    -------
    distance : float
        Geodesic distance between the two ROI elements.
    """
    roi_geo_h5 = h5py.File(fn_geo, 'r')
    roi_tris = roi_geo_h5["mesh/elm/triangle_number_list"][:]
    roi_nodes = roi_geo_h5["mesh/nodes/node_coord"][:]

    distances = pynibs.geodesic_dist(
        tris=roi_tris,
        nodes=roi_nodes,
        source=idx0,
        source_is_node=False
    )[1]
    distance = distances[idx1]

    if distance == np.inf:
        print(f'****** Attention! ****** \n'
              f'Distance between {idx0} and {idx1} could not be measured geodesically, euclidean distance is used. \n'
              f'******************')
        distances = pynibs.euclidean_dist(
            tris=roi_tris,
            nodes=roi_nodes,
            source=idx0,
            source_is_node=False
        )[1]
        distance = distances[idx1]

    return distance


def assign_found_hotspot_single(fn_geo, hotspot_idcs, found_idx):
    """
    Assigns the found hotspot to the nearest real hotspot if there is only one hotspot found.

    Parameters
    ----------
    fn_geo : str
        Path to the geo.hdf5-file.
    hotspot_idcs : list
        A list of length 2 containing the indices of the real hotspots.
    found_idx : int
        The index of the found hotspot.

    Returns
    -------
    tuple
        A tuple containing the rounded distance between the real hotspot and the found hotspot,
        and the index indicating which real hotspot is assigned to the found hotspot.
    """
    print('Hotspot is assigned.')

    # a and b real hotspots, c found hotspot
    dist_ac = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[0], found_idx)
    dist_bc = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[1], found_idx)

    if dist_ac <= dist_bc:
        # then a gets assigned c
        return np.round(dist_ac, 2), 0
    else:
        # then b gets assigned c, so switch places
        return np.round(dist_bc, 2), 1


def assign_found_hotspots(fn_geo, hotspot_idcs, found_idcs):
    """
    Assigns the found hotspots to the real hotspots if there are two hotspots.

    Parameters
    ----------
    fn_geo : str
        Path to the geo.hdf5-file.
    hotspot_idcs : list
        A list of length 2 containing the indices of the real hotspots.
    found_idcs : list
        A list of length 2 containing the indices of the found hotspots.

    Returns
    -------
    tuple
        A tuple containing the rounded distances between the real hotspots and the found hotspots,
        and the updated list of found hotspot indices in the order they were assigned to the real hotspots.
    """
    print('Hotspots are assigned.')

    # a and b real hotspots, c and d found hotspots
    dist_ac = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[0], found_idcs[0])
    dist_bc = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[1], found_idcs[0])
    dist_ad = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[0], found_idcs[1])
    dist_bd = pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[1], found_idcs[1])

    if dist_ac + dist_bd <= dist_bc + dist_ad:
        # then a gets assigned c and b gets assigned d
        return np.round(dist_ac, 2), np.round(dist_bd, 2), found_idcs
    else:
        # then a gets assigned d and b gets assigned c, so switch places
        found_idcs_new = [found_idcs[1], found_idcs[0]]
        return np.round(dist_ad, 2), np.round(dist_bc, 2), found_idcs_new


def get_quadrant_samples(e_field0, e_field1):
    """
    Returns one point in each of the four representative stimulation states based on the provided electric field values.
    The stimulation states for two elements are all four possible combinations of the states "stimulated" and "not
    stimulated".

    Parameters
    ----------
    e_field0 : np.ndarray
        (n_zaps) The electric field values of element 0.
    e_field1 : np.ndarray
        (n_zaps) The electric field values of element 1.

    Returns
    -------
    np.ndarray
        A 2D np.ndarray containing the x and y coordinates of the chosen point in the quadrant.
    """
    # Quadrants:
    #       2 | 4
    #       ------
    #       1 | 3

    # First get portion of distances horizontally and vertically to shift the four values slightly into the middle
    # (offset). No offset would mean the reference points are at the corners of the rectangle spanned by the data.
    h = (max(e_field0) - min(e_field0))/24
    v = (max(e_field1) - min(e_field1))/24

    # 1 (bottom left)
    x = np.array(min(e_field0)+h)
    y = np.array(min(e_field1)+v)
    # 2 (top left)
    x = np.append(x, min(e_field0)+h)
    y = np.append(y, max(e_field1)-v)
    # 3 (bottom right)
    x = np.append(x, max(e_field0)-h)
    y = np.append(y, min(e_field1)+v)
    # 4 (top right)
    x = np.append(x, max(e_field0)-h)
    y = np.append(y, max(e_field1)-v)

    return np.vstack((x, y)).transpose()


def identify_shape(idx, scorematrix, e_subset, response):
    """
    Network type vectors are computed to determine most probable network type (vectors called shape_0 and shape_1).
    The predictions of decision tree classifiers trained on elements with high scores are used here to compute the most
    probable network type the hotspot element idx_h is involved in. At most 500 element pairs are considered.

    Parameters
    ----------
    idx : int
        Index of the hotspot element in question.
    scorematrix : np.ndarray of float
        (n_elms, n_elms) Upper triangle score matrix containing one score for every combination of elements.
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    response : np.ndarray
        (2, n_zaps) Response data: first entry original value, second entry binarized value.

    Returns
    -------
    shape : np.ndarray of int
        Vector indicating shape of decision trees used in scoring the element in question (network type vector).
    """
    num_coil_samples = int(response[0].shape[0])
    min_samples_leaf = max(int(0.05*num_coil_samples), 1)
    if VERBOSE:
        print(f'min_samples_leaf for identify_shape: {min_samples_leaf}')

    # shape parameter has 9 entries, ignore shape[0], shape[1:8] represent the 8 network types
    shape = np.zeros(9)

    iterate_idcs = idx

    # get indices of those elements that 'gave' element 'idx' its hotspot score (meaning: with these elements a score
    # higher than a calculated threshold was achieved)
    # because those are the elements the network identification is based on
    hotspot_idcs = pynibs.hotspots_by_score_percentiles(scorematrix, accumulated=False)[1]
    hotspot_pairs = hotspot_idcs.transpose()
    for i in np.arange(0, hotspot_pairs.shape[0], 1):
        if hotspot_pairs[i, 0] == idx:
            iterate_idcs = np.append(iterate_idcs, hotspot_pairs[i, 1])
        elif hotspot_pairs[i, 1] == idx:
            iterate_idcs = np.append(iterate_idcs, hotspot_pairs[i, 0])

    # if number of pairs is >500: only take the highest 500 scores as an indicator
    if i > 1000:
        iterate_idcs = np.argpartition(scorematrix[idx], -500)[-500:]
        print(f'Because of a exceptionally many elements element {idx} has a high accuracy with, only the 500 ones with'
              ' highest accuracy were used for network type computation.')
        shape[0]=-500

    elmts_to_iterate_over = iterate_idcs[iterate_idcs!=idx]

    clf = tree.DecisionTreeClassifier(max_depth=2, min_samples_leaf=min_samples_leaf)  # ccc
    # tried different parameter settings and adding max_leaf_nodes=3, didn't improve performance

    # calc classifier for every one of those elements
    for i in elmts_to_iterate_over:

        e_field0 = e_subset[:, idx]
        e_field1 = e_subset[:, i]
        if VERBOSE:
            print(f'index {idx} x {i}:')

        stacked_efields = np.vstack((e_field0, e_field1)).transpose()
        clf.fit(stacked_efields, response[1])

        # calculate prediction for every quadrant and translate into shape parameter
        # shape parameter has 9 entries, ignore shape[0], shape[1:8] represent the 8 network types
        q_samples = get_quadrant_samples(e_field0, e_field1)
        pred = clf.predict(q_samples)
        network_nr = 1 + pred[3] + 2*pred[2] + 4*pred[1]
        # shape[1] += pred[0] - worth a try
        # could use pred[0] for multiple things, like help distinguish shape 5 and 6
        shape[int(network_nr)] +=1

    if VERBOSE:
        print(shape)

    return shape


def identify_network_type(found_idcs, scorematrix, e_subset, response):
    """
    Identifies the most probable network type for the found indices using the function identify_shape().
    The network type vectors are called shape_0 and shape_1.

    Parameters
    ----------
    found_idcs : list
        A list containing the indices of the found hotspots.
    scorematrix : np.ndarray
        (n_elms, n_elms) Upper triangle score matrix containing the scores of each ROI element with all others.
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    response : np.ndarray
        (2, n_zaps) Response data: first entry original value, second entry binarized value.

    Returns
    -------
    found_network_type : int
        The identified network type.
    network_type_certainty : float
        The certainty level of the network type identification. (shape value of most probable network divided by
        value of the second most probable network)
    """
    # calculate network type vectors
    shape_0 = pynibs.identify_shape(found_idcs[0], scorematrix, e_subset, response)
    shape_1 = pynibs.identify_shape(found_idcs[1], scorematrix, e_subset, response)

    # axes are wrong for hotspot 1 (since it is on the y axis in the quadrant logic that the shape parameter is based
    # on), switch the asymmetric types: type 4<->6 and type 3<->5
    shape_1_h = shape_1[3]
    shape_1[3] = shape_1[5]
    shape_1[5] = shape_1_h
    shape_1_h = shape_1[4]
    shape_1[4] = shape_1[6]
    shape_1[6] = shape_1_h

    shape = np.add(shape_0, shape_1)
    shape_vector = shape
    print(f'Shape in sum: {shape}')

    # calc found_network type
    found_network_type = np.argmax(shape)

    # compute certainty level
    sec_shape = np.delete(shape, np.argmax(shape))
    network_type_certainty = 1 - (np.max(sec_shape) / np.max(shape))

    return found_network_type, shape_vector, network_type_certainty


def evaluate_network_identification(hotspot_dist, real_network_type, found_network_type):
    """
    Returns 1 if found_network_type is correct, 0 if not.

    Parameters
    ----------
    hotspot_dist : float
        Distance between hotspots.
    real_network_type : int
        Real network type.
    found_network_type : int
        Found network type.

    Returns
    -------
    eval : float
        Evaluation value indicating the accuracy of network identification.
    """
    # right type found: 100% accuracy
    eval = int(real_network_type == found_network_type)
    network_array = (real_network_type, found_network_type)

    # cases SH_0 and SH_1 are tolerated to be mixed up if the second supposed hotspot is close to the first one
    if (network_array == (4, 6) or network_array == (6, 4)) and hotspot_dist < 20:  # hhh hardcoded value here
        eval = 1

    return eval


def whole_network_detection(e_subset, response_data, scorematrix, hotspot_mask, base_path, config):
    """
    Based on the hotspot scores, 0-2 hotspot candidates are chosen. (Localization)
    If 0 are viable to be a hotspot, the result is a pseudonetwork. If only 1 viable hotspot was found, it is a single
    hotspot (type 4/6).
    If 2 hotspot candidates are found, the network type (1)-(8) is identified. (Identification)

    Parameters
    ----------
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    response_data : np.ndarray of float
        (2, n_zaps) Two arrays, data[0] contains the response corresponding to each coil position, data[1] contains its
        binarization (1: response affected, 0: response not affected).
    scorematrix : np.ndarray
        (n_elms, n_elms) Upper triangle score matrix containing the scores of each ROI element with all others.
    hotspot_mask : np.ndarray
        (n_elms) Hotspot score of every element, derived from the scorematrix.
    base_path : str
        The base path for the files.
    config : dict
        A dictionary containing the configuration parameters.

    Returns
    -------
    runtime_detection : float
       Running time of network detection.
    detection_result : np.ndarray
        (7) contains the result of the detection, consisting of
        (found_network_type, found_idcs, found_acc, found_distance, found_scores, network_type_certainty, shape_vector).
    """
    print("Hotspots are localized and network identified. \n **** \n")
    start = time.time()

    acc_thr = config['acc_thr']
    corr_thr = config['corr_thr']

    network_type_certainty = np.nan
    shape_vector = np.full(9, np.nan)
    found_distance = [np.nan, np.nan]
    if config['scoring_method'] == 'regress_data' or config['scoring_method'] == 'mi':   # single node approaches
        found_idcs, found_scores, found_acc, found_bool = \
            pynibs.find_distinct_single_hotspot(hotspot_mask, acc_thr)
    else:  # dual node approach
        found_idcs, found_scores, found_acc, found_bool = \
            pynibs.find_distinct_hotspots(scorematrix, hotspot_mask, e_subset, acc_thr, corr_thr)

    # 1 hotspot found
    if found_bool[0] and not found_bool[1]:
        print(f'A single-hotspot was detected.')
        found_network_type = 4
        print(f'Identified single hotspot: {found_idcs[0]}')

    # 2 hotspots found (only possible for dual hotspot approaches)
    elif found_bool[0] and found_bool[1]:
        print('Two potential hotspots were detected.')
        (found_network_type, shape_vector, network_type_certainty) = \
            pynibs.identify_network_type(found_idcs, scorematrix, e_subset, response_data)
        print(f'Identified network type: ({found_network_type}) for hotspots {found_idcs}')

    else:
        print('No hotspot was found.')
        found_network_type = 1

    stop = time.time()
    runtime_detection = np.round(stop - start, 2)

    # plot response data on plain spanned by both hotspot efields
    if found_bool[0] and found_bool[1]:
        if config['plot_std']:
            std_plt = pynibs.plot_data_std(response_data[0], e_subset[:, found_idcs[0]], e_subset[:, found_idcs[1]])
            fn_std_plt = os.path.join(base_path,
                                      f'plot_std_found_hotspots_{found_idcs[0]}_{found_idcs[1]}.png')  # nnn
            std_plt.savefig(fn_std_plt, dpi=600)
            std_plt.close()
        if config['plot_bin']:
            bin_plt = pynibs.plot_data_bin(response_data[1], e_subset[:, found_idcs[0]], e_subset[:, found_idcs[1]])
            fn_bin_plt = os.path.join(base_path,
                                      f'plot_bin_found_hotspots_{found_idcs[0]}_{found_idcs[1]}.png')  # nnn
            bin_plt.savefig(fn_bin_plt, dpi=600)
            bin_plt.close()
    if config['plot_curves']:
        if found_bool[0]:
            plot_idx0 = found_idcs[0]
            plt_curve = pynibs.plot_data_bin(np.zeros(response_data[1].shape), e_subset[:, plot_idx0], response_data[0])
            plt_curve.ylabel('response')
            plt_curve.savefig(f'{base_path}/plot_found_hotspot_{plot_idx0}_curve.png', dpi=600)
            plt_curve.close()
        if found_bool[1]:
            plot_idx1 = found_idcs[1]
            plt_curve1 = pynibs.plot_data_bin(np.zeros(response_data[1].shape), e_subset[:, plot_idx1], response_data[0])
            plt_curve1.ylabel('response')
            plt_curve1.xlabel('E-field $h_1')
            plt_curve1.savefig(f'{base_path}/plot_found_hotspot_{plot_idx1}_curve.png', dpi=600)
            plt_curve1.close()

    detection_result = found_network_type, found_idcs, found_acc, found_distance, found_scores, \
        network_type_certainty, shape_vector
    return runtime_detection, detection_result


def write_nda_test_results_csv(runtimes, e_subset, response_data, hotspot_mask, detection_result, fn_geo, config):
    """
    Evaluate network detection test results. Found networks are compared to the used settings for data generation.
    Then all used parameters and evaluation results are saved in a CSV file. See output_documentation.md for
    details about the output file.

    Parameters
    ----------
    runtimes : np.ndarray of float
        (3) The runtimes in s for (generation, scoring, detection).
    detection_result : np.ndarray
        (7) contains the result of the detection, consisting of
        (found_network_type, found_idcs, found_acc, found_distance, found_scores, network_type_certainty, shape_vector).
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements .
    response_data : np.ndarray of float
        (2, n_zaps) Two arrays, data[0] contains the response corresponding to each coil position, data[1] contains its
        binarization (1: response affected, 0: response not affected).
    hotspot_mask : np.ndarray
        (n_elms) Hotspot score of every element, derived from the scorematrix.
    detection_result : np.ndarray
        (7) contains the result of the detection, consisting of
        (found_network_type, found_idcs, found_acc, found_distance, found_scores, network_type_certainty, shape_vector).
    fn_geo : str
        Path to the geo.hdf5-file.
    config : dict
        A dictionary containing the configuration parameters.
    """
    print("Results are evaluated automatically.\n **** \n")

    # (1) read config params and results
    hotspot_idcs = (config['hotspot_elm0'], config['hotspot_elm1'])
    found_network_type, found_idcs, found_acc, found_distance, found_scores, \
        network_type_certainty, shape_vector  = detection_result

    # (2) hotspot assignment to enable network identification evaluation

    # if only one hotspot found: assign the nearest real hotspot
    if found_network_type == 4 or found_network_type == 6:
        assignment = assign_found_hotspot_single(fn_geo, hotspot_idcs, found_idcs[0])
        if assignment[1] == 1:  # switch hotspot order if necessary (in case of network type 6)
            found_network_type = 6
            found_idcs[1] = found_idcs[0]
            found_idcs[0] = np.nan
            found_distance[1] = assignment[0]
            found_scores[1] = found_scores[0]
            found_scores[0] = np.nan
            found_acc[1] = found_acc[0]
            found_acc[0] = np.nan
        else:  # else keep order (in case of network type 4)
            found_network_type = 4
            found_distance[0] = assignment[0]

    # 2 hotspots found: assign hotspots so that total distances are minimized for more precise evaluation
    elif found_network_type != 1:
        assignment = pynibs.assign_found_hotspots(fn_geo, hotspot_idcs, found_idcs)
        found_distance = [np.round(assignment[0], 2), np.round(assignment[1], 2)]
        # switch hotspot order if reassignment necessary
        if found_idcs[1] == assignment[2][0]:
            found_scores_h = found_scores[0]
            found_scores[0] = found_scores[1]
            found_scores[1] = found_scores_h
            found_acc_h = found_acc[0]
            found_acc[0] = found_acc[1]
            found_acc[1] = found_acc_h
            found_idcs = assignment[2]
            # adjust type if asymmetric type
            if found_network_type == 3:
                found_network_type = 5
            elif found_network_type == 5:
                found_network_type = 3

    print(f'For Evaluation: Identified network type: ({found_network_type}) for hotspots {found_idcs}')

    # (3) evaluation
    # collect additional info for result evaluation
    real_hotspot_dist = np.round(pynibs.calc_dist_pairwise(fn_geo, hotspot_idcs[0], hotspot_idcs[1]), 2)
    real_hotspot_corr = np.round(pynibs.compute_correlation_with_all_elements(e_subset, hotspot_idcs[0])[hotspot_idcs[1]], 2)
    hotspot_0_emax = np.round(np.max(e_subset[:, hotspot_idcs[0]]), 3)
    hotspot_1_emax = np.round(np.max(e_subset[:, hotspot_idcs[1]]), 3)
    num_hotspot_candidates = np.count_nonzero(hotspot_mask)
    found_accuracy = (np.round(found_acc[0], 3), np.round(found_acc[1], 3))
    if found_network_type in [4, 6, 1]:
        found_hotspots_dist, found_hotspots_corr = np.nan,np.nan
    else:  # compute found hotspot distance and correlation for dual hotspot types
        found_hotspots_dist = np.round(pynibs.calc_dist_pairwise(fn_geo, found_idcs[0], found_idcs[1]), 2)
        found_hotspots_corr = np.round(
            pynibs.compute_correlation_with_all_elements(e_subset, found_idcs[0])[found_idcs[1]], 2)

    # evaluate network type identification: 1 means the network was correctly identified, 0 otherwise
    network_types = ['NO', 'AND', '1_INH_0', 'SH_0', '0_INH_1', 'SH_1', 'XOR', 'OR']
    real_network_type = network_types.index(config['network_type'])+1
    identification_evaluation = pynibs.evaluate_network_identification(real_hotspot_dist, real_network_type, found_network_type)

    # evaluate hotspot localization: 1 means active hotspots were localized within 10mm, 0 otherwise
    if real_network_type == 1:
        localization_evaluation=identification_evaluation
    elif real_network_type == 4:
        localization_evaluation = int(found_distance[0] < 10)
    elif real_network_type == 6:
        localization_evaluation = int(found_distance[1] < 10)
    else:
        localization_evaluation = int((found_distance[0] < 10) and (found_distance[1]<10))

    # save information about the response
    response_max = np.round(np.max(response_data[0]), 3)
    response_mean = np.round(np.average(response_data[0]), 3)
    response_dev = np.round(np.std(response_data[0]), 3)

    # save all parameters from config file
    values = config.values()
    excel_values = np.array(1)  # first entry always 1, just cause
    for val in values:
        excel_values = np.append(excel_values, val)
    vals1 = excel_values[:7]
    vals2 = excel_values[7:-5]  # the last 5 values are not saved (fn_results and booleans)

    # save data in one row in {fn_results}, structure: configuration params, detection results, evaluation, additional info
    # configuration parameters
    output_csv = np.append(vals1, real_network_type)
    output_csv = np.append(output_csv, vals2)
    # detection results
    output_csv = np.append(output_csv,
                           [found_idcs[0], found_idcs[1],
                            np.round(found_scores[0], 2), np.round(found_scores[1], 2),
                            found_accuracy[0], found_accuracy[1],
                            found_hotspots_corr, found_hotspots_dist,
                           found_network_type, np.round(network_type_certainty, 2)])
    output_csv=np.append(output_csv, shape_vector)
    # evaluation results (only in case of artificial data / testing reasons)
    output_csv = np.append(output_csv,
                           [identification_evaluation, localization_evaluation,
                            real_hotspot_dist, real_hotspot_corr,
                            hotspot_0_emax, hotspot_1_emax,
                            found_distance[0], found_distance[1],
                            num_hotspot_candidates])
    # additional information and meta-data
    output_csv = np.append(output_csv,
                           [response_max, response_mean, response_dev,
                            runtimes[0], runtimes[1], runtimes[2]])

    # write csv artificial data
    while True:
        fn_results = config['fn_results']
        try:
            # open the evaluation csv file in the write mode
            with open(f'/data/pt_01756/studies/network_mapping/testing_NDA/15484.08/{fn_results}.csv', 'a',  # fff
                      newline='', encoding='UTF8') as f:
                # create the csv writer
                writer = csv.writer(f)
                # append output row
                writer.writerow(output_csv)
            break
        except:
            print('problem accessing eval csv')
            time.sleep(1)

    print(f'Saved results and evaluation in {fn_results}.csv \n **** \n ')


def write_nda_application_results_csv(runtimes, e_subset, response_data, hotspot_mask, detection_result, fn_geo, config):
    """
    Writes network detection results to a CSV file based on the provided parameters and configuration for real data.
    See output_documentation.md for more.

    Parameters
    ----------
    runtimes : np.ndarray of float
        (3), The runtimes in s for (generation, scoring, detection).
    e_subset : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of the used coil positions across all ROI elements.
    response_data : np.ndarray of float
        (2, n_zaps) Two arrays, data[0] contains the response corresponding to each coil position, data[1] contains its
        binarization (1: response affected, 0: response not affected).
    hotspot_mask : np.ndarray
        (n_elms) Hotspot score of every element, derived from the scorematrix.
    detection_result : np.ndarray
        (7) contains the result of the detection, consisting of
        (found_network_type, found_idcs, found_acc, found_distance, found_scores, network_type_certainty, shape_vector).
    fn_geo : str
        Path to the geo.hdf5-file.
    config : dict
        A dictionary containing the configuration parameters.
    """
    found_network_type, found_idcs, found_acc, found_distance, found_scores, \
        network_type_certainty, shape_vector = detection_result

    # collect additional info for result evaluation
    sample_size = e_subset.shape[0]
    found_accuracy = (np.round(found_acc[0], 3), np.round(found_acc[1], 3))
    # translate real network type id name into network name
    network_types = ['NO', 'AND', '1_INH_0', 'SH_0', '0_INH_1', 'SH_1', 'XOR', 'OR']
    found_network_name = network_types[found_network_type-1]
    if found_network_type in [4, 6, 1]:
        found_hotspots_dist, found_hotspots_corr = np.nan, np.nan
    else:  # compute found hotspot distance and correlation for dual hotspot types
        found_hotspots_dist = np.round(pynibs.calc_dist_pairwise(fn_geo, found_idcs[0], found_idcs[1]), 2)
        found_hotspots_corr = np.round(
            pynibs.compute_correlation_with_all_elements(e_subset, found_idcs[0])[found_idcs[1]], 2)

    # read relevant parameters from config file
    values = config.values()
    excel_values = np.array(1)  # hardcoded first entry in the result file
    for val in values:
        excel_values = np.append(excel_values, val)
    vals = excel_values[:-8]  # last entries not needed for evaluation

    # collect information about the response
    response_max = np.round(np.max(response_data[0]), 3)
    response_mean = np.round(np.average(response_data[0]), 3)
    response_dev = np.round(np.std(response_data[0]), 3)

    # save data in one row in {fn_results}, structure: config parameters, detection results, evaluation, additional info
    # configuration parameters
    output_csv = vals
    # detection results
    output_csv = np.append(output_csv,
                           [found_idcs[0], found_idcs[1],
                            np.round(found_scores[0], 2), np.round(found_scores[1], 2),
                            found_accuracy[0], found_accuracy[1],
                            found_hotspots_corr, found_hotspots_dist,
                            found_network_type, np.round(network_type_certainty, 2)])
    output_csv = np.append(output_csv, shape_vector)
    # additional information and meta-data
    output_csv = np.append(output_csv,
                           [sample_size, response_max, response_mean, response_dev,
                            runtimes[0], runtimes[1], runtimes[2]])

    # write csv real data
    while True:
        fn_results = config['fn_results']
        try:
            # open the evaluation csv file in write mode
            with open(f'/data/pt_01756/studies/network_mapping/evaluation_files_realdata/{fn_results}.csv', 'a', newline='',
                      encoding='UTF8') as f:
                # create the csv writer
                writer = csv.writer(f)
                # append output row
                writer.writerow(output_csv)
            break
        except:
            print('problem accessing result csv')
            time.sleep(1)
        if VERBOSE:
            print(f'Found hotspots: {found_idcs}. \n'
                  f'Saved results and evaluation in {fn_results} \n **** \n ')

    return found_network_name, found_idcs


def network_detection_algorithm_testing(e_matrix, roi_surf, fn_geo, base_path, config):
    """
    For testing reasons, experimental TMS data is generated and the network detection algorithm is applied to it.
    The e-field values are not generated but needed as input. The response values are computed based on the chosen
    network and the e-field values.
    A comparison between the algorithm output and the input settings is then saved in a .csv-file and can be used to
    assess the algorithm's performance.

    Parameters
    ----------
    e_matrix : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of all available coil positions across all ROI elements.
    roi_surf : ROI obj
       ROI surface object
    fn_geo : str
        Path to the geo.hdf5-file.
    base_path : str
        Path to the folder where results should end up.
    config : dict
        YAML configuration file content as a dictionary.
    """
    runtimes = np.zeros(shape=3)
    # apply network detection algorithm (steps 1-3) to artificially generated data and evaluate results:
    # (0) generate and (1) binarize response data
    e_subset, response = pynibs.write_network_detection_data_hdf5('artificial', e_matrix, None, base_path, config)
    # (2) scoring
    runtimes[1], scorematrix, hotspot_mask = pynibs.write_hotspot_scoremap_hdf5('artificial', e_subset, response, roi_surf, fn_geo, base_path, config)
    # (3) detection (localization and identification)
    runtimes[2], detection_result = pynibs.whole_network_detection(e_subset, response, scorematrix, hotspot_mask, base_path, config)
    # automatically evaluate results
    pynibs.write_nda_test_results_csv(runtimes, e_subset, response, hotspot_mask, detection_result, fn_geo, config)


def network_detection_algorithm_application(e_matrix, response_values, roi_surf, fn_geo, base_path, config):
    """
    The network detection algorithm is applied to given data. Based on the e-field values and response values provided,
    a dual node network is identified (type (1)-(8), for further explanation see config file)
    and active network nodes (hotspots) are localized (specified by the element IDs).
    The results are saved in a .csv-file specified in the config file.
    An "effect map" of the ROI can be created, showing where to expect the highest stimulation effects.

    Parameters
    ----------
    e_matrix : np.ndarray of float
        (n_zaps, n_elms) The efield magnitudes of all available coil positions across all ROI elements.
    response_values : np.ndarray
        (n_zaps) Response values (MEPs, reaction times, ...) corresponding to the coil configurations.
    roi_surf : ROI obj
       ROI surface object
    fn_geo : str
        Path to the geo.hdf5-file.
    base_path : str
        Path to the folder where results should end up.
    config : dict
        YAML configuration file content as a dictionary.
    """
    runtimes = np.zeros(shape=3)  # initiate array for saving the running times of the 3 algorithm steps

    # apply network detection algorithm
    # (1) binarization:
    e_subset, response  = pynibs.write_network_detection_data_hdf5('real', e_matrix, response_values, base_path, config)
    # (2) score calculation
    runtimes[1], scorematrix, hotspot_mask = pynibs.write_hotspot_scoremap_hdf5('real', e_subset, response, roi_surf, fn_geo, base_path, config)
    # (3) detection
    runtimes[2], detection_result = pynibs.whole_network_detection(e_subset, response, scorematrix, hotspot_mask, base_path, config)

    # save results in .csv (and write effect map if wanted)
    pynibs.write_nda_application_results_csv(runtimes, e_subset, response, hotspot_mask, detection_result, fn_geo, config)
    if config['write_effect_map']:
        pynibs.write_effect_map_hdf5('real', e_matrix, roi_surf, detection_result, base_path, config)

#
# ARCHIVE
#
'''
*******************************************
corr_coeff_map:
*******************************************

# just for testing reasons
def write_correlation_coeff_1elm_hdf5(elmidx, roi_surf, config, base_path):
    print("Correlation coefficients are being calculated.")
    jitter = (config['jitter_ratio'], config['jitter_scale'])
    rn_seed = config['rn_seed']
    hotspot_idcs = (config['hotspot_elm0'], config['hotspot_elm1'])
    num_coil_samples = config['sample_size']

    # load efields
    fn_data = f'data_jitter_{jitter}_hotspots_{hotspot_idcs}_seed_{rn_seed}.hdf5'  # nnn
    with h5py.File(f'{base_path}/{fn_data}', "r") as f:
        e_subset = np.array(f['e_subset'])

    # output naming
    fn_out_roi_scoremap = f"{base_path}/TEST_correlation_coeff_elm_{elmidx}_seed_{rn_seed}_data.hdf5"
    fn_out_roi_geo = f"{base_path}/TEST_correlation_coeff_elm_{elmidx}_seed_{rn_seed}_geo.hdf5"

    corr_coeff = pynibs.compute_correlation_with_all_elements(e_subset, elmidx)

    # save data as hdf5 _geo file (mapped)
    print(" > Creating .hdf5 geo files (mapped brain and roi) ...")
    pynibs.write_geo_hdf5_surf(out_fn=fn_out_roi_geo,
                               points=roi_surf.node_coord_mid,
                               con=roi_surf.node_number_list,
                               replace=True,
                               hdf5_path='/mesh')
    pynibs.write_data_hdf5_surf(data=corr_coeff,
                                data_names=['correlation_coeff'],
                                data_hdf_fn_out=fn_out_roi_scoremap,
                                geo_hdf_fn=fn_out_roi_geo,
                                replace=True)

    print(f'Saved in folder: {base_path} \n **** \n ')




*******************************************
all shape map functions 
*******************************************


#
# Computes the classifier (DecisionTree) score of each combination of ROI elements.
# (upper triangle matrix of the score matrix)
#
# @param efields     ... the efields in the ROI of all (investigated) coil positions
#                        shape: #coil pos x #roi elements
# @param data        ... the (binarized) MEPs elicited by each coil position
#                        (indices in 0-axis must correspond to the 0-axis of 'efields')
# @params weights    ... weight the data points using these weights
#                        (indices in 0-axis must correspond to the 0-axis of 'efields'
#                         and 'data')
# @return (np.ndarray) ... a upper triangle matrix containing the scores of each ROI
#                           element with all others
#
def compute_scores_and_shape_with_all_elements(efields, data, weights=None, masked_idcs=None):
    dim    = efields.shape[1]
    scores = np.zeros((dim,dim))
    shape = np.zeros((dim,dim))

    for j in range(dim):
        element_idx, scores_row, shape_row = pynibs.compute_scores_and_shape_with_single_element(j,efields, data, weights, masked_idcs)
        scores[element_idx] = scores_row
        shape[element_idx] = shape_row

    return scores, shape


#
# Computes the classifier (DecisionTree) score of each combination of ROI elements.
# (multi-core enabled, upper triangle matrix of the score matrix)
#
# @param efields     ... the efields in the ROI of all (investigated) coil positions
#                        shape: #coil pos x #roi elements
# @param data        ... the (binarized) MEPs elicited by each coil position
#                        (indices in 0-axis must correspond to the 0-axis of 'efields')
# @params weights    ... weight the data points using these weights
#                        (indices in 0-axis must correspond to the 0-axis of 'efields'
#                         and 'data')
# @return (np.ndarray) ... an upper triangle matrix containing the scores of each ROI element with all others
#
def compute_scores_and_shape_with_all_elements_MP(efields, data, weights=None, masked_idcs=None):
    dim    = efields.shape[1]
    scores = np.zeros((dim,dim))
    shape = np.zeros((dim,dim))

    elmts_to_iterate_over = masked_idcs if masked_idcs is not None else range(dim)

    num_processes=multiprocessing.cpu_count()
    with multiprocessing.Pool(processes=num_processes) as pool:
        # @TODO: monitor if chunksize=1 impairs performance; it was set to 1 to have a smoother tdqm-progress bar.
        mp_res = pool.starmap(
            pynibs.compute_scores_and_shape_with_single_element,
            tqdm(
                [(j,efields,data,weights,masked_idcs) for j in elmts_to_iterate_over],
                total=len(elmts_to_iterate_over)
            ),
            chunksize=1
        )
        pool.close()
        pool.join()

    for res in mp_res:
        scores[res[0]] = res[1]
        shape[res[0]] = res[2]

    return scores, shape



#
# Computes the classifier (DecisionTree) score of element 'element_idx' with
# all other elements within the ROI.
#
# @param element_idx ... the index of the element whose score with all other elements
#                        should be computed
# @param efields     ... the efields in the ROI of all (investigated) coil positions
#                        shape: #coil pos x #roi elements
# @param data        ... the (binarized) MEPs elicited by each coil position
#                        (indices in 0-axis must correspond to the 0-axis of 'efields')
# @params weights    ... weight the data points using these weights
#                        (indices in 0-axis must correspond to the 0-axis of 'efields'
#                         and 'data')
# @return (tuple)  ... a tuple with the idx of the investigated element + an array containing
#                      the scores of this element with each other array element
# slice(None) equivalent to the colon operator ':'
def compute_scores_and_shape_with_single_element(element_idx, efields, data, weights=None, masked_idcs=None):
    dim    = efields.shape[1]
    scores = np.zeros(dim)
    shape = np.zeros(dim)

    clf = tree.DecisionTreeClassifier(max_depth=2)

    elmts_to_iterate_over = masked_idcs if masked_idcs is not None else range(dim)

    for i in elmts_to_iterate_over:
        if i > element_idx:
            stacked_efields = np.vstack((efields[:, element_idx], efields[:, i])).transpose()
            clf.fit(stacked_efields, data, sample_weight=weights)
            scores[i] = clf.score(stacked_efields, data)
            if np.array_equal(clf.tree_.feature, [0, 0, -2, -2, 1, -2, -2]):
                shape[i] = 2
            else:
                shape[i] = 0
            #scores[i] = my_score(clf, stacked_efields, data)

    return (element_idx, scores, shape)
    # return scores



'''