# -*- coding: utf-8 -*-
"""
Created on 2018-07-01, Last Modified on 2020-02-10;
@author Yi.Q
"""
"""
Metrics to assess the performance on a classification model
Measures are from: https://en.wikipedia.org/wiki/Confusion_matrix;

———————————————————————————————————————————————————————————————————————————
         |                                       actual condition          |
         |—————————————————————————————————————————————————————————————————
         |                    |    actual positive    |  actual negative   |
         |—————————————————————————————————————————————————————————————————
Predicted| predicted positive |     true positive     |   false positive   |
condition|—————————————————————————————————————————————————————————————————
         | predicted negative |     false negative    |   true negative    |
—————————|—————————————————————————————————————————————————————————————————

Terms and calculations:
    thresholds: pre-set thresholds on which the follower metrics are calculated;
    TP: true positive, power, the number of obs. that is conditioned positive and predicted positive;
    FN: false negative, type II error, the number of obs. that is conditioned positive and predicted negative;
    FP: false positive, type I error, the number of obs. that is conditioned negative and predicted positive;
    TN: true negative, the number of obs. that is conditioned negative and predicted negative;

    Recall: TPR, probability of detecting true positive condition, = TP / (TP + FN);
    FNR: miss rate, = FN / (TP + FN);
    FPR: probability of false alarm, = FP / (FP + TN);
    TNR: tue negative rate, = TN / (FP + TN);

    Precision: Positive predictive value(PPV), = TP / (TP + FP);
    FDR: false discovery rate, FP / (TP + FP);
    FOR: false omission rate, FN / (FN + TN);
    NPV: negative predictive value, TN / (FN + TN);

    prevalence: = condition positive / total population;
    Accuracy: = (TP + TN) / (TP + FN + FP + TN);
    LR+: = TPR / FPR;
    LR-: = FNR / TNR;
    DOR: diagnostic odds ratio, = LR+ / LR-;
    F1: 2 / ((1/Recall) + (1/Precision));
"""
from typing import Union, Tuple, Optional

import numpy as np

from kolibri.classifier_evaluator.styles.format_confusion_matrix_by_prob_df import \
    convert_confusion_matrix_by_prob_to_table_with_reformat_precision

_FULL_METRICS = ['TP', 'FN', 'FP', 'TN',
                 'Recall', 'FNR', 'FPR', 'TNR', 'Precision', 'FOR', 'FDR', 'NPV',
                 'Prevalence', 'Accuracy', 'LR+', 'LR-', 'DOR', 'F1']

_FULL_METRICS_DUAL = ['Open_Count', 'Open_Positive', 'Open_Negative', 'Open_Prevalence']

_DEFAULT_POS_LABEL = True


def accuracy_rate(true: np.ndarray, predicted: np.ndarray) -> float:
    """
    calculate accuracy rate (ACC):
        ( true positive + true negative ) / sample size

    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted: numpy.ndarray(shape=(m), ), an array of predicted classes;
    :return: float, fraction, accuracy rate
    """
    sample_size = len(true)
    return (predicted == true).sum() / sample_size


def accuracy_rate_by_prob(true: np.ndarray, predicted_prob: np.ndarray,
                          threshold: float, pos_label: Union[str, bool, int] = _DEFAULT_POS_LABEL) -> float:
    """
    accuracy rate for binary classification according to a given threshold;

    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted_prob: numpy.ndarray(shape=(m), ),
        an array of predicted probabilities of being the positive class;
    :param threshold: float, [0, 1], the threshold set on predicted probabilities
        such that any predicted probability greater or equal to the threshold will be classified as the positive class,
        e.g., predicted_class(i | predicted_prob_i >= threshold) = positive;
    :param pos_label: [str, bool, int], positive class label, label that is considered as the positive class;
    :return: float, fraction, accuracy rate
    """
    # convert true series to positive - negative class
    true = (true == pos_label)
    predicted = (predicted_prob >= threshold)

    return accuracy_rate(true=true, predicted=predicted)


def _condition_counter(true: np.ndarray,
                       predicted: np.ndarray,
                       true_condition: Union[bool, int, str, list, tuple, set],
                       predicted_condition: Union[bool, int, str, list, tuple, set]) -> int:
    """
    count the number of instances such that their true value being true_condition and predicted being predicted_condition;
    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted: numpy.ndarray(shape=(m), ), an array of predicted classes;
    :param true_condition: [bool, int, str, list, tuple, set], the true condition(s) that is being count;
    :param predicted_condition: [bool, int, str, list, tuple, set], the predicted condition(s) that is being count;
    :return: total count of the instance with the joined condition, int;
    """
    if isinstance(true_condition, (str, bool, int)):
        true_condition = [true_condition]
    elif isinstance(true_condition, set):
        true_condition = list(true_condition)
    elif isinstance(true_condition, (list, tuple)):
        pass
    else:
        raise TypeError(f"Type of true conditions {type(true_condition)} is not acceptable.")

    if isinstance(predicted_condition, (str, bool, int)):
        predicted_condition = [predicted_condition]
    elif isinstance(predicted_condition, set):
        predicted_condition = list(predicted_condition)
    elif isinstance(predicted_condition, (list, tuple)):
        pass
    else:
        raise TypeError(f"Type of true conditions {type(predicted_condition)} is not acceptable.")

    return (np.isin(true, true_condition) & np.isin(predicted, predicted_condition)).sum().item()


def prevalence(series: np.ndarray,
               condition: Union[bool, int, str, list, tuple, set],
               rate: bool = False) -> Union[int, float]:
    """
    count the number of instances with (one of) the given condition(s);
    :param series: numpy.ndarray(shape=(m), ) containing different conditions;
    :param condition: [bool, int, str, list, tuple, set], a selected condition;
    :param rate: bool, return prevalence rate instead of count;
    :return: the count of the instances with the condition(s);
    """
    if isinstance(condition, (str, bool, int)):
        condition = [condition]
    elif isinstance(condition, set):
        condition = list(condition)
    elif isinstance(condition, (list, tuple)):
        pass
    else:
        raise TypeError(f"Type of true conditions {type(condition)} is not acceptable.")

    if rate:
        return np.isin(series, condition).sum() / series.size if series.size > 0 else 0
    else:
        return np.isin(series, condition).sum()


def recall_rate(true: np.ndarray,
                predicted: np.ndarray,
                condition: Union[bool, int, str, list, tuple, set]) -> float:
    """
    compute recall rate, refer module description for calculation;
    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted: numpy.ndarray(shape=(m), ), an array of predicted classes;
    :param condition: [bool, int, str, list, tuple, set], the true condition(s) that is being count;
    :return: recall rate, float;
    """
    joint_condition_count = _condition_counter(true=true,
                                               predicted=predicted,
                                               true_condition=condition,
                                               predicted_condition=condition)

    prevalence_count = prevalence(series=true,
                                  condition=condition,
                                  rate=False)

    recall = joint_condition_count / prevalence_count if prevalence_count > 0 else 0

    return recall


def recall_rate_by_prob(true: np.ndarray,
                        predicted_prob: np.ndarray,
                        threshold: float = 0.5,
                        pos_label: Union[bool, int, str] = _DEFAULT_POS_LABEL) -> float:
    """
    compute recall rate when predicted prob is given, refer module description for calculation;
    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted_prob: numpy.ndarray(shape=(m), ), an array of predicted probability of being positive classe;
    :param threshold: float, the threshold over which predicted probabilities will be counted as positive;
    :param pos_label: [bool, int, str], the true condition(s) that is being count;
    :return: recall rate, float;
    """

    true = true == pos_label
    predicted = predicted_prob >= threshold

    recall = recall_rate(true=true,
                         predicted=predicted,
                         condition=True)

    return recall


def precision_rate(true: np.ndarray,
                   predicted: np.ndarray,
                   condition: Union[bool, int, str, list, tuple, set]) -> float:
    """
    compute precision rate, refer module description for calculation;
    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted: numpy.ndarray(shape=(m), ), an array of predicted classes;
    :param condition: [bool, int, str, list, tuple, set], the true condition(s) that is being count;
    :return: precision rate, float;
    """
    joint_condition_count = _condition_counter(true=true,
                                               predicted=predicted,
                                               true_condition=condition,
                                               predicted_condition=condition)

    prevalence_count = prevalence(series=predicted,
                                  condition=condition,
                                  rate=False)

    precision = joint_condition_count / prevalence_count if prevalence_count > 0 else 0

    return precision


def precision_rate_by_prob(true: np.ndarray,
                           predicted_prob: np.ndarray,
                           threshold: float = 0.5,
                           pos_label: Union[bool, int, str] = _DEFAULT_POS_LABEL) -> float:
    """
    compute precision rate when predicted prob is given, refer module description for calculation;
    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted_prob: numpy.ndarray(shape=(m), ), an array of predicted probability of being positive classe;
    :param threshold: float, the threshold over which predicted probabilities will be counted as positive;
    :param pos_label: [bool, int, str], the true condition(s) that is being count;
    :return: precision rate, float;
    """

    true = true == pos_label
    predicted = predicted_prob >= threshold

    precision = precision_rate(true=true,
                               predicted=predicted,
                               condition=True)

    return precision


def confusion_matrix(true: np.ndarray, predicted: np.ndarray, normalize: Union[bool, str] = False) -> Union[
    tuple, dict]:
    """
    compute confusion matrix

        terms: pair condition: joined condition of both observed condition and predicted condition, e.g., (True, False);
               true condition: observed condition of an instance, e.g., True;
               predicted condition: predicted condition of an instance, e.g., True;
    :param true: true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted: numpy.ndarray(shape=(m), ), an array of predicted classes;
    :param normalize: [bool, 'true', 'predicted'], if the confusion matrix needs to be normalised:
        True: default, normalized along true classes; False: not normalized;
        'true': normalized along true classes; 'predicted': normalized along predicted classes;
        e.g., normalisation along true classes: normalized_condition (for i) = count(predicted = i, true = i) / (true = i)
    :return: dict, a confusion matrix:
        key: tuple, (true_class, predicted_class);
        value: [int, float], the number of instances of that pair condition (true_class, predicted_class), or
            normalisation of the number of the instances of that pair condition;
    """
    labels = [label for label in set(true).union(set(predicted))]  # .item convert numpy dtypes to native types;

    confusion_matrix_dict = dict()
    for true_label in labels:
        for predicted_label in labels:
            confusion_matrix_dict[(true_label, predicted_label)] = \
                _condition_counter(true=true, predicted=predicted,
                                   true_condition=true_label, predicted_condition=predicted_label)

    if normalize:
        normalize_factor = 0 if normalize != 'predicted' else 1
        normalized_confusion_matrix_dict = normalize_confusion_matrix(confusion_matrix_dict=confusion_matrix_dict,
                                                                      normalize_index=normalize_factor)
        return confusion_matrix_dict, normalized_confusion_matrix_dict
    else:
        return confusion_matrix_dict


def normalize_confusion_matrix(confusion_matrix_dict: dict, normalize_index: int = 0) -> dict:
    """
    normalize a confusion matrix;

    :param confusion_matrix_dict: dict, a confusion matrix, e.g.,
        {('a', 'a'): 1, ('a', 'b'): 2, ('b', 'a'): 3, ('b', 'b'): 4};
    :param normalize_index: int, [0, 1], the position index of the confusion matrix dict key (tuple) where normalization
    is performed along;
    :return: a normalized confusion matrix;
    """
    labels = set([key[0] for key in confusion_matrix_dict.keys()])
    normalized_confusion_matrix_dict = dict()

    sum_dict = {label: 0 for label in labels}
    for condition, condition_count in confusion_matrix_dict.items():
        sum_dict[condition[normalize_index]] += condition_count

    for condition, condition_count in confusion_matrix_dict.items():
        normalized_confusion_matrix_dict[condition] = \
            condition_count / sum_dict[condition[normalize_index]] if sum_dict[condition[normalize_index]] else 0

    return normalized_confusion_matrix_dict


def confusion_matrix_by_prob(true: np.ndarray,
                             predicted_prob: np.ndarray,
                             thresholds: Optional[Union[list, tuple, np.ndarray]] = None,
                             pos_label: Union[bool, str, int] = _DEFAULT_POS_LABEL,
                             output_metrics: Optional[list] = None,
                             table: bool = True,
                             **kwargs):
    """
    confusion matrix for binary classification according to a given set of thresholds;

    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted_prob: numpy.ndarray(shape=(m), ),
        an array of predicted probabilities of being the positive class;
    :param thresholds: [list, tuple, np.array, None] the thresholds set on predicted probabilities
        such that any predicted probability greater or equal to the threshold will be classified as the positive class;
    :param pos_label: [str, bool, int], positive class label, label that is considered as the positive class;
    :param output_metrics: [list, None], metrics to be outputted if selected;
    :param table: bool, if exported as a pd table table;
    :param kwargs:
        :param metric_order: [list, None], if table is selected to be the output, metric order specifies the order of
            metrics presented in the table;
    :return: dict, a set of confusion matrices, {threshold: {metric_name: metric_value, ...}, ...};
    """
    # convert true series to positive series
    true = true == pos_label

    # select output:
    if isinstance(output_metrics, list):
        for selected_metric in output_metrics:
            if selected_metric not in _FULL_METRICS:
                raise KeyError(f"metric {selected_metric} is not recognized.")
    elif output_metrics == 'confusion':
        output_metrics = ['TP', 'FN', 'FP', 'TN',
                          'Recall', 'FNR', 'FPR', 'TNR', 'Precision', 'FOR', 'FDR', 'NPV',
                          'Prevalence', 'Accuracy']
    else:
        output_metrics = _FULL_METRICS

    metrics_by_thresholds = dict()
    for threshold in thresholds:
        metrics_by_threshold = dict()
        predicted = predicted_prob >= threshold
        confusion_matrix_dict = confusion_matrix(true=true, predicted=predicted, normalize=False)

        confusion_matrix_nor_true = normalize_confusion_matrix(confusion_matrix_dict=confusion_matrix_dict,
                                                               normalize_index=0)

        confusion_matrix_nor_predicted = normalize_confusion_matrix(confusion_matrix_dict=confusion_matrix_dict,
                                                                    normalize_index=1)

        if 'TP' in output_metrics:
            metrics_by_threshold['TP'] = confusion_matrix_dict[(True, True)]

        if 'FN' in output_metrics:
            metrics_by_threshold['FN'] = confusion_matrix_dict[(True, False)]

        if 'FP' in output_metrics:
            metrics_by_threshold['FP'] = confusion_matrix_dict[(False, True)]

        if 'TN' in output_metrics:
            metrics_by_threshold['TN'] = confusion_matrix_dict[(False, False)]

        if 'Recall' in output_metrics:
            metrics_by_threshold['Recall'] = confusion_matrix_nor_true[(True, True)]

        if 'FNR' in output_metrics:
            metrics_by_threshold['FNR'] = confusion_matrix_nor_true[(True, False)]

        if 'FPR' in output_metrics:
            metrics_by_threshold['FPR'] = confusion_matrix_nor_true[(False, True)]

        if 'TNR' in output_metrics:
            metrics_by_threshold['TNR'] = confusion_matrix_nor_true[(False, False)]

        if 'Precision' in output_metrics:
            metrics_by_threshold['Precision'] = confusion_matrix_nor_predicted[(True, True)]

        if 'FOR' in output_metrics:
            metrics_by_threshold['FOR'] = confusion_matrix_nor_predicted[(True, False)]

        if 'FDR' in output_metrics:
            metrics_by_threshold['FDR'] = confusion_matrix_nor_predicted[(False, True)]

        if 'NPV' in output_metrics:
            metrics_by_threshold['NPV'] = confusion_matrix_nor_predicted[(False, False)]

        if 'Prevalence' in output_metrics:
            metrics_by_threshold['Prevalence'] = \
                (confusion_matrix_dict[(True, True)] + confusion_matrix_dict[(True, False)]) / sum(
                    confusion_matrix_dict.values())

        if 'Accuracy' in output_metrics:
            metrics_by_threshold['Accuracy'] = \
                (confusion_matrix_dict[(True, True)] + confusion_matrix_dict[(False, False)]) / sum(
                    confusion_matrix_dict.values())

        if 'LR+' in output_metrics:
            # positive likelihood ratio:
            try:
                metrics_by_threshold['LR+'] = confusion_matrix_nor_true[(True, True)] / confusion_matrix_nor_true[
                    (False, True)]
            except ZeroDivisionError:
                metrics_by_threshold['LR+'] = '-'

        if 'LR-' in output_metrics:
            # negative likelihood ratio:
            try:
                metrics_by_threshold['LR-'] = confusion_matrix_nor_true[(True, False)] / confusion_matrix_nor_true[
                    (False, False)]
            except ZeroDivisionError:
                metrics_by_threshold['LR-'] = '-'

        if 'DOR' in output_metrics:
            # diagnostic odds ratio:
            try:
                metrics_by_threshold['DOR'] = (confusion_matrix_nor_true[(True, True)] / confusion_matrix_nor_true[
                    (False, True)]) / \
                                              (confusion_matrix_nor_true[(True, False)] / confusion_matrix_nor_true[
                                                  (False, False)])
            except ZeroDivisionError:
                metrics_by_threshold['DOR'] = '-'

        if 'F1' in output_metrics:
            # F1 score:
            try:
                metrics_by_threshold['F1'] = 2 * (
                        confusion_matrix_nor_true[(True, True)] * confusion_matrix_nor_predicted[(True, True)]) / \
                                             (confusion_matrix_nor_true[(True, True)] + confusion_matrix_nor_predicted[
                                                 (True, True)])
            except ZeroDivisionError:
                metrics_by_threshold['F1'] = '-'

        metrics_by_thresholds[threshold] = metrics_by_threshold

    if table:
        if 'metric_order' in kwargs:
            metric_order = kwargs['metric_order']
        else:
            metric_order = None
        metrics_by_thresholds = \
            convert_confusion_matrix_by_prob_to_table_with_reformat_precision(
                metrics_by_thresholds=metrics_by_thresholds,
                metric_order=metric_order)

    return metrics_by_thresholds


def confusion_matrix_by_prob_with_dual_thresholds(true: np.ndarray,
                                                  predicted_prob: np.ndarray,
                                                  threshold_bounds: Union[list, tuple, set],
                                                  pos_label: Union[bool, str, int] = _DEFAULT_POS_LABEL,
                                                  output_metrics: Optional[list] = None,
                                                  unclassified_output_metrics: Optional[list] = None,
                                                  table: bool = True,
                                                  **kwargs):
    # modify threshold bound in case of only one threshold bound given:
    if isinstance(threshold_bounds, tuple) and isinstance(threshold_bounds[0], float):
        threshold_bounds = [threshold_bounds]

    # select output:
    if isinstance(unclassified_output_metrics, list):
        for selected_metric in unclassified_output_metrics:
            if selected_metric not in _FULL_METRICS_DUAL:
                raise KeyError(f"metric {selected_metric} is not recognized.")
    else:
        unclassified_output_metrics = _FULL_METRICS_DUAL

    metrics_by_thresholds_dual = dict()
    for threshold_bound in threshold_bounds:
        # filtering texts below lower threshold and upper threshold:
        lower_threshold, upper_threshold = threshold_bound
        true_sub = true[(predicted_prob <= lower_threshold) | (predicted_prob >= upper_threshold)]
        predicted_sub = predicted_prob[(predicted_prob <= lower_threshold) | (predicted_prob >= upper_threshold)]
        predicted_sub = (predicted_sub >= upper_threshold)  # all predicted_sub over upper threshold is set to 1:
        true_unclassified = true[(predicted_prob > lower_threshold) & (predicted_prob < upper_threshold)]

        metrics_by_thresholds = confusion_matrix_by_prob(true=true_sub,
                                                         predicted_prob=predicted_sub,
                                                         thresholds=[0.5],
                                                         pos_label=pos_label,
                                                         output_metrics=output_metrics,
                                                         table=False)

        # unpack structure:
        metrics_by_threshold = metrics_by_thresholds[0.5]  # default set to 0.5;

        # additional metrics:
        if 'Open_Count' in unclassified_output_metrics:
            metrics_by_threshold['Open_Count'] = len(true_unclassified)

        if 'Open_Positive' in unclassified_output_metrics:
            metrics_by_threshold['Open_Positive'] = (true_unclassified == pos_label).sum().item()

        if 'Open_Negative' in unclassified_output_metrics:
            metrics_by_threshold['Open_Negative'] = (true_unclassified != pos_label).sum().item()

        if 'Open_Prevalence' in unclassified_output_metrics:
            try:
                metrics_by_threshold['Open_Prevalence'] = (true_unclassified == pos_label).sum().item() / len(
                    true_unclassified)
            except ZeroDivisionError:
                metrics_by_threshold['Open_Prevalence'] = '-'

        metrics_by_thresholds_dual[threshold_bound] = metrics_by_threshold

    if table:
        if 'metric_order' in kwargs:
            metric_order = kwargs['metric_order']
        else:
            metric_order = None
        metrics_by_thresholds_dual = \
            convert_confusion_matrix_by_prob_to_table_with_reformat_precision(
                metrics_by_thresholds=metrics_by_thresholds_dual,
                metric_order=metric_order)

    return metrics_by_thresholds_dual


def roc(true: np.ndarray, predicted_prob: np.ndarray, pos_label: Union[bool, str, int] = True) -> Tuple[
    np.ndarray, np.ndarray, np.ndarray]:
    """
    Receiver operating characteristic (ROC):

    :param true: numpy.ndarray(shape=(m), ), an array of true classes;
    :param predicted_prob: numpy.ndarray(shape=(m), ),
        an array of predicted probabilities of being the positive class;
    :param pos_label: [str, bool, int], positive class label, label that is considered as the positive class;
    :return: tuple, (thresholds, fpr, tpr), thresholds: numpy.ndarray, fpr: numpy.ndarray, tpr: numpy.ndarray;
    """
    true = true == pos_label

    # sort by predicted prob:
    sorted_index = np.argsort(predicted_prob)[::-1]
    true = true[sorted_index]
    predicted_prob = predicted_prob[sorted_index]

    tp = 0
    fp = 0
    condition_positive = true.sum()
    condition_negative = (1 - true).sum()

    # set thresholds at predicted_prob:
    thresholds = predicted_prob
    tpr, fpr = [], []
    for i in range(len(thresholds)):
        if true[i] == 1:
            tp += 1
        elif true[i] == 0:
            fp += 1

        tpr_i = tp / condition_positive  # consistent with confusion matrix by prob;
        fpr_i = fp / condition_negative

        tpr.append(tpr_i)
        fpr.append(fpr_i)

    tpr, fpr = np.array(tpr), np.array(fpr)

    return fpr, tpr, thresholds


def roc_auc(**kwargs) -> float:
    """
    compute AUC (area under the curve):
    :param kwargs:
        set 1:
        fpr: numpy.ndarray(shape=(m), ), false positive rate;
        tpr: numpy.ndarray(shape=(m), ), true positive rate;
        set 2:
        true: numpy.ndarray(shape=(m), ), an array of true classes;
        predicted_prob: numpy.ndarray(shape=(m), ),
            an array of predicted probabilities of being the positive class;
        pos_label: [str, bool, int], positive class label, label that is considered as the positive class;
    :return: auc, float, area under the curve;
    """
    if ('fpr' in kwargs) and ('tpr' in kwargs):
        fpr = kwargs['fpr']
        tpr = kwargs['tpr']
    elif ('true' in kwargs) and ('predicted_prob' in kwargs):
        # if fpr and tpr are not given: re-compute roc:
        if 'pos_label' in kwargs:
            pos_label = kwargs['pos_label']
        else:
            pos_label = _DEFAULT_POS_LABEL

        fpr, tpr, thresholds = roc(true=kwargs['true'], predicted_prob=kwargs['predicted_prob'], pos_label=pos_label)
    else:
        raise KeyError(f"given arguments are not sufficient to compute AUC.")

    # fpr, tpr = np.insert(fpr, 0, 1), np.insert(tpr, 0, 1)  # add 1 to the first position of the array;
    # fpr, tpr = np.append(fpr, 0), np.append(tpr, 0)  # add 0 to the last position of the arrays;

    fpr, tpr = np.insert(fpr, 0, 0), np.insert(tpr, 0, 0)  # add 0 to the first position of the array;
    fpr, tpr = np.append(fpr, 1), np.append(tpr, 1)  # add 1 to the last position of the arrays;
    # auc = float(np.sum(1 / 2 * (tpr[0:-1] + tpr[1:]) * (- np.diff(fpr))))  # reverse as roc is cal differently;
    auc = float(np.sum(1 / 2 * (tpr[0:-1] + tpr[1:]) * np.diff(fpr)))
    return auc


def roc_margins(**kwargs) -> dict:
    """
    compute differences between a fpr and tpr:
        also computes
    :param kwargs:
        set 1:
        fpr: numpy.ndarray(shape=(m), ), false positive rate;
        tpr: numpy.ndarray(shape=(m), ), true positive rate;
        set 2:
        true: numpy.ndarray(shape=(m), ), an array of true classes;
        predicted_prob: numpy.ndarray(shape=(m), ),
            an array of predicted probabilities of being the positive class;
        pos_label: [str, bool, int], positive class label, label that is considered as the positive class;
    :return: roc margin dict, dict, including:
        fpr: false positive rate, numpy.ndarray(shape=(m), );
        tpr, true positive rate, numpy.ndarray(shape=(m), );
        margins: margin curve, numpy.ndarray(shape=(m), );
        max_margin: the maximum margin between fpr and tpr;
        max_margin_threshold: threshold for such maximum margin;
    """

    if ('fpr' in kwargs) and ('tpr' in kwargs) and ('thresholds' in kwargs):
        fpr = kwargs['fpr']
        tpr = kwargs['tpr']
        thresholds = kwargs['thresholds']
    elif ('true' in kwargs) and ('predicted_prob' in kwargs):
        # if fpr and tpr are not given: re-compute roc:
        if 'pos_label' in kwargs:
            pos_label = kwargs['pos_label']
        else:
            pos_label = _DEFAULT_POS_LABEL

        fpr, tpr, thresholds = roc(true=kwargs['true'], predicted_prob=kwargs['predicted_prob'], pos_label=pos_label)
    else:
        raise KeyError(f"given arguments are not sufficient to compute AUC.")

    margins = tpr - fpr  # the margin between tpr and fpr
    max_margin = margins.max()
    max_margin_threshold = thresholds[margins.argmax()]

    roc_margin_dict = {'fpr': fpr,
                       'tpr': tpr,
                       'thresholds': thresholds,
                       'margins': margins,
                       'max_margin': max_margin,
                       'max_margin_threshold': max_margin_threshold}

    return roc_margin_dict
