#!/usr/bin/env python
# coding=utf-8

"""Various functions for performance assessment

Most of these were imported from older C++ implementations.
"""

import logging

from functools import wraps

import numpy
import numpy.linalg

from numba import jit, objmode

logger = logging.getLogger(__name__)


def _lists_to_arrays(*args, **kwargs):
    ret, retkw = list(), dict()
    for v in args:
        ret.append(numpy.asarray(v) if isinstance(v, list) else v)
    for k, v in kwargs.items():
        retkw[k] = numpy.asarray(v) if isinstance(v, list) else v
    return ret, retkw


def array_jit(func):
    jit_func = jit(func, nopython=True)

    @wraps(jit_func)
    def new_func(*args, **kwargs):
        args, kwargs = _lists_to_arrays(*args, **kwargs)
        return jit_func(*args, **kwargs)

    new_func.jit_func = jit_func
    return new_func


@jit(nopython=True)
def _log_values(points, min_power):
    """Computes log-scaled values between :math:`10^\text{min_power}` and 1

    Parameters
    ==========

    points : int
        Number of points to consider

    min_power : int
        Negative integer with the minimum power


    Returns
    =======

    logscale : numpy.ndarray (float, 1D)

        A set of numbers forming a logarithm-based scale between
        :math:`10^\text{min_power}` and 1.

    """
    return 10 ** (numpy.arange(1 - points, 1) / int(points / (-min_power)))


@jit(nopython=True)
def _meaningful_thresholds(negatives, positives, points, min_far, is_sorted):
    """Returns non-repeatitive thresholds to generate ROC curves

    This function creates a list of FAR (and FRR) values that we are
    interesting to see on the curve.  Computes thresholds for those points.
    Sorts the thresholds so we get sorted numbers to plot on the curve and
    returns the thresholds.  Some points will be duplicate but in terms of
    resolution and accuracy this is better than just changing the threshold
    from ``min()`` of scores to ``max()`` of scores with equal spaces.


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The negative and positive scores, for which the meaningful threshold
        will be calculated.

    n_points : int

        The number of points, in which the ROC curve are calculated, which are
        distributed uniformly in the range ``[min(negatives, positives),
        max(negatives, positives)]``

    min_far : int

        Minimum FAR in terms of :math:`10^(\text{min_far}`. This value is also
        used for ``min_frr``. Values should be negative.


    is_sorted : bool

        Set this to ``True`` if both sets of scores are already sorted in
        ascending order.  If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    thresholds : numpy.ndarray (1D, float)

        The "meaningful" thresholds that would cause changes in the ROC.

    """

    half_points = points // 2

    # if not pre-sorted, copies and sorts
    neg = negatives if is_sorted else numpy.sort(negatives)
    pos = positives if is_sorted else numpy.sort(positives)

    frr_list = _log_values(half_points, min_far)
    far_list = _log_values(points - half_points, min_far)

    t = numpy.zeros((points,))
    t[:half_points] = [_jit_frr_threshold(neg, pos, k, True) for k in frr_list]
    t[half_points:] = [_jit_far_threshold(neg, pos, k, True) for k in far_list]

    t.sort()

    return t


def correctly_classified_negatives(negatives, threshold):
    """Evaluates correctly classifed negatives in a set, based on a threshold

    This method returns an array composed of booleans that pin-point, which
    negatives where correctly classified for the given threshold

    The pseudo-code for this function is:

    .. code-block:: python

       classified = []
       for k in negatives:
           if k < threshold:
               classified.append(True)
           else:
               classified.append(False)


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The scores generated by comparing objects of different classes

    threshold : float

        The threshold, for which scores should be considered to be
        correctly classified


    Returns
    =======

    classified : numpy.ndarray (1D, bool)

        The decision for each of the ``negatives``

    """
    return negatives < threshold


def correctly_classified_positives(positives, threshold):
    """Evaluates correctly classifed positives in a set, based on a threshold

    This method returns an array composed of booleans that pin-point, which
    positives where correctly classified for the given threshold

    The pseudo-code for this function is:

    .. code-block:: python

       classified = []
       for k in positives:
           if k >= threshold:
               classified.append(True)
           else:
               classified.append(False)


    Parameters
    ==========

    positives : numpy.ndarray (1D, float)

        The scores generated by comparing objects of different classes

    threshold : float

        The threshold, for which scores should be considered to be
        correctly classified


    Returns
    =======

    classified : numpy.ndarray (1D, bool)

        The decision for each of the ``positives``

    """
    return positives >= threshold


def det(negatives, positives, n_points, min_far=-8):
    """Calculates points of an Detection Error-Tradeoff (DET) curve

    Calculates the DET curve given a set of negative and positive scores and a
    desired number of points. Returns a two-dimensional array of doubles that
    express on its rows:

    You can plot the results using your preferred tool to first create a plot
    using rows 0 and 1 from the returned value and then replace the X/Y axis
    annotation using a pre-determined set of tickmarks as recommended by NIST.
    The derivative scales are computed with the :py:func:`ppndf` function.


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The list of negative and positive scores to compute the DET for

    n_points : int

        The number of points on the DET curve, for which the DET should be
        evaluated

    min_far : :class:`int`, Optional

        Minimum FAR in terms of :math:`10^\text{min_far}`. This value is also
        used for ``min_frr``. Values should be negative.


    Returns
    =======

    curve : numpy.ndarray (2D, float)

        The DET curve, with the FPR in the first and the FNR in the second
        row:

        0. X axis values in the normal deviate scale for the false-accepts
        1. Y axis values in the normal deviate scale for the false-rejections

    """
    return ppndf(roc(negatives, positives, n_points, min_far))


def pavx_1(y, ghat, index, len):
    """Calculates the pavx_1 function

    Calculates the pavx_1 function given a set of negative and positive scores
    and a desired number of points. Returns a two-dimensional array of doubles
    that express on its rows:

    You can plot the results using your preferred tool to first create a plot
    using rows 0 and 1 from the returned value and then replace the X/Y axis
    annotation using a pre-determined set of tickmarks as recommended by NIST.
    The derivative scales are computed with the :py:func:`ppndf` function.


    Parameters
    ==========

    y : numpy.ndarray (1D, float)

        The list of negative and positive scores to compute the DET for

    ghat : numpy.ndarray (1D, float)

        The list of mean values calculated in the pavx_1 function

    index : numpy.ndarray (1D, size_t)

        The list of indices calculated in the pavx_1 function

    len : numpy.ndarray (1D, size_t)

        The list of lengths calculated in the pavx_1 function


    Returns
    =======

    ci : size_t

        The index of the interval currently considered

    """
    # Sets output and working arrays to 0
    index = 0
    len = 0
    ghat = 0.0
    # ci is the index of the interval currently considered
    # ghat(ci) is the mean of y-values within this interval
    ci = 0
    index[0] = 0
    len[0] = 1
    ghat[0] = y[0]
    for j in range(1, y.shape[0]):
        # a new index interval "j" is created:
        ci += 1
        index[ci] = j
        len[ci] = 1
        ghat[ci] = y[j]
        while ci >= 1 and ghat[numpy.max(ci - 1, 0)] >= ghat[ci]:
            # "pool adjacent violators"
            nw = len[ci - 1] + len[ci]
            ghat[ci - 1] += (len[ci] / nw) * (ghat[ci] - ghat[ci - 1])
            len[ci - 1] = nw
            ci -= 1
    return ci


def pavx_2(ghat, index, ci):
    """Calculates the pavx_2 function

    Calculates the pavx_2 function given the pavx_1 function.

    Parameters
    ==========

    ghat : numpy.ndarray (1D, float)

        The list of mean values calculated in the pavx_1 function

    index : numpy.ndarray (1D, size_t)

        The list of indices calculated in the pavx_1 function

    ci : size_t

        The index of the interval currently considered

    """
    # define ghat for all indices
    n = index[ci]
    while n >= 1:
        r = numpy.array(range(index[ci], n))
        ghat[r] = ghat[ci]
        n = index[ci]
        ci -= 1
    return ghat


def pavxWidth(input, output):
    """Applies the Pool-Adjacent-Violators Algorithm and returns the width.

    This is a simplified C++ port of the isotonic regression code made available at the University of Bern website.

    Parameters
    ==========

    input : array_like (float, 1D)

        The input matrix for the PAV algorithm.

    output : array_like (float, 1D)

        The output matrix, must be of the same size as input

    Returns
    =======

    width : array_like (uint64, 1D)

        The width matrix will be created in the same size as input
    """
    input = numpy.array(input)
    output = numpy.array(output)

    # Define working arrays: An interval of indices is represented by its left
    # endpoint "index" and its length "len"
    N = input.shape[0]
    index = numpy.zeros(N, dtype=numpy.uint64)
    len = numpy.zeros(N, dtype=numpy.uint64)

    # First step
    ci = pavx_1(input, output, index, len)

    # Get the width vector
    width = len[: ci + 1]

    # Second step
    pavx_2(output, index, ci)

    return width


def rocch(negatives, positives):
    """Calculates the ROC Convex Hull (ROCCH) curve given a set of positive and negative scores


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the curve


    Returns
    =======

    curve : numpy.ndarray (2D, float)

        The ROC curve, with the first row containing the FPR, and the second
        row containing the FNR.

    """

    # Number of positive and negative scores
    Nt = len(positives)
    Nn = len(negatives)
    N = Nt + Nn

    # Creates a big array with all scores
    scores = numpy.concatenate((positives, negatives))

    # It is important here that scores that are the same (i.e. already in
    # order) should NOT be swapped.  "stable" has this property.
    perturb = numpy.argsort(scores, kind="stable")

    # Apply permutation
    Pideal = numpy.zeros((N,))
    Pideal[perturb < Nt] = 1.0

    # Applies the PAVA algorithm
    Popt = numpy.ndarray((N,))
    raise NotImplementedError(
        "An auto generated implementation of pavxWidth is available but no test has been done."
    )
    width = pavxWidth(Pideal, Popt)

    # Allocates output
    nbins = len(width)
    retval = numpy.zeros((2, nbins + 1))  # FAR, FRR

    # Fills in output
    left = 0
    fa = Nn
    miss = 0

    for i in range(nbins):
        retval[0, i] = fa / Nn  # pfa
        retval[1, i] = miss / Nt  # pmiss
        left += int(width[i])

        if left >= 1:
            miss = Pideal[:left].sum()
        else:
            miss = 0.0

        if Pideal.shape[0] - 1 >= left:
            fa = N - left - Pideal[left:].sum()
        else:
            fa = 0

    retval[0, nbins] = fa / Nn  # pfa
    retval[1, nbins] = miss / Nt  # pmiss

    return retval


@array_jit
def rocch2eer(pmiss_pfa):
    """Calculates the threshold that is as close as possible to the equal-error-rate (EER) given the input data


    .. todo::

       The parameter(s) ``pmiss_pfa`` are used, but not documented.


    Returns
    =======

    threshold : float

        The computed threshold, at which the EER can be obtained

    """

    assert pmiss_pfa.shape[0] == 2

    N = pmiss_pfa.shape[1]

    eer = 0.0
    XY = numpy.empty((2, 2))
    one = numpy.ones((2,))
    eerseg = 0.0
    epsilon = numpy.finfo(numpy.float64).eps

    for i in range(N - 1):
        # Define XY matrix
        XY[0, 0] = pmiss_pfa[0, i]  # pfa
        XY[1, 0] = pmiss_pfa[0, i + 1]  # pfa
        XY[0, 1] = pmiss_pfa[1, i]  # pmiss
        XY[1, 1] = pmiss_pfa[1, i + 1]  # pmiss

        # xx and yy should be sorted:
        assert XY[1, 0] <= XY[0, 0] and XY[0, 1] <= XY[1, 1]

        # Commpute "dd"
        abs_dd0 = abs(XY[0, 0] - XY[1, 0])
        abs_dd1 = abs(XY[0, 1] - XY[1, 1])

        if min(abs_dd0, abs_dd1) < epsilon:
            eerseg = 0.0

        else:
            # Finds line coefficients seg s.t. XY.seg = 1
            seg = numpy.linalg.solve(XY, one)
            # Candidate for the EER (to be compared to current value)
            eerseg = 1.0 / seg.sum()

        eer = max(eer, eerseg)

    return eer


def eer_rocch(negatives, positives):
    """Equal-error-rate (EER) given the input data, on the ROC Convex Hull

    It replicates the EER calculation from the Bosaris toolkit
    (https://sites.google.com/site/bosaristoolkit/).


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The set of negative scores to compute the threshold

    positives : numpy.ndarray (1D, float)

        The set of positive scores to compute the threshold


    Returns
    =======

    threshold : float

        The threshold for the equal error rate

    """
    return rocch2eer(rocch(negatives, positives))


@jit("float64(float64, float64, float64)", nopython=True)
def _abs_diff(a, b, cost):
    return abs(a - b)


@jit("float64(float64, float64, float64)", nopython=True)
def _weighted_err(far, frr, cost):
    return (cost * far) + ((1.0 - cost) * frr)


@jit(nopython=True)
def _minimizing_threshold(negatives, positives, criterion, cost=0.5):
    """Calculates the best threshold taking a predicate as input condition

    This method can calculate a threshold based on a set of scores (positives
    and negatives) given a certain minimization criterium, input as a
    functional predicate. For a discussion on ``positive`` and ``negative`` see
    :py:func:`farfrr`.  Here, it is expected that the positives and the
    negatives are sorted ascendantly.

    The predicate method gives back the current minimum given false-acceptance
    (FA) and false-rejection (FR) ratios for the input data. The API for the
    criterium is:

    predicate(fa_ratio : float, fr_ratio : float) -> float

    Please note that this method will only work with single-minimum smooth
    predicates.

    The minimization is carried out in a data-driven way.  Starting from the
    lowest score (might be a positive or a negative), it increases the
    threshold based on the distance between the current score and the following
    higher score (also keeping track of duplicate scores) and computes the
    predicate for each possible threshold.

    Finally, that threshold is returned, for which the predicate returned the
    lowest value.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)
        Negative scores, sorted ascendantly

    positives : numpy.ndarray (1D, float)
        Positive scores, sorted ascendantly

    criterion : str
        A predicate from one of ("absolute-difference", "weighted-error")

    cost : float
        Extra cost argument to be passed to criterion

    Returns
    =======

    threshold : float
        The optimal threshold given the predicate and the scores

    """
    if criterion not in ("absolute-difference", "weighted-error"):
        raise ValueError("Uknown criterion")

    def criterium(a, b, c):
        if criterion == "absolute-difference":
            return _abs_diff(a, b, c)
        else:
            return _weighted_err(a, b, c)

    if not len(negatives) or not len(positives):
        raise RuntimeError(
            "Cannot compute threshold when no positives or "
            "no negatives are provided"
        )

    # iterates over all possible far and frr points and compute the predicate
    # for each possible threshold...
    min_predicate = 1e8
    min_threshold = 1e8
    current_predicate = 1e8
    # we start with the extreme values for far and frr
    far = 1.0
    frr = 0.0

    # the decrease/increase for far/frr when moving one negative/positive
    max_neg = len(negatives)
    far_decrease = 1.0 / max_neg
    max_pos = len(positives)
    frr_increase = 1.0 / max_pos

    # we start with the threshold based on the minimum score

    # iterates until one of these goes bananas
    pos_it = 0
    neg_it = 0
    current_threshold = min(negatives[neg_it], positives[pos_it])

    # continues until one of the two iterators reaches the end of the list
    while pos_it < max_pos and neg_it < max_neg:
        # compute predicate
        current_predicate = criterium(far, frr, cost)

        if current_predicate <= min_predicate:
            min_predicate = current_predicate
            min_threshold = current_threshold

        if positives[pos_it] >= negatives[neg_it]:
            # compute current threshold
            current_threshold = negatives[neg_it]
            neg_it += 1
            far -= far_decrease

        else:  # pos_val <= neg_val
            # compute current threshold
            current_threshold = positives[pos_it]
            pos_it += 1
            frr += frr_increase

        # skip until next "different" value, which case we "gain" 1 unit on
        # the "FAR" value, since we will be accepting that negative as a
        # true negative, and not as a false positive anymore.  we continue
        # to do so for as long as the current threshold matches the current
        # iterator.
        while neg_it < max_neg and current_threshold == negatives[neg_it]:
            neg_it += 1
            far -= far_decrease

        # skip until next "different" value, which case we "loose" 1 unit
        # on the "FRR" value, since we will be accepting that positive as a
        # false negative, and not as a true positive anymore.  we continue
        # to do so for as long as the current threshold matches the current
        # iterator.
        while pos_it < max_pos and current_threshold == positives[pos_it]:
            pos_it += 1
            frr += frr_increase

        # computes a new threshold based on the center between last and current
        # score, if we are **not** already at the end of the score lists
        if neg_it < max_neg or pos_it < max_pos:
            if neg_it < max_neg and pos_it < max_pos:
                current_threshold += min(negatives[neg_it], positives[pos_it])
            elif neg_it < max_neg:
                current_threshold += negatives[neg_it]
            else:
                current_threshold += positives[pos_it]
            current_threshold /= 2

    # now, we have reached the end of one list (usually the negatives) so,
    # finally compute predicate for the last time
    current_predicate = criterium(far, frr, cost)
    if current_predicate < min_predicate:
        min_predicate = current_predicate
        min_threshold = current_threshold

    # now we just double check choosing the threshold higher than all scores
    # will not improve the min_predicate
    if neg_it < max_neg or pos_it < max_pos:
        last_threshold = current_threshold
        if neg_it < max_neg:
            last_threshold = numpy.nextafter(negatives[-1], negatives[-1] + 1)
        elif pos_it < max_pos:
            last_threshold = numpy.nextafter(positives[-1], positives[-1] + 1)
        current_predicate = criterium(0.0, 1.0, cost)
        if current_predicate < min_predicate:
            min_predicate = current_predicate
            min_threshold = last_threshold

    # return the best threshold found
    return min_threshold


def eer_threshold(negatives, positives, is_sorted=False):
    """Calculates threshold as close as possible to the equal error rate (EER)

    The EER should be the point where the FPR equals the FNR. Graphically, this
    would be equivalent to the intersection between the ROC (or DET) curves and
    the identity.

    .. note::

       The scores will be sorted internally, requiring the scores to be copied.
       To avoid this copy, you can sort both sets of scores externally in
       ascendant order, and set the ``is_sorted`` parameter to ``True``.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The set of negative scores to compute the threshold

    positives : numpy.ndarray (1D, float)

        The set of positive scores to compute the threshold

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``negatives`` are already sorted in
        ascending order.  If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    threshold : float

        The threshold (i.e., as used in :py:func:`farfrr`) where FPR and FNR
        are as close as possible

    """

    # if not pre-sorted, copies and sorts
    neg = negatives if is_sorted else numpy.sort(negatives)
    pos = positives if is_sorted else numpy.sort(positives)

    return _minimizing_threshold(neg, pos, "absolute-difference")


@array_jit
def epc(
    dev_negatives,
    dev_positives,
    test_negatives,
    test_positives,
    n_points,
    is_sorted=False,
    thresholds=False,
):
    """Calculates points of an Expected Performance Curve (EPC)

    Calculates the EPC curve given a set of positive and negative scores and a
    desired number of points. Returns a two-dimensional
    :py:class:`numpy.ndarray` of type float with the shape of ``(2, points)``
    or ``(3, points)`` depending on the ``thresholds`` argument.  The rows
    correspond to the X (cost), Y (weighted error rate on the test set given
    the min. threshold on the development set), and the thresholds which were
    used to calculate the error (if the ``thresholds`` argument was set to
    ``True``), respectively. Please note that, in order to calculate the EPC
    curve, one needs two sets of data comprising a development set and a test
    set. The minimum weighted error is calculated on the development set and
    then applied to the test set to evaluate the weighted error rate at that
    position.

    The EPC curve plots the HTER on the test set for various values of 'cost'.
    For each value of 'cost', a threshold is found that provides the minimum
    weighted error (see :py:func:`min_weighted_error_rate_threshold`) on the
    development set. Each threshold is consecutively applied to the test set
    and the resulting weighted error values are plotted in the EPC.

    The cost points in which the EPC curve are calculated are distributed
    uniformly in the range :math:`[0.0, 1.0]`.

    .. note::

       It is more memory efficient, when sorted arrays of scores are provided
       and the ``is_sorted`` parameter is set to ``True``.


    Parameters
    ==========

    dev_negatives : numpy.ndarray (1D, float)

        Negative scores on the development set

    dev_positives : numpy.ndarray (1D, float)

        Positive scores on the development set

    test_negatives : numpy.ndarray (1D, float)

        Negative scores on the test set

    test_positives :  numpy.ndarray (1D, float)

        Positive scores on the test set

    n_points : int

        The number of weights for which the EPC curve should be computed

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``negatives`` are already sorted in
        ascending order.  If ``False``, scores will be sorted internally, which
        will require more memory.

    thresholds : :py:class:`bool`, Optional

        If ``True`` the function returns an array with the shape of ``(3,
        points)`` where the third row contains the thresholds that were
        calculated on the development set.


    Returns
    =======

    curve : numpy.ndarray (2D or 3D, float)

        The EPC curve, with the first row containing the weights and the second
        row containing the weighted errors on the test set.  If ``thresholds``
        is ``True``, there is also a third row which contains the thresholds
        that were calculated on the development set.

    """

    # if not pre-sorted, copies and sorts
    dev_neg = dev_negatives if is_sorted else numpy.sort(dev_negatives)
    dev_pos = dev_positives if is_sorted else numpy.sort(dev_positives)
    # numpy.linspace is more stable than numpy.arange for non-integer steps.
    # However, both arange and linspace are buggy in numba. Using objmode for a
    # workaround. TODO(amir): remove objmode once
    # https://github.com/numba/numba/issues/6768 is resolved.
    with objmode(alpha="float64[:]"):
        alpha = numpy.linspace(0, 1.0, n_points)
    thres = numpy.empty_like(alpha)
    mwer = numpy.empty_like(alpha)
    for i, k in enumerate(alpha):
        thres[i] = _jit_min_weighted_error_rate_threshold(
            dev_neg, dev_pos, k, True
        )
        tmp = _jit_farfrr(test_negatives, test_positives, thres[i])
        tmp2 = numpy.empty((2,))
        tmp2[0] = tmp[0]
        tmp2[1] = tmp[1]
        mwer[i] = numpy.mean(tmp2)

    if thresholds:
        return numpy.vstack((alpha, mwer, thres))
    return numpy.vstack((alpha, mwer))


def f_score(negatives, positives, threshold, weight=1.0):
    r"""Computes the F-score of the accuracy of the classification

    The F-score is a weighted mean of precision and recall measurements, see
    :py:func:`precision_recall`.  It is computed as:

    .. math::

       \mathrm{\text{f-score}}(w) = (1 + w^2)\frac{\mathrm{precision}\cdot{}\mathrm{recall}}{w^2\cdot{}\mathrm{precision} + \mathrm{recall}}

    The weight :math:`w` needs to be non-negative real value. In case the
    weight parameter is 1 (the default), the F-score is called F1 score and is
    a harmonic mean between precision and recall values.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The set of negative scores to compute the precision and recall

    positives : numpy.ndarray (1D, float)

        The set of positive scores to compute the precision and recall

    threshold : float

        The threshold to compute the precision and recall for

    weight : :py:class:`float`, Optional

        The weight :math:`w` between precision and recall


    Returns
    =======

    f_score : float

        The computed f-score for the given scores and the given threshold

    """
    weight = weight if weight > 0 else 1
    w2 = weight**2
    p, r = precision_recall(negatives, positives, threshold)
    if p == 0.0 and r == 0.0:
        return 0.0
    return (1 + w2) * (p * r) / ((w2 * p) + r)


@array_jit
def far_threshold(negatives, positives, far_value=0.001, is_sorted=False):
    """Threshold such that the real FPR is **at most** the requested ``far_value`` if possible


    .. note::

       The scores will be sorted internally, requiring the scores to be copied.
       To avoid this copy, you can sort the ``negatives`` scores externally in
       ascendant order, and set the ``is_sorted`` parameter to ``True``.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The set of negative scores to compute the FPR threshold

    positives : numpy.ndarray (1D, float)

        Ignored, but needs to be specified -- may be given as ``[]``

    far_value : :py:class:`float`, Optional

        The FPR value, for which the threshold should be computed

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``negatives`` are already sorted in
        ascending order.  If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    threshold : float

        The threshold such that the real FPR is at most ``far_value``

    """

    # N.B.: Unoptimized version ported from C++

    if far_value < 0.0 or far_value > 1.0:
        raise RuntimeError("`far_value' must be in the interval [0.,1.]")

    if len(negatives) < 2:
        raise RuntimeError("the number of negative scores must be at least 2")

    epsilon = numpy.finfo(numpy.float64).eps
    # if not pre-sorted, copies and sorts
    scores = negatives if is_sorted else numpy.sort(negatives)

    # handles special case of far == 1 without any iterating
    if far_value >= (1 - epsilon):
        return numpy.nextafter(scores[0], scores[0] - 1)

    # Reverse negatives so the end is the start. This way the code below will
    # be very similar to the implementation in the frr_threshold function. The
    # implementations are not exactly the same though.
    scores = numpy.flip(scores)

    # Move towards the end of array changing the threshold until we cross the
    # desired FAR value. Starting with a threshold that corresponds to FAR ==
    # 0.
    total_count = len(scores)
    current_position = 0

    # since the comparison is `if score >= threshold then accept as genuine`,
    # we can choose the largest score value + eps as the threshold so that we
    # can get for 0% FAR.
    valid_threshold = numpy.nextafter(
        scores[current_position], scores[current_position] + 1
    )
    current_threshold = 0.0

    while current_position < total_count:
        current_threshold = scores[current_position]
        # keep iterating if values are repeated
        while (
            current_position < (total_count - 1)
            and scores[current_position + 1] == current_threshold
        ):
            current_position += 1
        # All the scores up to the current position and including the current
        # position will be accepted falsely.
        future_far = (current_position + 1) / total_count
        if future_far > far_value:
            break
        valid_threshold = current_threshold
        current_position += 1

    return valid_threshold


_jit_far_threshold = far_threshold.jit_func


@array_jit
def farfrr(negatives, positives, threshold):
    """Calculates the false-acceptance (FA) ratio and the false-rejection (FR) ratio for the given positive and negative scores and a score threshold

    ``positives`` holds the score information for samples that are labeled to
    belong to a certain class (a.k.a., 'signal' or 'client').  ``negatives``
    holds the score information for samples that are labeled **not** to belong
    to the class (a.k.a., 'noise' or 'impostor'). It is expected that
    'positive' scores are, at least by design, greater than 'negative' scores.
    So, every 'positive' value that falls bellow the threshold is considered a
    false-rejection (FR). `negative` samples that fall above the threshold are
    considered a false-accept (FA).

    Positives that fall on the threshold (exactly) are considered correctly
    classified. Negatives that fall on the threshold (exactly) are considered
    **incorrectly** classified. This equivalent to setting the comparison like
    this pseudo-code:

    .. code-block:: python

       false_rejects = 0
       false_accepts = 0
       for k in positives:
           if k < threshold:
               false_rejects += 1
       for k in negatives:
           if k >= threshold:
               false_accepts += 1


    The output is in form of a tuple of two double-precision float numbers.
    The numbers range from 0 to 1. The first element of the pair is the false
    positive ratio (FPR), the second element the false negative ratio (FNR).

    The ``threshold`` value does not necessarily have to fall in the range
    covered by the input scores (negatives and positives altogether), but if it
    does not, the output will be either (1.0, 0.0) or (0.0, 1.0), depending on
    the side the threshold falls.

    It is possible that scores are inverted in the negative/positive sense.  In
    some setups the designer may have setup the system so 'positive' samples
    have a smaller score than the 'negative' ones. In this case, make sure you
    normalize the scores so positive samples have greater scores before feeding
    them into this method.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        The scores for comparisons of objects of different classes

    positives : numpy.ndarray (1D, float)

        The scores for comparisons of objects of the same class

    threshold : float

        The threshold to separate correctly and incorrectly classified scores


    Returns
    =======

    far : float

        The False Positve Rate (FPR) for the given threshold

    frr : float

        The False Negative Rate (FNR) for the given threshold

    """

    if numpy.isnan(threshold):
        print("Error: Cannot compute FPR (FAR) or FNR (FRR) with NaN threshold")
        return (1.0, 1.0)

    if not len(negatives):
        raise RuntimeError(
            "Cannot compute FPR (FAR) when no negatives are given"
        )

    if not len(positives):
        raise RuntimeError(
            "Cannot compute FNR (FRR) when no positives are given"
        )

    return (negatives >= threshold).sum() / len(negatives), (
        positives < threshold
    ).sum() / len(positives)


_jit_farfrr = farfrr.jit_func


@array_jit
def frr_threshold(negatives, positives, frr_value=0.001, is_sorted=False):
    """Computes the threshold such that the real FNR is **at most** the requested ``frr_value`` if possible


    .. note::

       The scores will be sorted internally, requiring the scores to be copied.
       To avoid this copy, you can sort the ``positives`` scores externally in
       ascendant order, and set the ``is_sorted`` parameter to ``True``.


    Parameters
    ==========

    negatives : numpy.ndarray (1D, float)

        Ignored, but needs to be specified -- may be given as ``[]``.

    positives : numpy.ndarray (1D, float)

        The set of positive scores to compute the FNR threshold.

    frr_value : :py:class:`float`, Optional

        The FNR value, for which the threshold should be computed.

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``positives`` are already sorted in
        ascending order. If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    threshold : float

        The threshold such that the real FRR is at most ``frr_value``.

    """

    # N.B.: Unoptimized version ported from C++

    if frr_value < 0.0 or frr_value > 1.0:
        raise RuntimeError("`frr_value' value must be in the interval [0.,1.]")

    if len(positives) < 2:
        raise RuntimeError("the number of positive scores must be at least 2")

    epsilon = numpy.finfo(numpy.float64).eps
    # if not pre-sorted, copies and sorts
    scores = positives if is_sorted else numpy.sort(positives)

    # handles special case of far == 1 without any iterating
    if frr_value >= (1 - epsilon):
        return numpy.nextafter(scores[-1], scores[-1] + 1)

    # Move towards the end of array changing the threshold until we cross the
    # desired FRR value. Starting with a threshold that corresponds to FRR ==
    # 0.
    total_count = len(scores)
    current_position = 0

    # since the comparison is `if score >= threshold then accept as genuine`,
    # we can choose the largest score value + eps as the threshold so that we
    # can get for 0% FAR.
    valid_threshold = numpy.nextafter(
        scores[current_position], scores[current_position] + 1
    )
    current_threshold = 0.0

    while current_position < total_count:
        current_threshold = scores[current_position]
        # keep iterating if values are repeated
        while (
            current_position < (total_count - 1)
            and scores[current_position + 1] == current_threshold
        ):
            current_position += 1
        # All the scores up to the current position and including the current
        # position will be accepted falsely.
        future_frr = current_position / total_count
        if future_frr > frr_value:
            break
        valid_threshold = current_threshold
        current_position += 1

    return valid_threshold


_jit_frr_threshold = frr_threshold.jit_func


def min_hter_threshold(negatives, positives, is_sorted=False):
    """Calculates the :py:func:`min_weighted_error_rate_threshold` with ``cost=0.5``

    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the threshold

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``positives`` are already sorted in
        ascending order. If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    threshold : float

        The threshold for which the weighted error rate is minimal

    """
    return min_weighted_error_rate_threshold(
        negatives, positives, 0.5, is_sorted
    )


@array_jit
def min_weighted_error_rate_threshold(
    negatives, positives, cost, is_sorted=False
):
    """Calculates the threshold that minimizes the error rate

    The ``cost`` parameter determines the relative importance between
    false-accepts and false-rejections. This number should be between 0 and 1
    and will be clipped to those extremes. The value to minimize becomes:
    :math:`ER_{cost} = cost * FPR + (1-cost) * FNR`. The higher the cost, the
    higher the importance given to **not** making mistakes classifying
    negatives/noise/impostors.

    .. note::

       The scores will be sorted internally, requiring the scores to be copied.
       To avoid this copy, you can sort both sets of scores externally in
       ascendant order, and set the ``is_sorted`` parameter to ``True``.


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the threshold

    cost : float

        The relative cost over FPR with respect to FNR in the threshold
        calculation

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if the ``positives`` are already sorted in
        ascending order. If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    threshold : float

        The threshold for which the weighted error rate is minimal

    """

    # if not pre-sorted, copies and sorts
    neg = negatives if is_sorted else numpy.sort(negatives)
    pos = positives if is_sorted else numpy.sort(positives)
    if cost > 1.0:
        cost = 1.0
    elif cost < 0.0:
        cost = 0.0

    return _minimizing_threshold(neg, pos, "weighted-error", cost)


_jit_min_weighted_error_rate_threshold = (
    min_weighted_error_rate_threshold.jit_func
)


# @jit([(numba.float64[:, :],)], nopython=True)
def ppndf(p):
    """Returns the Deviate Scale equivalent of a false rejection/acceptance ratio

    The algorithm that calculates the deviate scale is based on function
    ``ppndf()`` from the NIST package DETware version 2.1, freely available on
    the internet. Please consult it for more details. By 20.04.2011, you could
    find such package `here <http://www.itl.nist.gov/iad/mig/tools/>`_.

    The input to this function is a cumulative probability.  The output from
    this function is the Normal deviate that corresponds to that probability.
    For example:

    -------+--------
     INPUT | OUTPUT
    -------+--------
     0.001 | -3.090
     0.01  | -2.326
     0.1   | -1.282
     0.5   |  0.0
     0.9   |  1.282
     0.99  |  2.326
     0.999 |  3.090
    -------+--------


    Parameters
    ==========

    p : numpy.ndarray (2D, float)

        The value (usually FPR or FNR) for which the PPNDF should be calculated


    Returns
    =======

    ppndf : numpy.ndarray (2D, float)

        The derivative scale of the given value

    """

    # threshold
    epsilon = numpy.finfo(numpy.float64).eps
    p_new = numpy.copy(p)
    p_new = numpy.where(p_new >= 1.0, 1.0 - epsilon, p_new)
    p_new = numpy.where(p_new <= 0.0, epsilon, p_new)

    q = p_new - 0.5
    abs_q_smaller = numpy.abs(q) <= 0.42
    abs_q_bigger = ~abs_q_smaller

    retval = numpy.zeros_like(p_new)

    # first part q<=0.42
    q1 = q[abs_q_smaller]
    r = numpy.square(q1)
    opt1 = (
        q1
        * (
            ((-25.4410604963 * r + 41.3911977353) * r + -18.6150006252) * r
            + 2.5066282388
        )
        / (
            (
                ((3.1308290983 * r + -21.0622410182) * r + 23.0833674374) * r
                + -8.4735109309
            )
            * r
            + 1.0
        )
    )
    retval[abs_q_smaller] = opt1

    # second part q>0.42
    # r = sqrt (log (0.5 - abs(q)));
    q2 = q[abs_q_bigger]
    r = p_new[abs_q_bigger]
    r[q2 > 0] = 1 - r[q2 > 0]
    if (r <= 0).any():
        raise RuntimeError("measure::ppndf(): r <= 0.0!")

    r = numpy.sqrt(-1 * numpy.log(r))
    opt2 = (
        ((2.3212127685 * r + 4.8501412713) * r + -2.2979647913) * r
        + -2.7871893113
    ) / ((1.6370678189 * r + 3.5438892476) * r + 1.0)
    opt2[q2 < 0] *= -1
    retval[abs_q_bigger] = opt2

    return retval


@array_jit
def precision_recall(negatives, positives, threshold):
    r"""Calculates the precision and recall (sensitivity) values given negative and positive scores and a threshold

    Precision and recall are computed as:

    .. math::

       \mathrm{precision} = \frac{tp}{tp + fp}

       \mathrm{recall} = \frac{tp}{tp + fn}

    where :math:`tp` are the true positives, :math:`fp` are the false positives
    and :math:`fn` are the false negatives.

    ``positives`` holds the score information for samples that are labeled to
    belong to a certain class (a.k.a., 'signal' or 'client').  ``negatives``
    holds the score information for samples that are labeled **not** to belong
    to the class (a.k.a., 'noise' or 'impostor'). For more precise details
    about how the method considers error rates, see :py:func:`farfrr`.


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the measurements

    threshold : float

        The threshold to compute the measures for


    Returns
    =======

    precision : float

        The precision value for the given negatives and positives

    recall : float

        The recall value for the given negatives and positives

    """

    if not len(positives) or not len(negatives):
        raise RuntimeError(
            "Cannot compute precision or recall when no "
            "positives or no negatives are given"
        )

    FP = (negatives >= threshold).sum()
    TP = (positives >= threshold).sum()
    CP = TP + FP
    if CP == 0:
        CP = 1
    return TP / CP, TP / len(positives)


_jit_precision_recall = precision_recall.jit_func


@array_jit
def precision_recall_curve(negatives, positives, n_points):
    """Calculates the precision-recall curve given a set of positive and negative scores and a number of desired points

    The points in which the curve is calculated are distributed uniformly in
    the range ``[min(negatives, positives), max(negatives, positives)]``


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the measurements

    n_points : int

        The number of thresholds for which precision and recall should be
        evaluated


    Returns
    =======

    curve : numpy.ndarray (2D, float)

        2D array of floats that express the X (precision) and Y (recall)
        coordinates.

    """
    curve = numpy.empty((2, n_points))
    for i, k in enumerate(
        _meaningful_thresholds(negatives, positives, n_points, -8, False)
    ):
        x, y = _jit_precision_recall(negatives, positives, k)
        curve[0, i] = x
        curve[1, i] = y
    return curve
    # return numpy.array(
    #     [

    #         for k in
    #     ]
    # ).T


@array_jit
def roc(negatives, positives, n_points, min_far=-8):
    """Calculates points of an Receiver Operating Characteristic (ROC)

    Calculates the ROC curve given a set of negative and positive scores and a
    desired number of points.


    Parameters
    ==========

    ``negatives, positives`` : numpy.ndarray (1D, float)

        The negative and positive scores, for which the ROC curve should be
        calculated.

    n_points : int

        The number of points, in which the ROC curve are calculated, which are
        distributed uniformly in the range ``[min(negatives, positives),
        max(negatives, positives)]``

    min_far : int

        Minimum FAR in terms of :math:`10^(\text{min_far}`. This value is also
        used for ``min_frr``. Values should be negative.


    Returns
    =======

    curve : numpy.ndarray (2D, float)

        A two-dimensional array of doubles that express the X (FPR) and Y (FNR)
        coordinates in this order

    """

    t = _meaningful_thresholds(negatives, positives, n_points, min_far, False)
    curve = numpy.empty((2, len(t)))
    for i, k in enumerate(t):
        curve[:, i] = _jit_farfrr(negatives, positives, k)
    return curve


@array_jit
def roc_for_far(negatives, positives, far_list, is_sorted=False):
    """Calculates the ROC curve for a given set of positive and negative scores and the FPR values, for which the FNR should be computed

    .. note::

       The scores will be sorted internally, requiring the scores to be copied.
       To avoid this copy, you can sort both sets of scores externally in
       ascendant order, and set the ``is_sorted`` parameter to ``True``.


    Parameters
    ==========

    negatives, positives : numpy.ndarray (1D, float)

        The set of negative and positive scores to compute the curve

    far_list : numpy.ndarray (1D, float)

        A list of FPR values, for which the FNR values should be computed

    is_sorted : :py:class:`bool`, Optional

        Set this to ``True`` if both sets of scores are already sorted in
        ascending order.  If ``False``, scores will be sorted internally, which
        will require more memory.


    Returns
    =======

    curve : numpy.ndarray (2D, float)

        The ROC curve, which holds a copy of the given FPR values in row 0, and
        the corresponding FNR values in row 1.

    """
    if len(negatives) == 0:
        raise RuntimeError("The given set of negatives is empty.")

    if len(positives) == 0:
        raise RuntimeError("The given set of positives is empty.")

    # if not pre-sorted, copies and sorts
    neg = negatives if is_sorted else numpy.sort(negatives)
    pos = positives if is_sorted else numpy.sort(positives)

    # Get the threshold for the requested far values and calculate far and frr
    # values based on the threshold.
    curve = numpy.empty((2, len(far_list)))
    for i, k in enumerate(far_list):
        curve[:, i] = _jit_farfrr(
            neg, pos, _jit_far_threshold(neg, pos, k, True)
        )
    return curve
