#AUTOGENERATED! DO NOT EDIT! File to edit: dev/06_labeling.ipynb (unless otherwise specified).

__all__ = ['labeler_agreement']

#Cell
import numpy as np

#Cell
def labeler_agreement(df_full, mode='gold', label_col='new_y'):
    """Compute estimate of inter-labeler agreement for a binary classification
    task from a Figure 8 full labeling report. We compute agreement scores for
    each example, then average these scores over all examples to obtain our
    estimate of overall agreement. Remember that due to quality control
    measures, only labelers deemed "reliable" are included - if we were to
    open the task to any labeler, performance would drop.

    Note that this is one of several ways we could estimate human level
    performance, each with its own pros and cons:

    - Take the aggregated report and compute a simple mean of confidence
    scores for all examples.
    - Take the full report and compute a simple mean of trust scores
    (gold set accuracy) for all labelers.
    - Take the full report and compute average agreement scores for each
    example. (This is the method used here.)

    Parameters
    ----------
    df_full: pd.DataFrame
        Full report from Figure 8, where each row contains a single judgment.
    mode: str
        One of ('gold', 'all', 'regular'). "gold" will compute the estimate
        only using labels for the gold standard set (i.e. the test questions).
        "all" will use the whole set for computation. "regular" will use only
        non-test questions.
    label_col: str
        Name of column containing labels to assess.

    Returns
    -------
    float: Value between 0 and 1 measuring the percent of labelers who
        selected the majority class, averaged over all examples in the job.
    """
    if mode == 'gold':
        df = df_full[df_full._golden]
    elif mode == 'regular':
        df = df_full[~df_full._golden]
    elif mode == 'all':
        df = df_full

    def agreement(seq):
        """Binary inter-labeler agreement score.

        Parameters
        ----------
        seq: list, np.array, pd.Series
            A sequence of binary labels for a single example.

        Returns
        -------
        float: Value between .5 and 1 measuring inter-labeler agreement.
        """
        avg = np.mean(seq)
        return max(avg, 1-avg)

    return df.groupby('_unit_id')[label_col].apply(agreement).mean()