# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/ml.evaluation.ipynb.

# %% auto 0
__all__ = ['find_binary_clf_threshold', 'classification_summary', 'evaluate_slmc']

# %% ../../nbs/ml.evaluation.ipynb 3
import numpy as np
from fastcore.basics import ifnone

# %% ../../nbs/ml.evaluation.ipynb 4
def find_binary_clf_threshold(targets, preds, metrics, *, threshold_range=None, weight=None, metric_names=None, show=True):
    """Find the best threshold for binary classification given targets and predicted probabilities"""
    if weight is None:
        weight = [1.0] * len(metrics)
    if threshold_range is None:
        threshold_range = np.arange(0, 1, 0.05)
    weight = (np.array(weight) / np.sum(weight)).reshape(len(metrics), -1)
    scores = np.array([
        [metric(targets, preds > threshold) for threshold in threshold_range]
        for metric in metrics
    ])
    wavg_scores = (scores * weight).sum(axis=0)
    best_idx = wavg_scores.argmax()
    best_threshold =  threshold_range[best_idx]
    best_score =  wavg_scores[best_idx]

    if show:
        import matplotlib.pyplot as plt
        if metric_names is None:
            metric_names = [None]*len(metrics)
        metric_names = [
            ifnone(name, ifnone(getattr(metric, '__name__', None), f"Metric - {i}")) 
            for i, (metric, name) in enumerate(zip(metrics, metric_names))
        ]
        fig = plt.figure(figsize=(8, 4))
        plt.plot(threshold_range, scores.T, label=metric_names)
        plt.plot(threshold_range, wavg_scores.T, label='Weighted avg', linewidth=2)
        plt.plot([best_threshold], [best_score], 'ro')
        plt.axvline(x=best_threshold, color='gray', linestyle='--')
        plt.legend(loc='best')
        plt.show()
    
    return best_threshold

# %% ../../nbs/ml.evaluation.ipynb 7
import pandas as pd
from sklearn.metrics import classification_report

def classification_summary(targets, preds, class_names, show=False, figsize=(16, 12)):
    label_indices = list(range(len(class_names)))
    clf_dict = classification_report(
        targets,
        preds,
        labels=label_indices,
        target_names=class_names,
        output_dict=True,
    )
    df = pd.DataFrame.from_records(clf_dict).T
    df = df.loc[class_names + sorted(list(set(df.index.values).difference(class_names)))] 
    if show:
        from sklearn.metrics import ConfusionMatrixDisplay
        import matplotlib.pyplot as plt

        fig, ax = plt.subplots(figsize=figsize)
        ConfusionMatrixDisplay.from_predictions(
            targets, 
            preds, 
            labels=label_indices, 
            display_labels=class_names, 
            ax=ax,
        )
    return df

def evaluate_slmc(learn, *, dls=None, dl=None, class_names=None, **kwargs):
    if dl is None:
        assert dls is not None and dls.valid is not None, "You must specify either dls or dl"
        dl = dls.valid
    if class_names is None:
        assert dls is not None, "You must specify either dls or class_names"
        assert dls.vocab is not None, "dls must have vocab"
        class_names = dls.vocab
    _, targets, preds = learn.get_preds(dl=dl, with_decoded=True)
    return classification_summary(targets.cpu().numpy(), preds.cpu().numpy(), class_names=list(class_names), **kwargs)
