# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/08_evaluations.ipynb.

# %% ../nbs/08_evaluations.ipynb 25
from __future__ import annotations
from .utils import *
from sklearn.metrics import classification_report,ConfusionMatrixDisplay
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt


# %% auto 0
__all__ = ['evaluate_classification_model_metadata', 'show_top_n_predictions', 'evaluate_classification_model']

# %% ../nbs/08_evaluations.ipynb 26
def _show_metrics(label,pred,metric_funcs):
    results=''
    for m_func in metric_funcs:
        m_name=callable_name(m_func)
        results+=f'{m_name}: {m_func(label,pred):.4f} . '
    return results

# %% ../nbs/08_evaluations.ipynb 27
def evaluate_classification_model_metadata(df:pd.DataFrame, # The main dataframe containing the predictions
                                           metadatas:str|list, # Metadata(s) to perform analysis
                                           label_name:str, # Label's column name
                                           pred_name:str, # Prediction's column name
                                           metric_funcs:list, # Metric(s) to calculate
                                          ):
    metadatas = val2iterable(metadatas)
    metric_funcs = val2iterable(metric_funcs)
    
    for metadata in metadatas:
        print_msg(f'{label_name} Analysis on metadata: {metadata}',20)
        print_msg('Distribution',10)
        print(pd.concat((df[metadata].value_counts(),df[metadata].value_counts(normalize=True)),axis=1))
        print()
        print_msg(f'Metrics for each value in {metadata}',10)
        print(f'- For all data:\n{_show_metrics(df[label_name].values,df[pred_name].values,metric_funcs)}')
        for val in np.sort(df[metadata].unique()):
            y_true=df.loc[df[metadata]==val,label_name].values
            y_pred=df.loc[df[metadata]==val,pred_name].values
            print(f'- For {val}:\n{_show_metrics(y_true,y_pred,metric_funcs)}')

# %% ../nbs/08_evaluations.ipynb 31
def show_top_n_predictions(df:pd.DataFrame, # The main dataframe containing the predictions
                          text_name:str, # Text's column name
                          label_name:str, # Label's column name
                          pred_name:str, # Prediction's column name
                          prob_name:str, # Prediction probability's column name
                          is_incorrect=True, # To show top correct or incorrect sentences
                          ascending=False, # To sort by prob_name ascendingly or descendingly
                          n_show=10, # Number of sentences to show
                         ):
    if is_incorrect:
        df = df[df[pred_name]!=df[label_name]]
    else:
        df = df[df[pred_name]==df[label_name]]
    df = df.sort_values(prob_name,ascending=ascending)[[text_name,label_name,pred_name,prob_name]].head(n_show).values
    for row in df:
        print(f'Text: {row[0]}')
        print(f'True label: {row[1]}, {"but" if is_incorrect else ""} predict {row[2]}, with confidence {row[3]:.4f}')
        print('-'*60)

# %% ../nbs/08_evaluations.ipynb 35
def evaluate_classification_model(df:pd.DataFrame, # The main dataframe containing the predictions
                              text_name:str, # Text's column name
                              label_name:str, # Label's column name
                              pred_name:str, # Prediction's column name
                              prob_name:str, # Prediction probability's column name
                              metric_funcs:list=[], # Metric(s) to calculate
                              metadatas:str|list=[], # Metadata(s) to perform analysis
                              n_show=10, # Number of sentences to show
                              cm_figsize=(20,20), # Confusion matrix's figure size
                             ):

    y_true = df[label_name].values
    y_pred = df[pred_name].values
    print_msg(f'{label_name} Analysis',20)
    print()
    print_msg('Classification Report',15)
    print(classification_report(y_true,y_pred))
    print()
    print_msg('Confusion Matrix',15)
    fig,ax = plt.subplots(figsize=cm_figsize)
    ConfusionMatrixDisplay.from_predictions(y_true,y_pred,xticks_rotation='vertical',ax=ax)
    plt.show()
    print()
    evaluate_classification_model_metadata(df,metadatas,label_name,pred_name,metric_funcs)
    print()
    print_msg("Let's look at some wrong predictions with high confidence",15)
    show_top_n_predictions(df,text_name,label_name,pred_name,prob_name,
                           is_incorrect=True,
                           ascending=False,
                           n_show=n_show)
