# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/musique/musique.benchmark.ipynb.

# %% auto 0
__all__ = ['benchmark']

# %% ../../nbs/musique/musique.benchmark.ipynb 4
from typing import Callable

import pandas as pd
from tqdm.auto import tqdm

from .eval import compute_scores_dataframe, aggregate_scores

tqdm.pandas()

# %% ../../nbs/musique/musique.benchmark.ipynb 5
def benchmark(
    dataf: pd.DataFrame,
    pipeline: Callable[[dict], dict],
    ignore_errors: bool = False,
) -> tuple[pd.DataFrame, dict]:

    def process(example):
        output = pipeline(example, ignore_errors=ignore_errors)
        example["predicted_answer"] = output["answer"]
        example["raw_output"] = output
        example["answers"] = [example["answer"], *example["answer_aliases"]]
        return example

    dataf = dataf.progress_apply(process, axis=1)
    dataf = compute_scores_dataframe(dataf)
    scores = aggregate_scores(dataf)
    return dataf, scores
