import itertools
from unittest.mock import MagicMock

from synthegrator.code_solver import DummyCodeSolverAutoRegressive
from synthegrator.execution_threading import evaluate_all_solutions, solve_and_evaluate_problems
from synthegrator.solution_eval import SolutionEvaluator
from synthegrator.synthdatasets.human_eval import yield_human_eval


class FailEvaluator(SolutionEvaluator):
    def evaluate(self, solution):
        raise ValueError("This evaluator always fails")


def test_catch_fail_with_trace():
    solver = DummyCodeSolverAutoRegressive(MagicMock())
    problems = itertools.islice(yield_human_eval(), 10)
    evals = solve_and_evaluate_problems(
        solver,
        problems,
        max_threads_solve=1,
        max_threads_eval=2,
        evaluator=FailEvaluator(None),
    )
    evals = list(evals)
    assert len(evals) == 10
    assert evals[0].main_metric is None
    assert evals[0].exception is not None
    assert str(evals[0].exception) == "This evaluator always fails"
    assert evals[0].exception.__traceback__ is not None
    assert evals[0].exception_traceback is not None
    assert __file__ in evals[0].exception_traceback