"""
Templates for benchmark evaluation.

This module contains prompt templates used for evaluating model outputs
against reference answers.
"""

# Template for SimpleQA grading
SIMPLEQA_GRADER_TEMPLATE = """
You are evaluating a question-answering system. Given a question, the correct answer,
and the model's response, determine if the model's answer is correct.

Question: {question}

Correct Answer: {correct_answer}

Model Response: {response}

First, extract the model's final answer, ignoring explanations or reasoning.
Then determine if the model's answer is correct by comparing it to the correct answer.
Allow for minor variations in wording but be strict about factual accuracy.

Provide your evaluation in this format:
Extracted Answer: [The model's final answer, extracted from the full response]
Reasoning: [Your explanation for why the answer is correct or incorrect]
Correct: [yes/no]
"""

# Template for BrowseComp grading
BROWSECOMP_GRADER_TEMPLATE = """
Judge whether the following [response] to [question] is correct or not based on
the precise and unambiguous [correct_answer] below.

[question]: {question}

[response]: {response}

Your judgement must be in the format and criteria specified below:

extracted_final_answer: The final exact answer extracted from the [response].
Put the extracted answer as 'None' if there is no exact, final answer to extract
from the response.

[correct_answer]: {correct_answer}

reasoning: Explain why the extracted_final_answer is correct or incorrect based
on [correct_answer], focusing only on if there are meaningful differences between
[correct_answer] and the extracted_final_answer.

correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given
above, or is within a small margin of error for numerical problems. Answer 'no' otherwise.

confidence: The extracted confidence score between 0% and 100% from [response].
Put 100 if there is no confidence score available.
"""

# Template for formatted BrowseComp queries
BROWSECOMP_QUERY_TEMPLATE = """
{question}

Your response should be in the following format:
Explanation: {{your explanation for your final answer}}
Exact Answer: {{your succinct, final answer}}
Confidence: {{your confidence score between 0% and 100% for your answer}}
"""
