"""
MERIT Evaluation Prompts

This module contains prompts used for metric evaluation of RAG and LLM systems.
"""

import sys
import re
import inspect

from ..core.prompts import Prompt

# Correctness Evaluation Prompt
CORRECTNESS_EVALUATION_PROMPT = Prompt("""You are an Evaulator responsible for evaluating AI systems helping to evaluate the correctness of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A reference document
2. A input
3. A reference answer (ground truth)
4. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is correct based on the reference answer and document. The evaluation should consider:
1. Factual accuracy: Does the model's answer contain factual errors compared to the reference?
2. Completeness: Does the model's answer cover all key points from the reference?
3. Relevance: Does the model's answer address the input directly?

Here is the reference document:

<document>
{document_content}
</document>

Here is the input:

<input>
{input}
</input>

Here is the reference answer (ground truth):

<reference_answer>
{reference_answer}
</reference_answer>

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "correctness": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation",
  "errors": ["List of factual errors or omissions, if any"]
}

Make sure to include commas between the fields in the JSON object.

The correctness should be a float between 0.0 (completely incorrect) and 1.0 (completely correct).
""", 
  defaults={}
)

# Relevance Evaluation Prompt
RELEVANCE_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the relevance of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A input
2. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is relevant to the input. The evaluation should consider:
1. Directness: Does the model's answer directly address the input?
2. Focus: Does the model's answer stay focused on the input without irrelevant information?
3. Appropriateness: Is the model's answer appropriate for the input?

Here is the input:

<input>
{input}
</input>

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "relevance": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation"
}

Make sure to include commas between the fields in the JSON object.

The relevance should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Faithfulness Evaluation Prompt
FAITHFULNESS_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the faithfulness of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A reference document
2. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is faithful to the reference document. The evaluation should consider:
1. Hallucination: Does the model's answer contain information not present in the reference document?
2. Contradiction: Does the model's answer contradict information in the reference document?
3. Attribution: Does the model's answer correctly attribute information to the reference document?

Here is the reference document:

<document>
{document_content}
</document>

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "faithfulness": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation",
  "hallucinations": ["List of hallucinated statements, if any"]
}

Make sure to include commas between the fields in the JSON object.

The faithfulness should be a float between 0.0 (completely unfaithful) and 1.0 (completely faithful).
""")

# Coherence Evaluation Prompt
COHERENCE_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the coherence of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is coherent. The evaluation should consider:
1. Logical flow: Does the model's answer have a logical flow?
2. Consistency: Is the model's answer internally consistent?
3. Readability: Is the model's answer easy to read and understand?

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "coherence": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation"
}

Make sure to include commas between the fields in the JSON object.

The coherence should be a float between 0.0 (completely incoherent) and 1.0 (completely coherent).
""")

# Fluency Evaluation Prompt
FLUENCY_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the fluency of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is fluent. The evaluation should consider:
1. Grammar: Is the model's answer grammatically correct?
2. Spelling: Is the model's answer free of spelling errors?
3. Natural language: Does the model's answer sound natural and fluent?

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "fluency": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation",
  "errors": ["List of grammatical or spelling errors, if any"]
}

Make sure to include commas between the fields in the JSON object.

The fluency should be a float between 0.0 (completely not fluent) and 1.0 (completely fluent).
""")

# Context Precision with Reference Prompt
CONTEXT_PRECISION_WITH_REFERENCE_PROMPT = Prompt("""You are an AI assistant helping to evaluate the precision of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A user input
2. A reference answer
3. A retrieved context

Your task is to evaluate whether the retrieved context is relevant to answering the user input, based on the reference answer. The evaluation should consider:
1. Relevance: Does the retrieved context contain information needed to answer the user input?
2. Precision: Does the retrieved context contain mostly relevant information without excessive irrelevant content?

Here is the user input:

<user_input>
{user_input}
</user_input>

Here is the reference answer:

<reference_answer>
{reference_answer}
</reference_answer>

Here is the retrieved context to evaluate:

<retrieved_context>
{retrieved_context}
</retrieved_context>

Provide your evaluation as a JSON object with the following structure:
{
  "is_relevant": true/false,
  "relevance_score": 0.0-1.0,
  "explanation": "Detailed explanation of why the context is or is not relevant"
}

Make sure to include commas between the fields in the JSON object.

The relevance_score should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Context Precision without Reference Prompt
CONTEXT_PRECISION_WITHOUT_REFERENCE_PROMPT = Prompt("""You are an AI assistant helping to evaluate the precision of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A user input
2. A system response
3. A retrieved context

Your task is to evaluate whether the retrieved context is relevant to answering the user input, based on the system response. The evaluation should consider:
1. Relevance: Does the retrieved context contain information needed to generate the response?
2. Precision: Does the retrieved context contain mostly relevant information without excessive irrelevant content?

Here is the user input:

<user_input>
{user_input}
</user_input>

Here is the system response:

<system_response>
{system_response}
</system_response>

Here is the retrieved context to evaluate:

<retrieved_context>
{retrieved_context}
</retrieved_context>

Provide your evaluation as a JSON object with the following structure:
{{
  "is_relevant": true/false,
  "contextprecision": 0.0-1.0,
  "explanation": "Detailed explanation of why the context is or is not relevant"
}}

Make sure to include commas between the fields in the JSON object.

The relevance_score should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Context Recall Prompt (for ContextRecallMetric)
CONTEXT_RECALL_PROMPT = Prompt("""
You are an expert evaluator assessing the recall of retrieved context for a given user query and a model's answer.
Your goal is to determine if the provided 'retrieved_context' contains all the necessary information that *should have been* there to adequately answer the 'user_input' and support the 'model_answer'.

User Input:
<user_input>
{user_input}
</user_input>

Model Answer:
<model_answer>
{model_answer}
</model_answer>

Retrieved Context:
<retrieved_context>
{retrieved_context}
</retrieved_context>

Evaluation Criteria:
- Score 1: The context comprehensively contains all information necessary to fully answer the user input AND fully support all claims made in the model answer.
- Score 0.1-0.9: The context is partially missing relevant information. It might support some parts of the answer but not others, or it might help answer the query but incompletely.
- Score 0: The context is completely irrelevant or missing almost all crucial information needed to answer the query or support the model answer.

Provide your evaluation as a JSON object with two keys: "score" (a float between 0.0 and 1.0) and "explanation" (a brief justification for your score, explaining what information is present or missing in the context).
Example JSON:
{
  "score": 0.7,
  "explanation": "The context supports the main claim in the answer but lacks specific details mentioned in the query regarding historical data."
}
""")

# Claim Extraction Prompt (for FaithfulnessMetric)
CLAIM_EXTRACTION_PROMPT = Prompt("""
You are an expert in analyzing text. Your task is to identify and extract individual factual claims from the given 'Model Answer'.
A claim is a statement that asserts something as a fact, which could be independently verified or refuted.
Present each claim as a separate item in a JSON list.

Model Answer:
<model_answer>
{model_answer}
</model_answer>

Example Output Format:
{
  "claims": [
    "Claim 1 statement.",
    "Claim 2 statement.",
    "Claim 3 statement."
  ]
}

If there are no factual claims, return an empty list: {"claims": []}.
""")

# Claim Verification Prompt (for FaithfulnessMetric)
CLAIM_VERIFICATION_PROMPT = Prompt("""
You are an expert evaluator. Your task is to determine if a given 'Claim' can be factually supported by the provided 'Retrieved Context'.
Do not make assumptions or use external knowledge. Base your decision solely on the information present in the 'Retrieved Context'.

Retrieved Context:
<retrieved_context>
{retrieved_context}
</retrieved_context>

Claim to Verify:
<claim>
{claim}
</claim>

Provide your evaluation as a JSON object with two keys:
1. "is_supported": boolean (true if the claim is supported by the context, false otherwise).
2. "explanation": string (a brief justification for your decision, referencing parts of the context if possible, or explaining why it's not supported).

Example Output Format:
{
  "is_supported": true,
  "explanation": "The context states 'X is Y', which directly supports the claim."
}
""")

# Response Question Generation Prompt (for RelevanceMetric)
RESPONSE_QUESTION_GENERATION_PROMPT = Prompt("""
You are an expert in question generation. Given the 'Model Answer' below, generate {num_questions} distinct questions that this answer could be a response to.
Focus on creating questions that the provided answer directly and comprehensively addresses.

Model Answer:
<model_answer>
{model_answer}
</model_answer>

Return your output as a JSON object with a single key "generated_questions", which is a list of strings (the questions).
Example Output Format (for num_questions=2):
{
  "generated_questions": [
    "What is question 1?",
    "Can you explain question 2?"
  ]
}
""")

