"""
MERIT Evaluation Prompts

This module contains prompts used for metric evaluation of RAG and LLM systems.
"""

import sys
import re
import inspect

from ..core.prompts import Prompt

# Correctness Evaluation Prompt
CORRECTNESS_EVALUATION_PROMPT = Prompt("""You are an Evaulator responsible for evaluating AI systems helping to evaluate the correctness of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A reference document
2. A input
3. A reference answer (ground truth)
4. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is correct based on the reference answer and document. The evaluation should consider:
1. Factual accuracy: Does the model's answer contain factual errors compared to the reference?
2. Completeness: Does the model's answer cover all key points from the reference?
3. Relevance: Does the model's answer address the input directly?

Here is the reference document:

<document>
{document_content}
</document>

Here is the input:

<input>
{input}
</input>

Here is the reference answer (ground truth):

<reference_answer>
{reference_answer}
</reference_answer>

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "correctness": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation",
  "errors": ["List of factual errors or omissions, if any"]
}

Make sure to include commas between the fields in the JSON object.

The correctness should be a float between 0.0 (completely incorrect) and 1.0 (completely correct).
""", 
  defaults={}
)

# Relevance Evaluation Prompt
RELEVANCE_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the relevance of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A input
2. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is relevant to the input. The evaluation should consider:
1. Directness: Does the model's answer directly address the input?
2. Focus: Does the model's answer stay focused on the input without irrelevant information?
3. Appropriateness: Is the model's answer appropriate for the input?

Here is the input:

<input>
{input}
</input>

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "relevance": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation"
}

Make sure to include commas between the fields in the JSON object.

The relevance should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Faithfulness Evaluation Prompt - Identifies claims in the response
FAITHFULNESS_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the faithfulness of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with the model's answer to be evaluated.

Your task is to identify all the factual claims in the model's answer.

## What is a claim?
A claim is a statement that asserts something as fact that can be verified as true or false. Claims should be:
- Atomic: Each claim should express a single fact
- Specific: Claims should be precise and unambiguous
- Factual: Claims should assert something about the world, not opinions or subjective judgments
- Independent: Each claim should stand on its own

## Guidelines for identifying claims:
1. Break compound statements into individual atomic claims
2. Separate different facts into distinct claims
3. Do not include the same information in multiple claims
4. Exclude subjective opinions, recommendations, or hypotheticals
5. Focus on factual content that can be verified

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your analysis as a JSON object with the following structure:
{
  "claims": [
    "Claim 1 text here",
    "Claim 2 text here",
    "Claim 3 text here"
  ],
  "total_claims_count": X
}

Make sure to include commas between the fields in the JSON object.

## Examples:

Example 1:
Answer: "Einstein was born in Germany on 20th March 1879 and developed the theory of relativity."
Claims:
1. "Einstein was born in Germany"
2. "Einstein was born on 20th March 1879"
3. "Einstein developed the theory of relativity"

Example 2:
Answer: "The Great Wall of China is 13,171 miles long, was built during the Ming Dynasty, and is visible from space."
Claims:
1. "The Great Wall of China is 13,171 miles long"
2. "The Great Wall of China was built during the Ming Dynasty"
3. "The Great Wall of China is visible from space"

Remember to extract all distinct factual claims and avoid duplicates or overlapping claims.
""")

# Claim Verification Prompt - Checks if a claim can be inferred from the context
CLAIM_VERIFICATION_PROMPT = Prompt("""You are an AI assistant helping to verify claims against a reference document.

I will provide you with:
1. A reference document
2. A claim to verify

Your task is to determine if the claim can be inferred from the reference document.

Here is the reference document:

<document>
{document_content}
</document>

Here is the claim to verify:

<claim>
{claim}
</claim>

Provide your verification as a JSON object with the following structure:
{
  "supported": true/false,
  "explanation": "Detailed explanation of why the claim is or is not supported by the document"
}

Make sure to include commas between the fields in the JSON object.

Example:
For the claim "Einstein was born in Germany" with document "Albert Einstein (born 14 March 1879) was a German-born theoretical physicist", the verification would be:
{
  "supported": true,
  "explanation": "The document states Einstein was 'German-born', which supports the claim that he was born in Germany."
}
""")

# Coherence Evaluation Prompt
COHERENCE_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the coherence of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is coherent. The evaluation should consider:
1. Logical flow: Does the model's answer have a logical flow?
2. Consistency: Is the model's answer internally consistent?
3. Readability: Is the model's answer easy to read and understand?

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "coherence": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation"
}

Make sure to include commas between the fields in the JSON object.

The coherence should be a float between 0.0 (completely incoherent) and 1.0 (completely coherent).
""")

# Fluency Evaluation Prompt
FLUENCY_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the fluency of answers generated by a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. The model's answer to be evaluated

Your task is to evaluate whether the model's answer is fluent. The evaluation should consider:
1. Grammar: Is the model's answer grammatically correct?
2. Spelling: Is the model's answer free of spelling errors?
3. Natural language: Does the model's answer sound natural and fluent?

Here is the model's answer to evaluate:

<model_answer>
{model_answer}
</model_answer>

Provide your evaluation as a JSON object with the following structure:
{
  "fluency": 0.0-1.0,
  "explanation": "Detailed explanation of the evaluation",
  "errors": ["List of grammatical or spelling errors, if any"]
}

Make sure to include commas between the fields in the JSON object.

The fluency should be a float between 0.0 (completely not fluent) and 1.0 (completely fluent).
""")

# Context Precision with Reference Prompt
CONTEXT_PRECISION_WITH_REFERENCE_PROMPT = Prompt("""You are an AI assistant helping to evaluate the precision of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A user input
2. A reference answer
3. A retrieved context

Your task is to evaluate whether the retrieved context is relevant to answering the user input, based on the reference answer. The evaluation should consider:
1. Relevance: Does the retrieved context contain information needed to answer the user input?
2. Precision: Does the retrieved context contain mostly relevant information without excessive irrelevant content?

Here is the user input:

<user_input>
{user_input}
</user_input>

Here is the reference answer:

<reference_answer>
{reference_answer}
</reference_answer>

Here is the retrieved context to evaluate:

<retrieved_context>
{retrieved_context}
</retrieved_context>

Provide your evaluation as a JSON object with the following structure:
{
  "is_relevant": true/false,
  "relevance_score": 0.0-1.0,
  "explanation": "Detailed explanation of why the context is or is not relevant"
}

Make sure to include commas between the fields in the JSON object.

The relevance_score should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Context Precision without Reference Prompt
CONTEXT_PRECISION_WITHOUT_REFERENCE_PROMPT = Prompt("""You are an AI assistant helping to evaluate the precision of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A user input
2. A system response
3. A retrieved context

Your task is to evaluate whether the retrieved context is relevant to answering the user input, based on the system response. The evaluation should consider:
1. Relevance: Does the retrieved context contain information needed to generate the response?
2. Precision: Does the retrieved context contain mostly relevant information without excessive irrelevant content?

Here is the user input:

<user_input>
{user_input}
</user_input>

Here is the system response:

<system_response>
{system_response}
</system_response>

Here is the retrieved context to evaluate:

<retrieved_context>
{retrieved_context}
</retrieved_context>

Provide your evaluation as a JSON object with the following structure:
{
  "is_relevant": true/false,
  "relevance_score": 0.0-1.0,
  "explanation": "Detailed explanation of why the context is or is not relevant"
}

Make sure to include commas between the fields in the JSON object.

The relevance_score should be a float between 0.0 (completely irrelevant) and 1.0 (completely relevant).
""")

# Response Relevancy Evaluation Prompt
RESPONSE_RELEVANCY_EVALUATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the relevancy of responses to user inputs in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A user input
2. The system's response to be evaluated

Your task is to generate a set of questions that could be asked based on the system's response. These questions should reflect the content of the response.

Here is the user input:

<user_input>
{user_input}
</user_input>

Here is the system's response to evaluate:

<system_response>
{system_response}
</system_response>

Generate exactly {num_questions} questions that could be asked based on the system's response. These questions should:
1. Be directly derived from the content of the response
2. Cover the main points and information presented in the response
3. Be formulated as if someone was asking about the information in the response

Provide your output as a JSON object with the following structure:
{
  "questions": [
    "Question 1 here?",
    "Question 2 here?",
    "Question 3 here?"
  ],
  "explanation": "Brief explanation of how these questions relate to the response"
}

Make sure to include commas between the fields in the JSON object.
""")

# Context Recall Evaluation Prompt - Identifies claims in reference answer
CONTEXT_RECALL_REFERENCE_CLAIMS_PROMPT = Prompt("""You are an AI assistant helping to evaluate the recall of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with a reference answer.

Your task is to identify all the factual claims in the reference answer.

## What is a claim?
A claim is a statement that asserts something as fact that can be verified as true or false. Claims should be:
- Atomic: Each claim should express a single fact
- Specific: Claims should be precise and unambiguous
- Factual: Claims should assert something about the world, not opinions or subjective judgments
- Independent: Each claim should stand on its own

## Guidelines for identifying claims:
1. Break compound statements into individual atomic claims
2. Separate different facts into distinct claims
3. Do not include the same information in multiple claims
4. Exclude subjective opinions, recommendations, or hypotheticals
5. Focus on factual content that can be verified

Here is the reference answer to analyze:

<reference_answer>
{reference_answer}
</reference_answer>

Provide your analysis as a JSON object with the following structure:
{
  "claims": [
    "Claim 1 text here",
    "Claim 2 text here",
    "Claim 3 text here"
  ],
  "total_claims_count": X
}

Make sure to include commas between the fields in the JSON object.
""")

# Context Recall Claim Verification Prompt - Checks if a claim from reference can be attributed to retrieved contexts
CONTEXT_RECALL_CLAIM_VERIFICATION_PROMPT = Prompt("""You are an AI assistant helping to evaluate the recall of retrieved contexts in a RAG (Retrieval-Augmented Generation) system.

I will provide you with:
1. A claim from the reference answer
2. The retrieved contexts

Your task is to determine if the claim can be attributed to the retrieved contexts.

Here is the claim from the reference answer:

<claim>
{claim}
</claim>

Here are the retrieved contexts:

<retrieved_contexts>
{retrieved_contexts}
</retrieved_contexts>

Provide your verification as a JSON object with the following structure:
{
  "supported": true/false,
  "explanation": "Detailed explanation of why the claim can or cannot be attributed to the retrieved contexts"
}

Make sure to include commas between the fields in the JSON object.
""")
