from pydantic import BaseModel, Field
from schema_agents.schema import Message
from typing import Any, Dict, List, Optional, Union
from schema_agents.role import Role

class EvalInput(BaseModel):
    """Input for evaluating scores of LLM-based system."""
    question: str = Field(description="The question that was asked.")
    reference_answer: str = Field(description="The answer that was expected.")
    llm_answer: str = Field(description="The answer that was generated by the LLM-based system.")

class EvalScores(BaseModel):
    """Scores of evaluating llm answer."""
    similarity_score: float = Field(description="Float between 0 and 5 representing the similarity score, where 5 means the same and 0 means not similar, how similar in meaning is the `llm_answer` to the `reference_answer`. It should be 0 if there is factual error detected! ")
    
def create_eval_agent():
    async def bot_answer_evaluate(req: EvalInput, role: Role) -> EvalScores:
        """Return the answer to the question."""
        response = await role.aask(req, EvalScores)
        return response
    
    eval_bot = Role(
        name="Thomas",
        profile="Evaluator",
        goal="Evaluate the performance of the LLM-based system.",
        constraints=None,
        actions=[bot_answer_evaluate],
        model="gpt-4-1106-preview"
    )
    return eval_bot

async def evaluate(question, reference_answer, llm_answer):
    eval_bot = create_eval_agent()
    eval_input = EvalInput(question=question, reference_answer=reference_answer, llm_answer=llm_answer)
    scores = await eval_bot.handle(Message(content=eval_input.json(), data=eval_input, role="User"))
    similarity_score = scores[0].data.similarity_score
    return similarity_score
    