"""Evaluators to optimize task: foo.

THIS IS A TEMPLATE FOR YOU TO CHANGE!

Evaluators compute scores for prompts run over the configured dataset:
https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/db688c10-764b-42ec-acce-4d62419600ed
"""
from typing_extensions import TypedDict
from langsmith.schemas import Run, Example

# Modify these evaluators to measure the requested criteria.
# For most prompt optimization tasks, run.outputs["output"] will contain an AIMessage
# (Advanced usage) If you are defining a custom system to optimize, then the outputs will contain the object returned by your system


class Outputs(TypedDict):
    tweet: str


def example_evaluator(run: Run, example: Example) -> dict:
    """An example evaluator. Larger numbers are better."""
    # The Example object contains the inputs and reference labels from a single row in your dataset (if provided).

    # We've copied the inputs & outputs for the first example in the configured dataset.
    prompt_inputs = example.inputs
    # {
    #   "topic": "Gerrymandering"
    # }
    reference_outputs = example.outputs  # aka labels
    # None
    # The comments above autogenerated from example:
    # https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/db688c10-764b-42ec-acce-4d62419600ed/e/0043effe-b5f1-48cc-ad28-739f3117ce02

    # The Run object contains the full trace of your system. Typically you run checks on the outputs,
    # often comparing them to the reference_outputs
    outputs: Outputs = run.outputs

    # Implement your evaluation logic here
    score = all(
        key in outputs for key in ["tweet"]
    )  # Replace with the actual score calculation
    return {
        # The evaluator keys here define the metric you are measuring
        # You can provide further descriptions for these in the config.json
        "key": "my_example_criterion",
        "score": score,
        "comment": (
            "CHANGEME: It's good practice to return "
            "information that can help the metaprompter fix mistakes, "
            "such as Pass/Fail or expected X but got Y, etc. "
            "This comment instructs the LLM how to improve."
            "The placeholder metric checks that the content is less than 180 in length."
        ),
    }


evaluators = [example_evaluator]
