import asyncio
import copy
import dataclasses
import difflib
from asyncio import CancelledError
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    DefaultDict,
    Dict,
    List,
    Optional,
    Text,
    Tuple,
    Union,
)
from urllib.parse import urlparse

import requests
import structlog
from tqdm import tqdm

import rasa.shared.utils.io
from rasa.core.channels import CollectingOutputChannel, UserMessage
from rasa.core.config.available_endpoints import AvailableEndpoints
from rasa.core.config.configuration import Configuration
from rasa.core.constants import (
    ACTIVE_FLOW_METADATA_KEY,
    PARENT_FLOW_ID_METADATA_KEY,
    PARENT_STEP_ID_METADATA_KEY,
    STEP_ID_METADATA_KEY,
)
from rasa.core.exceptions import AgentNotReady
from rasa.core.persistor import StorageType
from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
from rasa.e2e_test.constants import TEST_CASE_NAME, TEST_FILE_NAME
from rasa.e2e_test.e2e_config import create_llm_judge_config
from rasa.e2e_test.e2e_test_case import (
    KEY_STUB_CUSTOM_ACTIONS,
    ActualStepOutput,
    Fixture,
    Metadata,
    TestCase,
    TestStep,
)
from rasa.e2e_test.e2e_test_result import (
    NO_RESPONSE,
    NO_SLOT,
    TestFailure,
    TestResult,
)
from rasa.exceptions import ValidationError
from rasa.llm_fine_tuning.conversations import Conversation
from rasa.shared.agents.agent_setup import AgentsConnectionCleanup
from rasa.shared.constants import RASA_DEFAULT_FLOW_PATTERN_PREFIX
from rasa.shared.core.constants import MOCKED_DATETIME_SLOT
from rasa.shared.core.events import (
    ActionExecuted,
    BotUttered,
    Event,
    FlowCompleted,
    FlowStarted,
    SlotSet,
    UserUttered,
)
from rasa.shared.core.flows.flow_path import FlowPath, PathNode
from rasa.shared.core.trackers import DialogueStateTracker
from rasa.shared.exceptions import RasaException
from rasa.shared.nlu.constants import COMMANDS
from rasa.telemetry import track_e2e_test_run
from rasa.utils.endpoints import EndpointConfig

if TYPE_CHECKING:
    from rasa.core.agent import Agent
    from rasa.shared.core.flows import FlowsList

structlogger = structlog.get_logger()

TEST_TURNS_TYPE = Dict[int, Union[TestStep, ActualStepOutput]]


class E2ETestRunner:
    def __init__(
        self,
        model_path: Optional[Text] = None,
        model_server: Optional[EndpointConfig] = None,
        remote_storage: Optional[StorageType] = None,
        endpoints: Optional[AvailableEndpoints] = None,
        sub_agents_path: Optional[Text] = None,
        **kwargs: Any,
    ) -> None:
        """Initializes the E2E test suite runner.

        Args:
            model_path: Path to the model.
            model_server: Model server configuration.
            remote_storage: Remote storage to use for model retrieval.
            endpoints: Endpoints configuration.
            **kwargs: Additional arguments
        """
        import rasa.core.agent

        structlogger.info(
            "e2e_test_runner.init",
            event_info="Started running end-to-end testing.",
        )

        test_case_path = kwargs.get("test_case_path")
        self.llm_judge_config = create_llm_judge_config(test_case_path)

        are_custom_actions_stubbed = (
            endpoints
            and endpoints.action
            and endpoints.action.kwargs.get(KEY_STUB_CUSTOM_ACTIONS)
        )
        if endpoints and not are_custom_actions_stubbed:
            self._action_server_is_reachable(endpoints)

        sub_agents = Configuration.get_instance().available_agents

        async def load_agent_with_agents_connection_cleanup() -> "Agent":
            # Create a wrapper coroutine that handles graceful agents connection
            # cleanup, this prevents abrupt closure when asyncio.run finishes.
            async with AgentsConnectionCleanup():
                return await rasa.core.agent.load_agent(
                    model_path=model_path,
                    model_server=model_server,
                    remote_storage=remote_storage,
                    endpoints=endpoints,
                    sub_agents=sub_agents,
                )
            # Defensive return for mypy - never actually reached
            return

        self.agent = asyncio.run(load_agent_with_agents_connection_cleanup())

        if not self.agent.is_ready():
            raise AgentNotReady(
                "Agent needs to be prepared before usage. "
                "Please check that the agent was able to "
                "load the trained model."
            )

    async def run_prediction_loop(
        self,
        collector: CollectingOutputChannel,
        steps: List[TestStep],
        sender_id: Text,
        test_case_metadata: Optional[Metadata] = None,
        input_metadata: Optional[List[Metadata]] = None,
    ) -> TEST_TURNS_TYPE:
        """Runs dialogue prediction.

        Args:
            collector: Output channel.
            steps: List of steps to run.
            sender_id: The test case name with added timestamp suffix.
            test_case_metadata: Metadata of the test case.
            input_metadata: List of metadata.

        Returns:
            Test turns: {turn_sequence (int): TestStep or ActualStepOutput}.
        """
        turns: TEST_TURNS_TYPE = {}
        event_cursor = 0

        if not self.agent.processor:
            return turns

        tracker = await self.agent.processor.fetch_tracker_with_initial_session(
            sender_id, output_channel=collector
        )
        # turn -1 i used to contain events that happen during
        # the start of the session and before the first user message
        # TestStep is a placeholder just for the sake of having a turn
        # to specify the actor
        turns[-1], event_cursor = self.get_actual_step_output(
            tracker,
            TestStep(
                actor="bot",
                text=None,
            ),
            event_cursor,
        )

        # This variable tracks the position of the last user step.
        # We use it to determine from which turn onward we should check
        # for failures in case of the fail-fast logic.
        last_user_step_position = 0
        for position, step in enumerate(steps):
            if step.actor != "user":
                turns[position] = step

                # Check for failures after each bot step
                try:
                    self.fail_fast_check(
                        test_turns=turns,
                        last_user_step_position=last_user_step_position,
                    )
                except RasaException:
                    break

                continue
            elif not step.text:
                rasa.shared.utils.io.raise_warning(
                    f"The test case '{sender_id}' contains a `user` step in line "
                    f"{position + 1} without a text value. "
                    f"Skipping this step and proceeding to the next user step.",
                    UserWarning,
                )
                continue
            # Update to the current position because we have a valid user step
            last_user_step_position = position

            metadata = test_case_metadata.metadata if test_case_metadata else {}

            if input_metadata:
                step_metadata = self.filter_metadata_for_input(
                    step.metadata_name, input_metadata
                )
                step_metadata_dict = step_metadata.metadata if step_metadata else {}
                metadata = self.merge_metadata(
                    sender_id, step.text, metadata, step_metadata_dict
                )

            try:
                await self.agent.handle_message(
                    UserMessage(
                        step.text,
                        collector,
                        sender_id,
                        metadata=metadata,
                    )
                )
            except CancelledError:
                structlogger.error(
                    "e2e_test_runner.run_prediction_loop",
                    error=f"Message handling timed out for user message '{step.text}'.",
                    exc_info=True,
                )
            except Exception as exc:
                structlogger.error(
                    "e2e_test_runner.run_prediction_loop",
                    error=f"An exception occurred while handling "
                    f"user message '{step.text}'. Error: {exc}",
                )
            tracker = await self.agent.tracker_store.retrieve(sender_id)  # type: ignore[assignment]
            turns[position], event_cursor = self.get_actual_step_output(
                tracker, step, event_cursor
            )
        return turns

    def fail_fast_check(
        self, test_turns: TEST_TURNS_TYPE, last_user_step_position: int
    ) -> None:
        """Checks whether there are any test failures in 'test_turns'.

        Args:
            test_turns (TEST_TURNS_TYPE): The transcript of test cases and events.
            last_user_step_position (int): The start position in the test turns.

        Raises:
            RasaException if a failure is found.
        """
        failures_found = self.find_test_failures(
            test_turns=test_turns,
            test_case=None,
            last_user_step_position=last_user_step_position,
        )
        if failures_found:
            raise RasaException("Test failure found. Aborting early.")

    @staticmethod
    def merge_metadata(
        sender_id: Text,
        step_text: Text,
        test_case_metadata: Dict[Text, Text],
        step_metadata: Dict[Text, Text],
    ) -> Dict[Text, Text]:
        """Merges the test case and user step metadata.

        Args:
            sender_id: The test case name with added timestamp suffix.
            step_text: The user step text.
            test_case_metadata: The test case metadata dict.
            step_metadata: The user step metadata dict.

        Returns:
            A dictionary with the merged metadata.
        """
        if not test_case_metadata:
            return step_metadata
        if not step_metadata:
            return test_case_metadata

        keys_to_overwrite = []

        for key in step_metadata.keys():
            if key in test_case_metadata.keys():
                keys_to_overwrite.append(key)

        if keys_to_overwrite:
            test_case_name = sender_id.rsplit("_", 1)[0]
            structlogger.warning(
                "e2e_test_runner.merge_metadata",
                message=f"Metadata {keys_to_overwrite} exist in both the test case "
                f"'{test_case_name}' and the user step '{step_text}'. "
                "The user step metadata takes precedence and will "
                "override the test case metadata.",
            )

        merged_metadata = copy.deepcopy(test_case_metadata)
        merged_metadata.update(step_metadata)

        return merged_metadata

    @staticmethod
    def get_actual_step_output(
        tracker: DialogueStateTracker,
        test_step: TestStep,
        event_cursor: int,
    ) -> Tuple[ActualStepOutput, int]:
        """Returns the events that are generated from the current step.

        Args:
            tracker: The tracker for the current test case.
            test_step: The test step.
            event_cursor: The event cursor where the previous step left off.

        Returns:
        The events generated from the current step and the updated event
        cursor position.
        """
        session_events = list(tracker.events)[event_cursor:]
        if session_events:
            event_cursor += len(session_events)
            return (
                ActualStepOutput.from_test_step(
                    test_step,
                    [
                        event
                        for event in session_events
                        if isinstance(event, (BotUttered, UserUttered, SlotSet))
                    ],
                ),
                event_cursor,
            )
        else:
            structlogger.warning(
                "e2e_test_runner.get_actual_step_output",
                message=f"No events found for '{tracker.sender_id}' after processing "
                f"test step '{test_step.text}'.",
            )
            # if there are no events, we still want to return an
            # ActualStepOutput object with the test step as the
            # response to be able to generate a diff
            return (
                ActualStepOutput.from_test_step(
                    test_step,
                    [
                        UserUttered(text=test_step.text),
                        BotUttered(text=NO_RESPONSE),
                    ],
                ),
                event_cursor,
            )

    @classmethod
    def generate_test_result(
        cls,
        test_turns: TEST_TURNS_TYPE,
        test_case: TestCase,
    ) -> TestResult:
        """Generates test result.

        Args:
            test_turns: the turns that happened when running the test case or test step.
            test_case: the `TestCase` instance.

        Returns:
        Test result.
        """
        difference = []
        error_line = None
        test_failures = cls.find_test_failures(test_turns, test_case)
        if test_failures:
            first_failure = test_failures[0][0]
            difference = cls.human_readable_diff(test_turns, test_failures)
            error_line = first_failure.error_line if first_failure else None

        return TestResult(
            pass_status=len(test_failures) == 0,
            test_case=test_case,
            difference=difference,
            error_line=error_line,
        )

    def _get_additional_splitting_conditions(
        self,
        step: TestStep,
        input_metadata: List[Metadata],
        tracker: DialogueStateTracker,
        test_case: TestCase,
    ) -> Dict[str, Any]:
        """Returns additional splitting conditions for the user message."""
        additional_splitting_conditions: Dict[str, Any] = {"text": step.text}

        if not step.metadata_name:
            return additional_splitting_conditions

        step_metadata = self.filter_metadata_for_input(
            step.metadata_name, input_metadata
        )
        step_metadata_dict = step_metadata.metadata if step_metadata else {}

        test_case_metadata = self.filter_metadata_for_input(
            test_case.metadata_name, input_metadata
        )
        test_case_metadata_as_dict = (
            test_case_metadata.metadata if test_case_metadata else {}
        )

        metadata: Dict[str, Any] = self.merge_metadata(
            tracker.sender_id,
            step.text,
            test_case_metadata_as_dict,
            step_metadata_dict,
        )
        metadata["model_id"] = tracker.model_id
        metadata["assistant_id"] = tracker.assistant_id

        additional_splitting_conditions["metadata"] = metadata

        return additional_splitting_conditions

    @staticmethod
    def _get_current_user_turn_and_prior_events(
        tracker: DialogueStateTracker,
        additional_splitting_conditions: Dict[str, Any],
        step: TestStep,
    ) -> Tuple[List[Event], List[Event]]:
        """Returns the current user turn and prior events."""
        actual_events = tracker.events

        # this returns 2 lists, the first list contains the events until the user
        # message and the second list contains the events after the
        # user message, including the user message
        step_events = rasa.shared.core.events.split_events(
            actual_events,
            UserUttered,
            additional_splitting_conditions=additional_splitting_conditions,
            include_splitting_event=True,
        )

        if len(step_events) < 2:
            structlogger.error(
                "e2e_test_runner.run_assertions.user_message_not_found",
                message=f"User message '{step.text}' was not found in "
                f"the actual events. The user message "
                f"properties which were searched: "
                f"{additional_splitting_conditions}",
            )
            return [], []

        post_step_events = step_events[1]
        prior_events = step_events[0]

        # subset of events until the next user message
        turn_events = []
        for event in post_step_events:
            # we reached the next user message
            if isinstance(event, UserUttered) and step.text != event.text:
                break

            turn_events.append(event)

        return turn_events, prior_events

    @staticmethod
    def _slice_turn_events(
        step: TestStep,
        matching_event: Event,
        turn_events: List[Event],
        prior_events: List[Event],
    ) -> Tuple[List[Event], List[Event]]:
        """Slices the turn events when assertion order is enabled."""
        if not step.assertion_order_enabled:
            return turn_events, prior_events

        if not matching_event:
            return turn_events, prior_events

        matching_event_index = turn_events.index(matching_event)
        if matching_event_index + 1 < len(turn_events):
            prior_events += turn_events[: matching_event_index + 1]
            turn_events = turn_events[matching_event_index + 1 :]

        return turn_events, prior_events

    async def run_assertions(
        self,
        sender_id: str,
        test_case: TestCase,
        input_metadata: Optional[List[Metadata]],
    ) -> TestResult:
        """Runs the assertions defined in the test case."""
        tracker = await self.agent.processor.get_tracker(sender_id)  # type: ignore[union-attr]

        assertion_failure = None
        assertion_failure_found = False
        input_metadata = input_metadata if input_metadata else []

        for index, step in enumerate(test_case.steps):
            if not step.assertions:
                structlogger.debug(
                    "e2e_test_runner.run_assertions.no_assertions.skipping_step",
                    step=step,
                )
                continue

            additional_splitting_conditions = self._get_additional_splitting_conditions(
                step, input_metadata, tracker, test_case
            )

            turn_events, prior_events = self._get_current_user_turn_and_prior_events(
                tracker, additional_splitting_conditions, step
            )

            if not turn_events:
                return TestResult(
                    pass_status=False,
                    test_case=test_case,
                    difference=[],
                    error_line=step.line,
                    assertion_failure=None,
                )

            for assertion in step.assertions:
                structlogger.debug(
                    "e2e_test_runner.run_assertions.running_assertion",
                    test_case_name=test_case.name,
                    step_text=step.text,
                    assertion_type=assertion.type(),
                )

                assertion_order_error_msg = ""

                if step.assertion_order_enabled:
                    assertion_order_error_msg = (
                        " You have enabled assertion order, "
                        "you should check the order in which the "
                        "assertions are listed for this user step."
                    )

                assertion_failure, matching_event = assertion.run(
                    turn_events,
                    prior_events=prior_events,
                    assertion_order_error_message=assertion_order_error_msg,
                    llm_judge_config=self.llm_judge_config,
                    step_text=step.text,
                    step_index=index,
                )

                if assertion_failure:
                    assertion_failure_found = True
                    structlogger.debug(
                        "e2e_test_runner.run_assertions.assertion_failure_found",
                        test_case_name=test_case.name,
                        error_line=assertion_failure.error_line,
                    )
                    break

                turn_events, prior_events = self._slice_turn_events(
                    step, matching_event, turn_events, copy.deepcopy(prior_events)
                )

            if assertion_failure_found:
                # don't continue with the next steps if an assertion failed
                break

        return TestResult(
            pass_status=not assertion_failure,
            test_case=test_case,
            difference=[],
            error_line=assertion_failure.error_line if assertion_failure else None,
            assertion_failure=assertion_failure,
        )

    @classmethod
    def _resolve_successful_bot_utter(
        cls,
        expected_result: TestStep,
        test_response: ActualStepOutput,
    ) -> str:
        """Returns the diff text for a successful bot utter test step."""
        for event in test_response.bot_uttered_events:
            if expected_result.matches_event(event):
                # remove the event that is already matched so
                # that we dont compare against it again
                test_response.remove_bot_uttered_event(event)
                if expected_result.text is not None:
                    text = f"{expected_result.actor}: {expected_result.text}"
                elif expected_result.template is not None:
                    text = f"{expected_result.actor}: {expected_result.template}"
                break

        return text

    @classmethod
    def _resolve_successful_set_slot(
        cls,
        expected_result: TestStep,
        test_response: ActualStepOutput,
    ) -> str:
        """Returns the diff text for a successful set slot test step."""
        slot_name = expected_result.get_slot_name()
        text = f"slot_was_set: {slot_name}"

        for event in test_response.slot_set_events:
            if expected_result.matches_event(event):
                # remove the event that is already matched so
                # that we dont compare against it again
                test_response.remove_slot_set_event(event)
                if event.value is not None:
                    text += f": {event.value} ({type(event.value).__name__})"
                break

        return text

    @classmethod
    def _resolve_successful_slot_was_not_set(cls, expected_result: TestStep) -> str:
        """Returns the diff text for a successful slot was not set test step."""
        slot_name = expected_result.get_slot_name()
        slot_value = expected_result.get_slot_value()
        text = f"slot_was_not_set: {slot_name}"
        if expected_result.is_slot_instance_dict():
            text += f": {slot_value}"

        return text

    @classmethod
    def _find_first_non_matched_utterance(
        cls,
        test_response: ActualStepOutput,
        bot_utter_test_steps: List[TestStep],
    ) -> Optional[BotUttered]:
        """Finds the first non matched utterance in events (if any)."""
        bot_events = test_response.bot_uttered_events
        matching_events = []
        for event in bot_events:
            for test_step in bot_utter_test_steps:
                if test_step.matches_event(event):
                    matching_events.append(event)
                    break
        not_matching_events = [
            event for event in bot_events if event not in matching_events
        ]
        return not_matching_events[0] if not_matching_events else None

    @classmethod
    def _handle_fail_diff(
        cls,
        failed_step: TestStep,
        test_response: ActualStepOutput,
        bot_utter_test_steps: List[TestStep],
    ) -> Tuple[str, str]:
        """Handles generating the diff text for a failed test step."""
        diff_test_text = ""
        diff_actual_text = ""

        if failed_step.text is not None:
            diff_test_text = f"{failed_step.actor}: {failed_step.text}"
            diff_actual_text = NO_RESPONSE
            event: Optional[Union[BotUttered, SlotSet]] = (
                cls._find_first_non_matched_utterance(
                    test_response, bot_utter_test_steps
                )
            )
            if event and isinstance(event, BotUttered):
                test_response.remove_bot_uttered_event(event)
                diff_actual_text = f"bot: {event.text}"

        elif failed_step.template is not None:
            diff_test_text = f"{failed_step.actor}: {failed_step.template}"
            diff_actual_text = NO_RESPONSE
            event = cls._find_first_non_matched_utterance(
                test_response, bot_utter_test_steps
            )
            if event:
                test_response.remove_bot_uttered_event(event)
                diff_actual_text = f"bot: {event.metadata.get('utter_action')}"

        elif failed_step.slot_was_set:
            slot_name = failed_step.get_slot_name()

            diff_test_text = f"slot_was_set: {slot_name}"
            if failed_step.is_slot_instance_dict():
                slot_value = failed_step.get_slot_value()
                diff_test_text += f": {slot_value} ({type(slot_value).__name__})"

            diff_actual_text = NO_SLOT
            for event in test_response.slot_set_events:
                if slot_name == event.key:
                    diff_actual_text = (
                        f"slot_was_set: {event.key}: {event.value} "
                        f"({type(event.value).__name__})"
                    )
                    test_response.remove_slot_set_event(event)
                    break

        elif failed_step.slot_was_not_set:
            slot_name = failed_step.get_slot_name()

            diff_test_text = f"slot_was_not_set: {slot_name}"
            if failed_step.is_slot_instance_dict():
                slot_value = failed_step.get_slot_value()
                diff_test_text += f": {slot_value}"

            diff_actual_text = NO_SLOT
            for event in test_response.slot_set_events:
                if slot_name == event.key:
                    diff_actual_text = (
                        f"slot_was_set: {event.key}: {event.value} "
                        f"({type(event.value).__name__})"
                    )
                    test_response.remove_slot_set_event(event)
                    break

        return diff_test_text, diff_actual_text

    @classmethod
    def _select_bot_utter_turns(
        cls,
        test_turns: TEST_TURNS_TYPE,
        start_index: int,
    ) -> List[TestStep]:
        """Selects the TestSteps from the test turns that match BotUttered events."""
        bot_utter_turns = []
        for index in range(start_index, len(test_turns) - 1):
            test_turn = test_turns[index]
            if isinstance(test_turn, TestStep):
                if test_turn.text is not None or test_turn.template is not None:
                    bot_utter_turns.append(test_turn)
            elif isinstance(test_turn, ActualStepOutput):
                break

        return bot_utter_turns

    @classmethod
    def human_readable_diff(
        cls,
        test_turns: TEST_TURNS_TYPE,
        fail_positions: List[Tuple[TestFailure, int]],
    ) -> List[str]:
        """Returns a human readable diff of the test case and the actual conversation.

        Given an ordered list of test steps and actual conversation events, this
        method will return a human readable diff of the two.
        The diff uses difflib to compare the two lists and will highlight
        differences between the two in a pytest style diff.

        Args:
            test_turns: The transcript of test cases and events.
            fail_positions: The positions of the test failures.

        Returns:
            The human readable diff.
        """
        actual_transcript = []
        expected_transcript = []
        failure_points = {position for _, position in fail_positions}
        # This will only be used when the TestCase is not started
        # with a user step
        latest_response: ActualStepOutput = test_turns[-1]  # type: ignore[assignment]

        for index in range(max(failure_points) + 1):
            test_result = test_turns[index]
            if index in failure_points:
                diff_test_text, diff_actual_text = cls._handle_fail_diff(
                    test_result,
                    latest_response,
                    cls._select_bot_utter_turns(test_turns, index),
                )  # test_result can only be TestStep in failure_points
                actual_transcript.append(diff_actual_text)
                expected_transcript.append(diff_test_text)
                continue

            if isinstance(test_result, TestStep):
                if test_result.text is not None or test_result.template is not None:
                    diff_test_text = cls._resolve_successful_bot_utter(
                        expected_result=test_result,
                        test_response=latest_response,
                    )

                if test_result.slot_was_set:
                    diff_test_text = cls._resolve_successful_set_slot(
                        expected_result=test_result,
                        test_response=latest_response,
                    )

                if test_result.slot_was_not_set:
                    diff_test_text = cls._resolve_successful_slot_was_not_set(
                        expected_result=test_result,
                    )

            elif isinstance(test_result, ActualStepOutput):
                latest_response = test_result
                event = test_result.get_user_uttered_event()
                if event:
                    diff_test_text = f"{test_result.actor}: {event.text}"
                else:
                    raise RasaException(
                        f"Bot did not catch user "
                        f"event: {test_result.actor}: {test_result.text}."
                    )
            else:
                raise ValueError(f"Unexpected test result type: {type(test_result)}")

            # test passed in these cases so it's the same
            actual_transcript.append(diff_test_text)
            expected_transcript.append(diff_test_text)

        return list(difflib.ndiff(actual_transcript, expected_transcript))

    @classmethod
    def find_test_failures(
        cls,
        test_turns: TEST_TURNS_TYPE,
        test_case: Optional[TestCase],
        last_user_step_position: int = 0,
    ) -> List[Tuple[TestFailure, int]]:
        """Finds the test failures in the transcript.

        Args:
            test_turns (TEST_TURNS_TYPE): The transcript of test cases and events.
            test_case (Optional[TestCase]): The test case.
            last_user_step_position (int): The start position in the test turns.

        Returns:
            The test failures or an empty list if there is no test failure.
        """
        # This will only be used when the TestCase is not started
        # with a user step
        latest_response_event = test_turns[-1]
        if not isinstance(latest_response_event, ActualStepOutput):
            raise ValueError(
                f"Expected ActualStepOutput, got {type(latest_response_event)}"
            )
        latest_response: ActualStepOutput = latest_response_event
        failures = []
        match = None
        for position in range(last_user_step_position, len(test_turns) - 1):
            if position not in test_turns:
                continue
            turn_value = test_turns[position]
            if isinstance(turn_value, ActualStepOutput):
                latest_response = turn_value
            elif isinstance(turn_value, TestStep):
                if turn_value.text is not None or turn_value.template is not None:
                    match = cls._does_match_exist(
                        latest_response.bot_uttered_events, turn_value
                    )
                if turn_value.slot_was_set:
                    match = cls._does_match_exist(
                        latest_response.slot_set_events, turn_value
                    )
                if turn_value.slot_was_not_set:
                    match = not cls._does_match_exist(
                        latest_response.slot_set_events, turn_value
                    )
                if not match:
                    failures.append((TestFailure(test_case, turn_value.line), position))
            else:
                raise ValueError(f"Unexpected turn value type: {type(turn_value)}")

        return failures

    @classmethod
    def _does_match_exist(
        cls,
        actual_events: Optional[List[Union[UserUttered, BotUttered, SlotSet]]],
        expected: TestStep,
    ) -> bool:
        if not actual_events:
            return False

        match = False
        for event in actual_events:
            if expected.matches_event(event):
                return True
        return match

    async def set_up_fixtures(
        self,
        fixtures: List[Fixture],
        sender_id: Text,
    ) -> None:
        """Sets slots in the tracker as defined by the input fixtures.

        Args:
            fixtures: List of `Fixture` objects.
            sender_id: The conversation id.
        """
        if not fixtures:
            return
        if not self.agent.processor:
            return

        tracker = await self.agent.processor.fetch_tracker_with_initial_session(
            sender_id, output_channel=CollectingOutputChannel()
        )

        for fixture in fixtures:
            for slot_name, slot_value in fixture.slots_set.items():
                tracker.update(SlotSet(slot_name, slot_value))

        await self.agent.tracker_store.save(tracker)

    @staticmethod
    def filter_fixtures_for_test_case(
        test_case: TestCase, fixtures: List[Fixture]
    ) -> List[Fixture]:
        """Filters the input fixtures for the input test case.

        Args:
            test_case: The test case.
            fixtures: The fixtures.

        Returns:
        The filtered fixtures.
        """
        return list(
            filter(
                lambda fixture: test_case.fixture_names
                and fixture.name in test_case.fixture_names,
                fixtures,
            )
        )

    @staticmethod
    def filter_metadata_for_input(
        metadata_name: Optional[Text], test_suite_metadata: List[Metadata]
    ) -> Optional[Metadata]:
        """Filters the test suite metadata for a metadata name.

        Args:
            metadata_name: The test case or user step metadata name.
            test_suite_metadata: The top level list of all metadata definitions.

        Returns:
            The filtered metadata.
        """
        if not metadata_name:
            return None

        filtered_metadata = list(
            filter(
                lambda metadata: metadata_name and metadata.name == metadata_name,
                test_suite_metadata,
            )
        )

        if not filtered_metadata:
            structlogger.warning(
                "e2e_test_runner.filter_metadata_for_input",
                message=f"Metadata '{metadata_name}' is not defined in the input "
                f"metadata.",
            )
            return None

        return filtered_metadata[0]

    async def run_tests(
        self,
        input_test_cases: List[TestCase],
        input_fixtures: List[Fixture],
        fail_fast: bool = False,
        **kwargs: Any,
    ) -> List["TestResult"]:
        """Runs the test cases.

        Args:
            input_test_cases: Input test cases.
            input_fixtures: Input fixtures.
            fail_fast: Whether to fail fast.
            **kwargs: Additional arguments which are passed here.

        Returns:
            List of test results.
        """
        results = []
        input_metadata = kwargs.get("input_metadata", None)

        # telemetry call for tracking test runs
        track_e2e_test_run(input_test_cases, input_fixtures, input_metadata)

        # Validate and convert all fixtures upfront before running any tests
        # This ensures we fail fast on validation errors rather than running
        # some tests before hitting an invalid fixture, and stores converted
        # values in fixtures to avoid repeated validation/conversion
        input_fixtures = self._validate_and_convert_all_fixtures(input_fixtures)

        for test_case in input_test_cases:
            test_case_name = test_case.name.replace(" ", "_")
            # Add the name of the file and the current test case name being
            # executed in order to properly retrieve stub custom action
            if self.agent.endpoints and self.agent.endpoints.action:
                self.agent.endpoints.action.kwargs[TEST_FILE_NAME] = Path(
                    test_case.file
                ).name
                self.agent.endpoints.action.kwargs[TEST_CASE_NAME] = test_case_name

            sender_id = self.generate_sender_id(test_case.name)
            _, test_result = await self._process_test_case(
                test_case, sender_id, input_fixtures, input_metadata
            )

            results.append(test_result)

            coverage = kwargs.get("coverage", False)
            if coverage:
                tracker = await self.agent.tracker_store.retrieve(sender_id)
                if tracker:
                    # Get flows for slot-to-step mapping (to track collect steps)
                    flows = None
                    if self.agent.processor:
                        flows = await self.agent.processor.get_flows()
                    test_result.tested_paths, test_result.tested_commands = (
                        self._get_tested_flow_paths_and_commands(
                            tracker.events, test_result, flows
                        )
                    )

            if fail_fast and not test_result.pass_status:
                break

        return results

    async def _run_test_case(
        self,
        sender_id: str,
        input_fixtures: List[Fixture],
        input_metadata: Optional[List[Metadata]],
        test_case: TestCase,
    ) -> TEST_TURNS_TYPE:
        collector = CollectingOutputChannel()

        if input_fixtures:
            test_fixtures = self.filter_fixtures_for_test_case(
                test_case, input_fixtures
            )
            await self.set_up_fixtures(test_fixtures, sender_id)

        test_case_metadata = None
        if input_metadata:
            test_case_metadata = self.filter_metadata_for_input(
                test_case.metadata_name, input_metadata
            )

        return await self.run_prediction_loop(
            collector,
            test_case.steps,
            sender_id,
            test_case_metadata,
            input_metadata,
        )

    @staticmethod
    def generate_sender_id(test_case_name: str) -> str:
        # add timestamp suffix to ensure sender_id is unique
        return f"{test_case_name}_{datetime.now()}"

    def _validate_and_convert_all_fixtures(
        self, fixtures: List[Fixture]
    ) -> List[Fixture]:
        """Validate and convert all fixtures upfront before running any tests.

        This ensures we fail fast on validation errors (e.g., invalid mocked_datetime)
        rather than running some tests before hitting an invalid fixture.
        Collects all validation errors and reports them in a single ValidationError.
        Also converts mocked_datetime values to ISO format and stores them in fixtures
        to avoid repeated validation/conversion.

        Args:
            fixtures: List of all fixtures to validate and convert.

        Returns:
            List of fixtures with converted mocked_datetime values.

        Raises:
            ValidationError: If any fixture has invalid mocked_datetime values.
                The error message includes all invalid fixtures.
        """
        validation_errors: List[
            Tuple[str, str]
        ] = []  # List of (fixture_name, mocked_datetime_value)
        converted_fixtures = []

        # Single pass: validate and convert fixtures
        for fixture in fixtures:
            converted_slots = fixture.slots_set.copy()
            for slot_name, slot_value in fixture.slots_set.items():
                if slot_name == MOCKED_DATETIME_SLOT:
                    try:
                        # Validate and convert to ISO format
                        converted_value = self._get_validated_mocked_datetime(
                            slot_value
                        )
                        converted_slots[slot_name] = converted_value
                    except ValidationError:
                        # Collect the error for later reporting
                        validation_errors.append((fixture.name, str(slot_value)))

            # Create new fixture with converted values (even if validation failed,
            # we'll raise an error after processing all fixtures)
            converted_fixture = dataclasses.replace(fixture, slots_set=converted_slots)
            converted_fixtures.append(converted_fixture)

        # If there are any validation errors, raise a single comprehensive error
        if validation_errors:
            valid_formats = [
                "YYYY-MM-DDTHH:MM:SS±HH:MM  e.g. '2024-01-15T14:30:00+05:30'",
                "YYYY-MM-DDTHH:MM:SS±HHMM   e.g. '2024-01-15T14:30:00+0530'",
                "YYYY-MM-DD HH:MM:SS        e.g. '2024-01-15 14:30:00'",
                "YYYY-MM-DDTHH:MM:SS        e.g. '2024-01-15T14:30:00'",
                "YYYY-MM-DD                 e.g. '2024-01-15'",
            ]

            error_details = "\n".join(
                f"{i + 1}. Fixture - `{fixture_name}`, mocked_datetime value: "
                f"`{mocked_datetime_value}`"
                for i, (fixture_name, mocked_datetime_value) in enumerate(
                    validation_errors
                )
            )

            raise ValidationError(
                code="e2e_test_runner.validate_mocked_datetime.invalid_value_format",
                event_info=(
                    "Unable to convert to a valid datetime. Invalid `mocked_datetime` "
                    "value present in the following fixtures."
                    + f"\n\n{error_details}\n\n"
                    + "Accepted formats include:\n"
                    + "\n".join(f"  * {fmt}" for fmt in valid_formats)
                ),
            )

        return converted_fixtures

    def _get_validated_mocked_datetime(
        self, mocked_datetime_value: Any
    ) -> Optional[str]:
        """Validates and converts mocked_datetime to ISO 8601 format.

        Args:
            mocked_datetime_value: The value of the mocked_datetime slot.
                Expected to be a string from YAML fixtures, or None.

        Returns:
            An ISO 8601 format string (timezone-aware) or None.
            Example: '2024-01-15T14:30:45+00:00'

        Raises:
            ValidationError: If the mocked_datetime value cannot be converted
                to a datetime object.
        """
        if mocked_datetime_value is None:
            return None

        if not isinstance(mocked_datetime_value, str):
            # YAML fixtures only provide strings, so other types are invalid
            raise ValidationError(
                code="e2e_test_runner.validate_mocked_datetime.invalid_value_type",
                event_info="Unable to convert to a valid datetime.",
            )

        valid_datetime_formats = {
            "%Y-%m-%dT%H:%M:%S%z": True,  # already timezone-aware
            "%Y-%m-%d %H:%M:%S": False,  # needs UTC
            "%Y-%m-%dT%H:%M:%S": False,  # needs UTC
            "%Y-%m-%d": False,  # needs UTC
        }

        for datetime_format, timezone_aware in valid_datetime_formats.items():
            try:
                parsed = datetime.strptime(mocked_datetime_value, datetime_format)
                # Ensure timezone-aware
                if not timezone_aware:
                    parsed = parsed.replace(tzinfo=timezone.utc)
                # Return ISO format string
                return parsed.isoformat()
            except ValueError:
                continue

        # If we get here, the conversion failed. i.e. all parsing attempts fail
        raise ValidationError(
            code="e2e_test_runner.validate_mocked_datetime.invalid_value_format",
            event_info="Unable to convert to a valid datetime.",
        )

    async def _process_test_case(
        self,
        test_case: TestCase,
        sender_id: str,
        input_fixtures: List[Fixture],
        input_metadata: Optional[List[Metadata]],
    ) -> Tuple[TEST_TURNS_TYPE, TestResult]:
        """Runs a single test case and returns the test turns and result."""
        test_turns = await self._run_test_case(
            sender_id, input_fixtures, input_metadata, test_case
        )

        if not test_case.uses_assertions():
            test_result = self.generate_test_result(test_turns, test_case)
        else:
            test_result = await self.run_assertions(
                sender_id, test_case, input_metadata
            )

        return test_turns, test_result

    async def run_tests_for_fine_tuning(
        self,
        input_test_cases: List[TestCase],
        input_fixtures: List[Fixture],
        input_metadata: Optional[List[Metadata]],
    ) -> List[Conversation]:
        """Runs the test cases for fine-tuning.

        Converts passing test cases into conversation objects containing the
        prompts and llm commands per user message.

        Args:
            input_test_cases: Input test cases.
            input_fixtures: Input fixtures.
            input_metadata: Input metadata.

        Returns:
            List of conversations.
        """
        import rasa.llm_fine_tuning.annotation_module

        conversations = []

        for i in tqdm(range(len(input_test_cases))):
            test_case = input_test_cases[i]
            sender_id = self.generate_sender_id(test_case.name)

            test_turns, test_result = await self._process_test_case(
                test_case, sender_id, input_fixtures, input_metadata
            )

            if not test_result.pass_status:
                structlogger.warning(
                    "annotation_module.skip_test_case.failing_e2e_test",
                    test_case=test_case.name,
                    file=test_case.file,
                )
                continue

            tracker = await self.agent.tracker_store.retrieve(sender_id)
            conversation = rasa.llm_fine_tuning.annotation_module.generate_conversation(
                test_turns, test_case, tracker, test_case.uses_assertions()
            )

            if conversation:
                conversations.append(conversation)

        return conversations

    async def run_tests_to_convert_tests_to_du_tests(
        self,
        input_test_cases: List[TestCase],
        input_fixtures: List[Fixture],
        input_metadata: Optional[List[Metadata]],
        converting_method: Callable[
            [TEST_TURNS_TYPE, TestCase, bool, bool],
            Optional[DialogueUnderstandingTestCase],
        ],
    ) -> Tuple[
        List[DialogueUnderstandingTestCase], List[DialogueUnderstandingTestCase]
    ]:
        """Runs the test cases to convert them into dialogue understanding tests.

        Converts test cases into dialogue understanding test cases.

        Args:
            input_test_cases: Input test cases.
            input_fixtures: Input fixtures.
            input_metadata: Input metadata.
            converting_method: The method to convert the e2e test case into a
              dialogue understanding test case.

        Returns:
            List of ready dialogue understanding test cases and list of
            dialogue understanding test cases to review.
        """
        ready_du_test_cases = []
        to_review_du_test_cases = []

        for i in tqdm(range(len(input_test_cases))):
            test_case = input_test_cases[i]
            sender_id = self.generate_sender_id(test_case.name)

            test_turns, test_result = await self._process_test_case(
                test_case, sender_id, input_fixtures, input_metadata
            )

            du_test_case = converting_method(
                test_turns,
                test_case,
                test_case.uses_assertions(),
                test_result.pass_status,
            )

            if du_test_case:
                if test_result.pass_status:
                    ready_du_test_cases.append(du_test_case)
                else:
                    to_review_du_test_cases.append(du_test_case)

        return ready_du_test_cases, to_review_du_test_cases

    @staticmethod
    def _action_server_is_reachable(
        endpoints: AvailableEndpoints, module: str = "e2e_test_runner"
    ) -> None:
        """Calls the action server health endpoint."""
        if not endpoints.action:
            structlogger.debug(
                f"{module}._action_server_is_reachable",
                message="No action endpoint configured. Skipping the health check "
                "of the action server.",
            )
            return

        if endpoints.action.actions_module:
            structlogger.debug(
                f"{module}._action_server_is_reachable",
                message="Rasa server is configured to run custom actions directly. "
                "Skipping the health check of the action server.",
            )
            return

        if not endpoints.action.url:
            structlogger.debug(
                f"{module}._action_server_is_reachable",
                message="Action endpoint URL is not defined in the endpoint "
                "configuration.",
            )
            return

        structlogger.debug(
            f"{module}._action_server_is_reachable",
            message="Detected action URL in the endpoint configuration.\n"
            f"Action Server URL: {endpoints.action.url}\n"
            "Sending a health request to the action endpoint.",
        )
        url = urlparse(endpoints.action.url)
        # replace /<path> with just /health
        url = url._replace(path="/health").geturl()  # type: ignore[assignment]
        try:
            response = requests.get(url, timeout=3)
        except requests.exceptions.ConnectionError as error:
            raise RasaException(
                "Action endpoint could not be reached. "
                "Actions server URL is defined in your endpoint configuration as "
                f"'{endpoints.action.url}'.\n"
                "Please make sure your action server is running and properly "
                "configured. Since running tests without a action server may "
                f"lead to unpredictable results.\n{error}"
            )

        if response.status_code != 200:
            raise RasaException(
                "Action endpoint is responding, but health status responded with "
                f"code {response.status_code}. Make sure your action server"
                " is properly configured and that the '/health' endpoint is available."
            )

        structlogger.debug(
            f"{module}._action_server_is_reachable",
            message="Action endpoint has responded successfully.\n"
            f"Response message: {response.text}\n"
            f"Response status code: {response.status_code}.",
        )

    def _get_tested_flow_paths_and_commands(
        self,
        events: List[Event],
        test_result: TestResult,
        flows: Optional["FlowsList"] = None,
    ) -> Tuple[Optional[List[FlowPath]], Dict[str, Dict[str, int]]]:
        """Extract tested paths and commands from dialog events.

        A flow path consists of bot utterances, custom actions, and inferred
        step executions (call/link steps, collect steps with prefilled slots).

        Args:
            events: The list of dialog events.
            test_result: The result of the test incl. the pass status.
            flows: Optional flow definitions for slot-to-step mapping.

        Returns:
            Tuple[flow_paths: Optional[List[FlowPath]], tested_commands:
            Dict[str, Dict[str, int]]], where tested_commands is a
            dictionary like
            {"flow1": {"set slot": 5, "clarify": 1}, "flow2": {"set slot": 3}}
        """
        tested_paths = []
        # we want to create a flow path per flow the e2e test covers
        # as an e2e test can cover multiple flows, we might end up creating
        # multiple flow paths
        _tested_commands: DefaultDict[str, DefaultDict[str, int]] = defaultdict(
            lambda: defaultdict(int)
        )
        flow_paths_stack: list[FlowPath] = []

        # Build slot-to-step mapping for tracking collect steps with prefilled slots
        slot_to_collect_step = self._build_slot_to_collect_step_mapping(flows)

        for event in events:
            if isinstance(event, FlowStarted) and not event.flow_id.startswith(
                RASA_DEFAULT_FLOW_PATTERN_PREFIX
            ):
                # When a child flow starts due to a call/link step,
                # mark the parent's call/link step as visited using the metadata
                # that was added by the flow executor.
                if (
                    flow_paths_stack
                    and PARENT_FLOW_ID_METADATA_KEY in event.metadata
                    and PARENT_STEP_ID_METADATA_KEY in event.metadata
                ):
                    parent_flow_id = event.metadata[PARENT_FLOW_ID_METADATA_KEY]
                    parent_step_id = event.metadata[PARENT_STEP_ID_METADATA_KEY]
                    # Find the parent flow path on the stack and add the call/link step
                    for flow_path in reversed(flow_paths_stack):
                        if flow_path.flow == parent_flow_id:
                            call_step_node = PathNode(
                                step_id=parent_step_id,
                                flow=parent_flow_id,
                            )
                            # Only add if not already present
                            if call_step_node not in flow_path.nodes:
                                flow_path.nodes.append(call_step_node)
                            break

                flow_paths_stack.append(FlowPath(event.flow_id))

            elif (
                isinstance(event, FlowCompleted)
                and len(flow_paths_stack) > 0
                and event.flow_id == flow_paths_stack[-1].flow
            ):
                # flow path is completed as the flow ended
                tested_paths.append(flow_paths_stack.pop())

            elif isinstance(event, BotUttered):
                if (
                    flow_paths_stack
                    and STEP_ID_METADATA_KEY in event.metadata
                    and ACTIVE_FLOW_METADATA_KEY in event.metadata
                ):
                    flow_paths_stack[-1].nodes.append(self._create_path_node(event))

            elif isinstance(event, ActionExecuted):
                # we are only interested in custom actions
                if (
                    flow_paths_stack
                    and self.agent.domain
                    and self.agent.domain.is_custom_action(event.action_name)
                    and STEP_ID_METADATA_KEY in event.metadata
                    and ACTIVE_FLOW_METADATA_KEY in event.metadata
                ):
                    flow_paths_stack[-1].nodes.append(self._create_path_node(event))

            elif isinstance(event, SlotSet):
                # Track collect steps with prefilled slots.
                # When a slot is set (e.g., from user message extraction), find the
                # collect step in the current flow that collects this slot.
                if flow_paths_stack and slot_to_collect_step:
                    current_flow_id = flow_paths_stack[-1].flow
                    slot_name = event.key
                    # Look up the collect step for this slot in the current flow
                    if (current_flow_id, slot_name) in slot_to_collect_step:
                        step_id = slot_to_collect_step[(current_flow_id, slot_name)]
                        collect_step_node = PathNode(
                            step_id=step_id,
                            flow=current_flow_id,
                        )
                        # Only add if not already present (avoid duplicates)
                        if collect_step_node not in flow_paths_stack[-1].nodes:
                            flow_paths_stack[-1].nodes.append(collect_step_node)

            # Time to gather tested commands
            elif isinstance(event, UserUttered):
                if event.parse_data and COMMANDS in event.parse_data:
                    commands = [
                        command["command"] for command in event.parse_data[COMMANDS]
                    ]
                    current_flow = (
                        flow_paths_stack[-1].flow if flow_paths_stack else "no_flow"
                    )
                    for command in commands:
                        _tested_commands[current_flow][command] += 1

        # It might be that an e2e test stops before a flow was completed.
        # Add the remaining flow paths to the tested paths list.
        while len(flow_paths_stack) > 0:
            tested_paths.append(flow_paths_stack.pop())

        # Convert _tested_commands to normal dicts
        tested_commands = {key: dict(value) for key, value in _tested_commands.items()}  # type: Dict[str, Dict[str, int]]

        return tested_paths, tested_commands

    @staticmethod
    def _create_path_node(event: Event) -> PathNode:
        flow_id = event.metadata[ACTIVE_FLOW_METADATA_KEY]
        step_id = event.metadata[STEP_ID_METADATA_KEY]
        return PathNode(step_id=step_id, flow=flow_id)

    @staticmethod
    def _build_slot_to_collect_step_mapping(
        flows: Optional["FlowsList"],
    ) -> Dict[Tuple[str, str], str]:
        """Build a mapping from (flow_id, slot_name) to collect step_id.

        This allows looking up which collect step corresponds to a given slot
        when the slot is set, so we can track collect steps even when the slot
        was prefilled (and thus no BotUttered event was emitted).

        Args:
            flows: The flow definitions.

        Returns:
            Dict mapping (flow_id, slot_name) to the collect step's step_id.
        """
        mapping: Dict[Tuple[str, str], str] = {}
        if not flows:
            return mapping

        for flow in flows.underlying_flows:
            for step in flow.get_collect_steps():
                # step.collect is the slot name being collected
                # step.id is the step's ID
                mapping[(flow.id, step.collect)] = step.id

        return mapping
