import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest
from inspect_ai.log import EvalLog
from typer.testing import CliRunner

from bench_af._abstract.configs import (
    EnvironmentConfig,
    ModelOrganismConfig,
)
from bench_af._abstract.objects import Environment, ModelOrganism
from bench_af.cli.cli import app
from bench_af.util import (
    list_detectors,
    list_environments,
    list_model_organisms,
    load_environment,
    load_model_organism,
)
from bench_af.util.load import (
    ValidateModelConfig,
)

runner = CliRunner()


class TestEnd2EndWorkflows:
    """End-to-end tests for complete workflows."""

    def test_list_commands_e2e(self):
        """Test that list commands work with actual registry."""
        # Test list models
        result = runner.invoke(app, ["list", "models"])
        assert result.exit_code == 0

        # Should return a list (might be empty without defaults)
        models = list_model_organisms()
        assert isinstance(models, list)

        # Test list environments
        result = runner.invoke(app, ["list", "environments"])
        assert result.exit_code == 0

        # Should return a list (might be empty without defaults)
        environments = list_environments()
        assert isinstance(environments, list)

        # Test list detectors (may be empty)
        result = runner.invoke(app, ["list", "detectors"])
        assert result.exit_code == 0

    def test_registry_loading_e2e(self):
        """Test that registry can actually load the default model organism."""
        # This test is skipped since defaults are not in main registry
        pytest.skip("No default implementations in registry")
        # Load the default model organism
        model_organism = load_model_organism("default")

        # Verify it's properly structured
        assert isinstance(model_organism, ModelOrganism)
        assert model_organism.name == "default"
        assert model_organism.inspect_load_name == "openai/gpt-4o"
        assert model_organism.model is not None
        assert model_organism.solver is not None

        # Verify it has supported environments
        assert len(model_organism.supported_environments) > 0

        # Get task info for default task
        environment = load_environment("default")
        assert isinstance(environment, Environment)
        assert environment.name == "default"

    @patch("cli.validate_model.eval")
    @patch("cli.run.load_environment")
    @patch("cli.run.load_model_organism")
    @patch("cli.run.load_config")
    @patch("cli.run._validate_config_path")
    @patch("cli.run._get_results_dir")
    def test_validate_model_e2e_with_mocked_eval(
        self,
        mock_get_results_dir,
        mock_validate_path,
        mock_load_config,
        mock_load_model,
        mock_load_env,
        mock_eval,
    ):
        """Test full validate-model workflow with mocked inspect_ai eval."""
        # Setup mock eval to return realistic results
        mock_eval_log = MagicMock(spec=EvalLog)
        mock_eval_log.status = "success"
        mock_eval_log.results = MagicMock()
        mock_eval_log.results.metrics = {"accuracy": MagicMock(value=0.9)}
        mock_eval.return_value = [mock_eval_log]

        # Setup mock return values for load functions
        from bench_af._abstract.objects import Environment, ModelOrganism

        mock_organism = MagicMock(spec=ModelOrganism)
        mock_organism.name = "default"
        mock_organism.supported_environments = {"default"}
        mock_organism.model = MagicMock()
        mock_organism.solver = MagicMock()
        mock_environment = MagicMock(spec=Environment)
        mock_environment.name = "default"
        # Mock the environment methods that might be called
        mock_environment.get_observed_task.return_value = MagicMock()
        mock_environment.get_unobserved_task.return_value = MagicMock()
        mock_load_model.return_value = mock_organism
        mock_load_env.return_value = mock_environment

        # Mock results directory
        mock_results_dir = Path("/tmp/mock_results")
        mock_get_results_dir.return_value = mock_results_dir

        # Create mock config with new Pydantic structure
        config_data = ValidateModelConfig(
            model_organism=ModelOrganismConfig(
                name="default", description="Default model organism", config={}
            ),
            environments=[
                EnvironmentConfig(
                    name="default",
                    description="Default environment",
                    eval_config={},
                )
            ],
        )
        mock_validate_path.return_value = Path(
            "test_config.json"
        )  # Mock successful validation
        mock_load_config.return_value = config_data

        # Run the full validate-model command
        result = runner.invoke(
            app, ["run", "validate-model", "--config", "test_config.json"]
        )

        # Should succeed
        assert result.exit_code == 0

        # Should have called eval with correct parameters
        mock_eval.assert_called_once()
        call_args = mock_eval.call_args

        # Verify the model organism was loaded correctly
        assert call_args.kwargs["model"] is not None
        assert call_args.kwargs["solver"] is not None
        assert call_args.kwargs["tasks"] is not None

    @patch("cli.run.load_config")
    def test_validate_model_unsupported_task_e2e(self, mock_load_config):
        """Test validate-model with a task not supported by the model."""
        # Mock config with unsupported task
        mock_config = {
            "model_organism": "default",
            "environment": "nonexistent_task",
            "eval_config": {},
            "raw_eval_logs": True,
        }
        mock_load_config.return_value = mock_config

        result = runner.invoke(
            app, ["run", "validate-model", "--config", "test_config.json"]
        )

        # Should fail with a clear error
        assert result.exit_code != 0

    @patch("cli.run.load_config")
    def test_validate_model_nonexistent_model_e2e(self, mock_load_config):
        """Test validate-model with nonexistent model."""
        # Mock config with nonexistent model
        mock_config = {
            "model_organism": "nonexistent_model",
            "environment": "default",
            "eval_config": {},
            "raw_eval_logs": True,
        }
        mock_load_config.return_value = mock_config

        result = runner.invoke(
            app, ["run", "validate-model", "--config", "test_config.json"]
        )

        # Should fail with registry error
        assert result.exit_code != 0

    @patch("cli.run.load_config")
    def test_validate_model_nonexistent_task_e2e(self, mock_load_config):
        """Test validate-model with nonexistent task."""
        # Mock config with nonexistent task
        mock_config = {
            "model_organism": "default",
            "environment": "nonexistent_task",
            "eval_config": {},
            "raw_eval_logs": True,
        }
        mock_load_config.return_value = mock_config

        result = runner.invoke(
            app, ["run", "validate-model", "--config", "test_config.json"]
        )

        # Should fail with task loading error
        assert result.exit_code != 0


class TestRegistrySystemE2E:
    """End-to-end tests for the registry system."""

    def test_registry_file_discovery(self):
        """Test that registry correctly discovers files in directories."""
        models = list_model_organisms()
        environments = list_environments()
        detectors = list_detectors()

        # Should return lists (may be empty if no files with expected structure exist)
        assert isinstance(models, list)
        assert isinstance(environments, list)
        assert isinstance(detectors, list)

        # Should at least find the test detector that has main.py
        assert "test" in detectors

    @pytest.mark.skip
    # this test loads all the models, which takes forever and we only use it occasionally
    def test_model_organism_loading_chain(self):
        """Test the complete model organism loading chain."""
        # List available models
        models = list_model_organisms()
        assert isinstance(models, list)

        # Load each available model
        for model_name in models:
            model_organism = load_model_organism(model_name)
            assert isinstance(model_organism, ModelOrganism)
            assert model_organism.name == model_name

    def test_environment_generation_chain(self):
        """Test the complete environment generation chain."""
        # List available environments
        environments = list_environments()
        assert isinstance(environments, list)

        # Only test if we have environments in registry
        if environments:
            # Generate environment for each available environment
            for environment_name in environments:
                environment = load_environment(environment_name)
                assert isinstance(environment, Environment)
                assert environment.name == environment_name


class TestDefaultImplementationsE2E:
    """End-to-end tests for default model organism and task."""

    def test_default_model_organism_structure(self):
        """Test default model organism has correct structure."""
        # This test is skipped since defaults are not in main registry
        pytest.skip("No default implementations in registry")
        model_organism = load_model_organism("default")

        # Verify all required fields
        assert model_organism.name == "default"
        assert model_organism.model is not None
        assert model_organism.solver is not None
        assert isinstance(model_organism.supported_environments, set)
        assert len(model_organism.supported_environments) > 0
        assert model_organism.inspect_load_name == "openai/gpt-4o"

        # Verify supported environments contains default environment
        assert "default" in model_organism.supported_environments

    @pytest.mark.skip(reason="No default implementations in registry")
    def test_default_environment_structure(self):
        """Test default environment has correct structure."""
        # This test is skipped since defaults are not in main registry

    @patch("cli.validate_model.eval")
    def test_default_model_task_compatibility(self, mock_eval):
        """Test that default model organism supports default task."""
        # This test is skipped since defaults are not in main registry
        pytest.skip("No default implementations in registry")
        # Setup mock
        mock_eval_log = MagicMock(spec=EvalLog)
        mock_eval_log.status = "success"
        mock_eval_log.results = MagicMock()
        mock_eval_log.results.metrics = {"accuracy": MagicMock(value=1.0)}
        mock_eval.return_value = [mock_eval_log]

        # Load default implementations
        model_organism = load_model_organism("default")
        environment = load_environment("default")

        # Verify the environment is in supported environments
        assert environment.name in model_organism.supported_environments

        # Test validation works
        from pathlib import Path

        from cli.validate_model import validate_model_organism

        mock_results_dir = Path("/tmp/test_results")
        result = validate_model_organism(
            model_organism=model_organism,
            environment=environment,
            raw_eval_logs=False,
            eval_config={},
            results_dir=mock_results_dir,
        )

        # Should return parsed results
        assert isinstance(result, str)
        assert "Accuracy" in result


class TestErrorHandlingE2E:
    """End-to-end tests for error handling."""

    def test_graceful_handling_of_missing_directories(self):
        """Test graceful handling when model/detector/task directories are missing."""
        # This test would require temporarily moving directories
        # For now, just test that the functions don't crash with empty results
        pass

    def test_malformed_model_organism_handling(self):
        """Test handling of malformed model organism files."""
        # Create a temporary malformed model file
        with tempfile.TemporaryDirectory() as temp_dir:
            temp_models_dir = Path(temp_dir) / "models"
            temp_models_dir.mkdir()

            # Create a malformed Python file
            malformed_file = temp_models_dir / "malformed.py"
            malformed_file.write_text(
                "# This is not a valid model organism\nsyntax error!"
            )

            # Test that loading fails gracefully
            with patch("util.list._REGISTRY_DIR", Path(temp_dir)):
                with pytest.raises(Exception):  # Should raise some kind of exception
                    load_model_organism("malformed")

    @patch("cli.run.load_config")
    def test_cli_error_propagation(self, mock_load_config):
        """Test that CLI properly propagates errors from underlying functions."""
        # Create mock config with new Pydantic structure for nonexistent model
        mock_config = ValidateModelConfig(
            model_organism=ModelOrganismConfig(
                name="definitely_nonexistent_model",
                description="Nonexistent model organism",
                config={},
            ),
            environments=[
                EnvironmentConfig(
                    name="default",
                    description="Default environment",
                    eval_config={},
                )
            ],
        )
        mock_load_config.return_value = mock_config

        # Test with invalid model name
        result = runner.invoke(
            app, ["run", "validate-model", "--config", "test_config.json"]
        )

        assert result.exit_code != 0
        # Error should be visible in output or exception
        assert result.exception is not None or "error" in result.stdout.lower()


class TestCLIWorkflowE2E:
    """End-to-end tests for CLI workflow integration."""

    def test_help_commands_work(self):
        """Test that all help commands work correctly."""
        # Main help
        result = runner.invoke(app, ["--help"])
        assert result.exit_code == 0
        assert "run" in result.stdout

        # Subcommand help
        result = runner.invoke(app, ["run", "validate-model", "--help"])
        assert result.exit_code == 0

        result = runner.invoke(app, ["list", "--help"])
        assert result.exit_code == 0

    def test_command_chaining_workflow(self):
        """Test a realistic workflow using multiple commands."""
        # First, list available models
        result = runner.invoke(app, ["list", "models"])
        assert result.exit_code == 0
        models_output = result.stdout

        # Then list available environments
        result = runner.invoke(app, ["list", "environments"])
        assert result.exit_code == 0
        tasks_output = result.stdout

        # Both should be successful outputs
        assert isinstance(models_output, str)
        assert isinstance(tasks_output, str)

        # Finally, attempt to validate using these discovered resources
        with (
            patch("cli.validate_model.eval") as mock_eval,
            patch("cli.run.load_config") as mock_load_config,
            patch("cli.run.load_model_organism") as mock_load_model,
            patch("cli.run.load_environment") as mock_load_env,
            patch("cli.run._validate_config_path") as mock_validate_path,
            patch("cli.run._get_results_dir") as mock_get_results_dir,
        ):
            mock_eval_log = MagicMock(spec=EvalLog)
            mock_eval_log.status = "success"
            mock_eval_log.results = MagicMock()
            mock_eval_log.results.metrics = {"accuracy": MagicMock(value=0.95)}
            mock_eval.return_value = [mock_eval_log]

            # Setup mock return values
            from bench_af._abstract.objects import Environment, ModelOrganism

            mock_organism = MagicMock(spec=ModelOrganism)
            mock_organism.name = "default"
            mock_organism.supported_environments = {"default"}
            mock_organism.model = MagicMock()
            mock_organism.solver = MagicMock()
            mock_environment = MagicMock(spec=Environment)
            mock_environment.name = "default"
            mock_environment.get_observed_task.return_value = MagicMock()
            mock_environment.get_unobserved_task.return_value = MagicMock()

            # Mock results directory
            mock_results_dir = Path("/tmp/mock_results")
            mock_get_results_dir.return_value = mock_results_dir

            # Create mock config with new Pydantic structure
            mock_config = ValidateModelConfig(
                model_organism=ModelOrganismConfig(
                    name="default", description="Default model organism", config={}
                ),
                environments=[
                    EnvironmentConfig(
                        name="default",
                        description="Default environment",
                        eval_config={},
                    )
                ],
            )
            mock_validate_path.return_value = Path(
                "test_config.json"
            )  # Mock successful validation
            mock_load_config.return_value = mock_config
            mock_load_model.return_value = mock_organism
            mock_load_env.return_value = mock_environment

            result = runner.invoke(
                app, ["run", "validate-model", "--config", "test_config.json"]
            )

            assert result.exit_code == 0
