"""
Tests for Pipeline config-driven factory methods.

Tests Pipeline.from_config_files(), from_config_dir(), from_config_file(), and from_dict().
"""

from pathlib import Path

import pytest

from pycharter import Pipeline
from pycharter.etl_generator.pipeline import _load_variables_file, _unwrap_file_section
from pycharter.shared.errors import ConfigError, ConfigValidationError, PyCharterError

# Minimal valid configs for testing (type: file for extract/load to avoid network/DB)
MINIMAL_EXTRACT = {"type": "file", "path": "/tmp/input.json", "format": "json"}
MINIMAL_LOAD = {"type": "file", "path": "/tmp/output.json", "format": "json"}
MINIMAL_TRANSFORM = [{"rename": {"old": "new"}}]


class TestPipelineFromDict:
    """Tests for Pipeline.from_dict()."""

    def test_from_dict_minimal(self):
        """Pipeline.from_dict with minimal extract and load."""
        config = {
            "name": "test_pipeline",
            "extract": MINIMAL_EXTRACT,
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline.name == "test_pipeline"
        assert pipeline.extractor is not None
        assert pipeline.loader is not None
        assert len(pipeline._transformers) == 0

    def test_from_dict_with_transform(self):
        """Pipeline.from_dict with transform section (config-driven: uses _transform_config)."""
        config = {
            "extract": MINIMAL_EXTRACT,
            "transform": MINIMAL_TRANSFORM,
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline._transform_config is not None
        assert "transform" in pipeline._transform_config
        assert len(pipeline._transform_config["transform"]) == 1

    def test_from_dict_transform_file_style_unwraps(self):
        """Pipeline.from_dict unwraps file-style transform (single key 'transform')."""
        # As when loading transform.yaml: config["transform"] = load_yaml("transform.yaml")
        config = {
            "extract": MINIMAL_EXTRACT,
            "load": MINIMAL_LOAD,
            "transform": {
                "transform": {
                    "rename": {"product_id": "id"},
                    "convert": {"price": "float"},
                }
            },
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline._transform_config is not None
        assert "transform" in pipeline._transform_config
        # Should be the inner dict (simple ops), not a list
        t = pipeline._transform_config["transform"]
        assert isinstance(t, dict)
        assert "rename" in t
        assert t["rename"] == {"product_id": "id"}

    def test_from_dict_jsonata_file_style_unwraps(self):
        """Pipeline.from_dict unwraps file-style jsonata (single key 'jsonata')."""
        # As when loading transform_jsonata.yaml: config["jsonata"] = load_yaml("transform_jsonata.yaml")
        config = {
            "extract": MINIMAL_EXTRACT,
            "load": MINIMAL_LOAD,
            "jsonata": {
                "jsonata": {
                    "mode": "record",
                    "expression": '{"id": product_id}',
                }
            },
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline._transform_config is not None
        assert "jsonata" in pipeline._transform_config
        j = pipeline._transform_config["jsonata"]
        assert j["mode"] == "record"
        assert "id" in j["expression"]

    def test_from_dict_missing_extract_raises(self):
        """Pipeline.from_dict raises when extract is missing."""
        config = {"load": MINIMAL_LOAD}
        with pytest.raises(ValueError, match="extract"):
            Pipeline.from_dict(config)

    def test_from_dict_missing_load_raises(self):
        """Pipeline.from_dict raises when load is missing."""
        config = {"extract": MINIMAL_EXTRACT}
        with pytest.raises(ValueError, match="load"):
            Pipeline.from_dict(config)

    def test_from_dict_with_variables(self):
        """Pipeline.from_dict resolves variables from config."""
        config = {
            "variables": {"DATA_DIR": "/data", "OUTPUT_DIR": "/out"},
            "extract": {
                "type": "file",
                "path": "${DATA_DIR}/in.json",
                "format": "json",
            },
            "load": {
                "type": "file",
                "path": "${OUTPUT_DIR}/out.json",
                "format": "json",
            },
        }
        pipeline = Pipeline.from_dict(config)
        assert pipeline.extractor is not None
        assert pipeline.loader is not None


class TestUnwrapFileSection:
    """Tests for _unwrap_file_section helper (file-style config normalization)."""

    def test_returns_none_when_value_none(self):
        assert _unwrap_file_section(None, "transform") is None

    def test_unwraps_single_key_dict(self):
        assert _unwrap_file_section(
            {"transform": {"rename": {"a": "b"}}}, "transform"
        ) == {"rename": {"a": "b"}}

    def test_passthrough_when_multiple_keys(self):
        val = {"transform": {"x": 1}, "jsonata": {"y": 2}}
        assert _unwrap_file_section(val, "transform") is val

    def test_passthrough_when_single_different_key(self):
        val = {"rename": {"x": "y"}}
        assert _unwrap_file_section(val, "transform") is val

    def test_unwraps_jsonata_section(self):
        assert _unwrap_file_section(
            {"jsonata": {"mode": "record", "expression": "x"}}, "jsonata"
        ) == {"mode": "record", "expression": "x"}


class TestPipelineFromConfigFiles:
    """Tests for Pipeline.from_config_files()."""

    def test_from_config_files_dicts(self):
        """Pipeline.from_config_files with dict inputs (config-driven: uses _transform_config)."""
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            transform=MINIMAL_TRANSFORM,
            validate=True,
        )
        assert pipeline.extractor is not None
        assert pipeline.loader is not None
        assert pipeline._transform_config is not None
        # transform may be stored as list (from_config_files) or dict with "transform" key (from_dict)
        if isinstance(pipeline._transform_config, list):
            assert len(pipeline._transform_config) == 1
        else:
            assert "transform" in pipeline._transform_config
            assert len(pipeline._transform_config["transform"]) == 1

    def test_from_config_files_dicts_no_transform(self):
        """Pipeline.from_config_files with transform=None."""
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            validate=True,
        )
        assert len(pipeline._transformers) == 0

    def test_from_config_files_with_variables(self):
        """Pipeline.from_config_files with variables."""
        pipeline = Pipeline.from_config_files(
            extract={"type": "file", "path": "${DATA_DIR}/in.json", "format": "json"},
            load={"type": "file", "path": "${OUTPUT_DIR}/out.json", "format": "json"},
            variables={"DATA_DIR": "/data", "OUTPUT_DIR": "/out"},
        )
        assert pipeline.extractor is not None
        assert pipeline.loader is not None

    def test_from_config_files_missing_file_raises(self):
        """Pipeline.from_config_files raises when file not found."""
        with pytest.raises(FileNotFoundError, match="not found"):
            Pipeline.from_config_files(
                extract="/nonexistent/extract.yaml",
                load=MINIMAL_LOAD,
            )


class TestPipelineFromConfigDir:
    """Tests for Pipeline.from_config_dir()."""

    def test_from_config_dir_success(self, tmp_path):
        """Pipeline.from_config_dir loads from directory with standard filenames."""
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: /tmp/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: /tmp/out.json\nformat: json\n"
        )
        (tmp_path / "transform.yaml").write_text(
            "transform:\n  - rename:\n      old: new\n"
        )
        pipeline = Pipeline.from_config_dir(tmp_path, validate=True)
        assert pipeline.name == tmp_path.name
        assert pipeline.extractor is not None
        assert pipeline.loader is not None
        assert pipeline._transform_config is not None
        assert "transform" in pipeline._transform_config
        assert len(pipeline._transform_config["transform"]) == 1

    def test_from_config_dir_with_variables(self, tmp_path):
        """Pipeline.from_config_dir resolves variables."""
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: ${DATA_DIR}/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: ${OUTPUT_DIR}/out.json\nformat: json\n"
        )
        pipeline = Pipeline.from_config_dir(
            tmp_path,
            variables={"DATA_DIR": "/data", "OUTPUT_DIR": "/out"},
        )
        assert pipeline.extractor is not None
        assert pipeline.loader is not None

    def test_from_config_dir_not_a_directory_raises(self, tmp_path):
        """Pipeline.from_config_dir raises when path is not a directory."""
        file_path = tmp_path / "file.txt"
        file_path.write_text("x")
        with pytest.raises(NotADirectoryError, match="Not a directory"):
            Pipeline.from_config_dir(file_path)

    def test_from_config_dir_missing_extract_raises(self, tmp_path):
        """Pipeline.from_config_dir raises when extract.yaml is missing."""
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: /tmp/out.json\nformat: json\n"
        )
        with pytest.raises(FileNotFoundError, match="extract"):
            Pipeline.from_config_dir(tmp_path)

    def test_from_config_dir_missing_load_raises(self, tmp_path):
        """Pipeline.from_config_dir raises when load.yaml is missing."""
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: /tmp/in.json\nformat: json\n"
        )
        with pytest.raises(FileNotFoundError, match="load"):
            Pipeline.from_config_dir(tmp_path)


class TestPipelineFromConfigFile:
    """Tests for Pipeline.from_config_file()."""

    def test_from_config_file_success(self, tmp_path):
        """Pipeline.from_config_file loads single file with all sections."""
        config_file = tmp_path / "pipeline.yaml"
        config_file.write_text("""
name: single_file_pipeline
extract:
  type: file
  path: /tmp/in.json
  format: json
transform:
  - rename: {old: new}
load:
  type: file
  path: /tmp/out.json
  format: json
""")
        pipeline = Pipeline.from_config_file(config_file, validate=True)
        assert pipeline.name == "single_file_pipeline"
        assert pipeline.extractor is not None
        assert pipeline.loader is not None
        assert pipeline._transform_config is not None
        # transform from YAML is a list; from_dict uses dict with "transform" key
        if isinstance(pipeline._transform_config, list):
            assert len(pipeline._transform_config) == 1
        else:
            assert "transform" in pipeline._transform_config
            assert len(pipeline._transform_config["transform"]) == 1

    def test_from_config_file_missing_raises(self):
        """Pipeline.from_config_file raises when file not found."""
        with pytest.raises(FileNotFoundError, match="not found"):
            Pipeline.from_config_file("/nonexistent/pipeline.yaml")

    def test_from_config_file_directory_raises(self, tmp_path):
        """Pipeline.from_config_file raises when path is a directory."""
        with pytest.raises(ValueError, match="Not a file"):
            Pipeline.from_config_file(tmp_path)

    def test_from_config_file_missing_extract_raises(self, tmp_path):
        """Pipeline.from_config_file raises when extract section missing."""
        config_file = tmp_path / "bad.yaml"
        config_file.write_text("load:\n  type: file\n  path: /tmp/out.json\n")
        with pytest.raises(ValueError, match="extract"):
            Pipeline.from_config_file(config_file)


class TestLoadVariablesFile:
    """Tests for _load_variables_file helper."""

    def test_loads_and_merges(self, tmp_path):
        """File variables are loaded and merged with caller variables."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("API_KEY: from_file\nDB_URL: postgres://localhost\n")
        result = _load_variables_file(vars_file, {"EXTRA": "caller_val"})
        assert result["API_KEY"] == "from_file"
        assert result["DB_URL"] == "postgres://localhost"
        assert result["EXTRA"] == "caller_val"

    def test_caller_overrides_file(self, tmp_path):
        """Caller-supplied variables take precedence over file variables."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("API_KEY: from_file\n")
        result = _load_variables_file(vars_file, {"API_KEY": "from_caller"})
        assert result["API_KEY"] == "from_caller"

    def test_missing_file_returns_caller_vars(self, tmp_path):
        """Missing variables file returns caller vars unchanged."""
        result = _load_variables_file(tmp_path / "nonexistent.yaml", {"A": "1"})
        assert result == {"A": "1"}

    def test_non_dict_file_raises(self, tmp_path):
        """Non-dict variables file raises ValueError."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("- item1\n- item2\n")
        with pytest.raises(ValueError, match="mapping"):
            _load_variables_file(vars_file, {})

    def test_values_are_stringified(self, tmp_path):
        """Numeric values in variables file are stringified."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("PORT: 5432\nDEBUG: true\n")
        result = _load_variables_file(vars_file, {})
        assert result["PORT"] == "5432"
        assert result["DEBUG"] == "True"

    def test_empty_file_returns_caller_vars(self, tmp_path):
        """Empty variables file returns caller vars unchanged."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("")
        result = _load_variables_file(vars_file, {"A": "1"})
        assert result == {"A": "1"}


class TestVariablesYaml:
    """Tests for variables.yaml support across factory methods."""

    def test_from_config_dir_loads_variables_yaml(self, tmp_path):
        """from_config_dir loads variables.yaml and resolves in configs."""
        (tmp_path / "variables.yaml").write_text("DATA_DIR: /data\nOUTPUT_DIR: /out\n")
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: ${DATA_DIR}/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: ${OUTPUT_DIR}/out.json\nformat: json\n"
        )
        pipeline = Pipeline.from_config_dir(tmp_path)
        assert pipeline.extractor is not None
        assert pipeline.loader is not None

    def test_from_config_dir_caller_overrides_variables_yaml(self, tmp_path):
        """Caller variables override variables.yaml values in from_config_dir."""
        (tmp_path / "variables.yaml").write_text("DATA_DIR: /from_file\n")
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: ${DATA_DIR}/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: /tmp/out.json\nformat: json\n"
        )
        pipeline = Pipeline.from_config_dir(
            tmp_path, variables={"DATA_DIR": "/from_caller"}
        )
        assert pipeline.extractor is not None

    def test_from_config_dir_no_variables_yaml_is_fine(self, tmp_path):
        """from_config_dir works without variables.yaml present."""
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: /tmp/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: /tmp/out.json\nformat: json\n"
        )
        pipeline = Pipeline.from_config_dir(tmp_path)
        assert pipeline.extractor is not None

    def test_from_config_file_inline_variables(self, tmp_path):
        """from_config_file resolves inline variables section."""
        config_file = tmp_path / "pipeline.yaml"
        config_file.write_text(
            "name: test\n"
            "variables:\n"
            "  DATA_DIR: /data\n"
            "extract:\n"
            "  type: file\n"
            "  path: ${DATA_DIR}/in.json\n"
            "  format: json\n"
            "load:\n"
            "  type: file\n"
            "  path: /tmp/out.json\n"
            "  format: json\n"
        )
        pipeline = Pipeline.from_config_file(config_file)
        assert pipeline.extractor is not None

    def test_from_config_file_caller_overrides_inline_variables(self, tmp_path):
        """Caller variables override inline variables in from_config_file."""
        config_file = tmp_path / "pipeline.yaml"
        config_file.write_text(
            "name: test\n"
            "variables:\n"
            "  DATA_DIR: /from_file\n"
            "extract:\n"
            "  type: file\n"
            "  path: ${DATA_DIR}/in.json\n"
            "  format: json\n"
            "load:\n"
            "  type: file\n"
            "  path: /tmp/out.json\n"
            "  format: json\n"
        )
        pipeline = Pipeline.from_config_file(
            config_file, variables={"DATA_DIR": "/from_caller"}
        )
        assert pipeline.extractor is not None

    def test_from_config_files_with_variables_path(self, tmp_path):
        """from_config_files accepts a variables.yaml path."""
        vars_file = tmp_path / "variables.yaml"
        vars_file.write_text("DATA_DIR: /data\nOUTPUT_DIR: /out\n")
        (tmp_path / "extract.yaml").write_text(
            "type: file\npath: ${DATA_DIR}/in.json\nformat: json\n"
        )
        (tmp_path / "load.yaml").write_text(
            "type: file\npath: ${OUTPUT_DIR}/out.json\nformat: json\n"
        )
        pipeline = Pipeline.from_config_files(
            extract=tmp_path / "extract.yaml",
            load=tmp_path / "load.yaml",
            variables=vars_file,
        )
        assert pipeline.extractor is not None
        assert pipeline.loader is not None

    def test_from_dict_inline_variables(self):
        """from_dict resolves inline variables key."""
        config = {
            "variables": {"DATA_DIR": "/data", "OUTPUT_DIR": "/out"},
            "extract": {
                "type": "file",
                "path": "${DATA_DIR}/in.json",
                "format": "json",
            },
            "load": {
                "type": "file",
                "path": "${OUTPUT_DIR}/out.json",
                "format": "json",
            },
        }
        pipeline = Pipeline.from_dict(config)
        assert pipeline.extractor is not None
        assert pipeline.loader is not None


class TestPipelineFromConfigFilesExpanded:
    """Expanded tests for Pipeline.from_config_files() — settings, validation, base_dir."""

    def test_settings_as_dict(self):
        """from_config_files accepts settings as a dict."""
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            settings={"name": "test_settings"},
            validate=True,
        )
        assert pipeline._settings == {"name": "test_settings"}

    def test_settings_as_file_path(self, tmp_path):
        """from_config_files accepts settings as a file path."""
        settings_file = tmp_path / "settings.yaml"
        settings_file.write_text("name: file_settings\n")
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            settings=settings_file,
            validate=True,
        )
        assert pipeline._settings["name"] == "file_settings"

    def test_extracts_validation_configs(self):
        """from_config_files extracts validation sections from extract/load."""
        extract = {
            **MINIMAL_EXTRACT,
            "validation": {"schema_path": "./schema.yaml", "on_error": "warn"},
        }
        load = {
            **MINIMAL_LOAD,
            "validation": {"use_contract": True, "on_error": "fail"},
        }
        pipeline = Pipeline.from_config_files(
            extract=extract,
            load=load,
            validate=True,
        )
        assert pipeline._extract_validation_config is not None
        assert pipeline._extract_validation_config["on_error"] == "warn"
        assert pipeline._load_validation_config is not None
        assert pipeline._load_validation_config["use_contract"] is True

    def test_infers_base_dir_from_extract_path(self, tmp_path):
        """from_config_files infers base_dir from extract file path."""
        extract_file = tmp_path / "extract.yaml"
        extract_file.write_text("type: file\npath: /tmp/in.json\nformat: json\n")
        pipeline = Pipeline.from_config_files(
            extract=extract_file,
            load=MINIMAL_LOAD,
            validate=True,
        )
        assert pipeline._base_dir == tmp_path.resolve()

    def test_explicit_base_dir_overrides_inferred(self, tmp_path):
        """Explicit base_dir parameter overrides inferred value."""
        extract_file = tmp_path / "extract.yaml"
        extract_file.write_text("type: file\npath: /tmp/in.json\nformat: json\n")
        custom_base = Path("/custom/base")
        pipeline = Pipeline.from_config_files(
            extract=extract_file,
            load=MINIMAL_LOAD,
            validate=True,
            base_dir=custom_base,
        )
        assert pipeline._base_dir == custom_base


class TestIncrementalWatermark:
    """Tests for incremental extraction watermark injection."""

    def test_incremental_config_parses_in_extract(self):
        """Incremental config is accepted in extract config."""
        config = {
            "extract": {
                "type": "file",
                "path": "/tmp/in.json",
                "format": "json",
                "incremental": {
                    "enabled": True,
                    "watermark_field": "updated_at",
                    "initial_value": "1970-01-01",
                },
            },
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline.extractor is not None
        # Raw extract config should contain incremental
        assert pipeline._extract_config_raw is not None
        assert "incremental" in pipeline._extract_config_raw

    def test_incremental_disabled_passthrough(self):
        """Incremental enabled=false is accepted and stored."""
        config = {
            "extract": {
                "type": "file",
                "path": "/tmp/in.json",
                "format": "json",
                "incremental": {
                    "enabled": False,
                    "watermark_field": "updated_at",
                },
            },
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline._extract_config_raw["incremental"]["enabled"] is False

    def test_watermark_in_database_query(self):
        """Database extract config with ${__watermark__} placeholder is valid."""
        config = {
            "extract": {
                "type": "database",
                "query": "SELECT * FROM users WHERE updated_at > '${__watermark__}'",
                "connection_string": "sqlite:///test.db",
                "incremental": {
                    "enabled": True,
                    "watermark_field": "updated_at",
                    "initial_value": "1970-01-01",
                },
            },
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline.extractor is not None

    def test_watermark_in_http_params(self):
        """HTTP extract config with ${__watermark__} in params is valid."""
        config = {
            "extract": {
                "type": "http",
                "url": "https://api.example.com/users",
                "params": {"updated_since": "${__watermark__}"},
                "incremental": {
                    "enabled": True,
                    "watermark_field": "updated_at",
                    "initial_value": "1970-01-01",
                },
            },
            "load": MINIMAL_LOAD,
        }
        pipeline = Pipeline.from_dict(config, validate=True)
        assert pipeline.extractor is not None

    def test_state_store_config_in_settings(self):
        """state_store config in settings is accepted."""
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            settings={"state_store": {"backend": "file", "path": "/tmp/state"}},
            validate=True,
        )
        assert pipeline._settings["state_store"]["backend"] == "file"

    def test_plugin_config_in_settings(self):
        """plugins config in settings is accepted."""
        pipeline = Pipeline.from_config_files(
            extract=MINIMAL_EXTRACT,
            load=MINIMAL_LOAD,
            settings={
                "plugins": {
                    "extractors": {"kafka": "mypackage:KafkaExtractor"},
                }
            },
            validate=True,
        )
        assert pipeline._settings["plugins"]["extractors"]["kafka"] is not None


class TestExceptionHierarchy:
    """Tests that ETL-related errors are part of PyCharterError hierarchy."""

    def test_config_validation_error_is_pycharter_error(self):
        """ConfigValidationError is a subclass of PyCharterError."""
        err = ConfigValidationError(
            "Invalid config", errors=[{"message": "missing type"}]
        )
        assert isinstance(err, PyCharterError)
        assert isinstance(err, Exception)

    def test_config_error_is_pycharter_error(self):
        """ConfigError is a subclass of PyCharterError."""
        err = ConfigError("File not found", path="/tmp/missing.yaml")
        assert isinstance(err, PyCharterError)

    def test_catch_pycharter_error(self):
        """Catching PyCharterError catches config validation errors."""
        try:
            raise ConfigValidationError("test")
        except PyCharterError as e:
            assert "test" in str(e)
