from collections import OrderedDict, namedtuple
from pathlib import Path
from typing import List, Tuple

from jinja2 import BaseLoader, Environment

from .models import (
    BronzeEventConfig,
    BronzeLayerConfig,
    LandingLayerConfig,
    PipelineConfig,
    PipelineTransformationType,
    TransformationStep,
)

SECTION_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "Missing Value Analysis",
    PipelineTransformationType.DROP_COLUMN: "Missing Value Analysis",
    PipelineTransformationType.CAP_OUTLIER: "Global Outlier Detection",
    PipelineTransformationType.WINSORIZE: "Global Outlier Detection",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "Segment-Aware Outlier Analysis",
    PipelineTransformationType.LOG_TRANSFORM: "Feature Distributions",
    PipelineTransformationType.SQRT_TRANSFORM: "Feature Distributions",
    PipelineTransformationType.YEO_JOHNSON: "Feature Distributions",
    PipelineTransformationType.CAP_THEN_LOG: "Feature Distributions",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "Feature Distributions",
    PipelineTransformationType.ENCODE: "Categorical Feature Analysis",
    PipelineTransformationType.SCALE: "Feature-Target Correlations",
    PipelineTransformationType.FEATURE_SELECT: "Feature Selection Recommendations",
    PipelineTransformationType.DERIVED_COLUMN: "Feature Engineering Recommendations",
    PipelineTransformationType.TYPE_CAST: "Data Consistency Checks",
}

ANCHOR_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "3.5-Missing-Value-Analysis",
    PipelineTransformationType.DROP_COLUMN: "3.5-Missing-Value-Analysis",
    PipelineTransformationType.CAP_OUTLIER: "3.8-Global-Outlier-Detection",
    PipelineTransformationType.WINSORIZE: "3.8-Global-Outlier-Detection",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "3.7-Segment-Aware-Outlier-Analysis",
    PipelineTransformationType.LOG_TRANSFORM: "4.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.SQRT_TRANSFORM: "4.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.YEO_JOHNSON: "4.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.CAP_THEN_LOG: "4.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "4.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.ENCODE: "4.6-Categorical-Feature-Analysis",
    PipelineTransformationType.SCALE: "4.5-Feature-Target-Correlations",
    PipelineTransformationType.FEATURE_SELECT: "4.9.1-Feature-Selection-Recommendations",
    PipelineTransformationType.DERIVED_COLUMN: "4.9.4-Feature-Engineering-Recommendations",
    PipelineTransformationType.TYPE_CAST: "3.11-Data-Consistency-Checks",
}

DEFAULT_NOTEBOOK_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "03_quality_assessment",
    PipelineTransformationType.DROP_COLUMN: "03_quality_assessment",
    PipelineTransformationType.CAP_OUTLIER: "03_quality_assessment",
    PipelineTransformationType.WINSORIZE: "03_quality_assessment",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "03_quality_assessment",
    PipelineTransformationType.TYPE_CAST: "03_quality_assessment",
    PipelineTransformationType.LOG_TRANSFORM: "04_relationship_analysis",
    PipelineTransformationType.SQRT_TRANSFORM: "04_relationship_analysis",
    PipelineTransformationType.YEO_JOHNSON: "04_relationship_analysis",
    PipelineTransformationType.CAP_THEN_LOG: "04_relationship_analysis",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "04_relationship_analysis",
    PipelineTransformationType.ENCODE: "04_relationship_analysis",
    PipelineTransformationType.SCALE: "04_relationship_analysis",
    PipelineTransformationType.FEATURE_SELECT: "04_relationship_analysis",
    PipelineTransformationType.DERIVED_COLUMN: "04_relationship_analysis",
}


_docs_base: str = "docs"


def _notebook_title(notebook: str) -> str:
    name = notebook.split("_", 1)[1] if "_" in notebook else notebook
    return name.replace("_", " ").title()


def provenance_docstring(step: TransformationStep) -> str:
    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
    if not notebook:
        return ""
    title = _notebook_title(notebook)
    anchor = ANCHOR_MAP.get(step.type)
    section = SECTION_MAP.get(step.type)
    base = _docs_base
    if anchor:
        return f"{title} {section}\n    {base}/{notebook}.html#{anchor}"
    return f"{title}\n    {base}/{notebook}.html"


def provenance_docstring_block(steps) -> str:
    seen = set()
    entries = []
    for step in steps:
        key = provenance_key(step)
        if not key or key in seen:
            continue
        seen.add(key)
        entry = provenance_docstring(step)
        if entry:
            entries.append(entry)
    if not entries:
        return ""
    body = "\n    ".join(entries)
    return f'    """\n    {body}\n    """'


def provenance_key(step: TransformationStep) -> str:
    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
    section = SECTION_MAP.get(step.type, "")
    return f"{notebook}:{section}" if notebook else ""


class StepGrouper:

    _TYPE_TO_FUNC = {
        PipelineTransformationType.DROP_COLUMN: "drop_unusable_columns",
        PipelineTransformationType.IMPUTE_NULL: "impute_remaining_nulls",
        PipelineTransformationType.CAP_OUTLIER: "cap_outliers",
        PipelineTransformationType.TYPE_CAST: "apply_type_casts",
        PipelineTransformationType.WINSORIZE: "winsorize_outliers",
        PipelineTransformationType.SEGMENT_AWARE_CAP: "cap_segment_aware_outliers",
        PipelineTransformationType.LOG_TRANSFORM: "apply_log_transforms",
        PipelineTransformationType.SQRT_TRANSFORM: "apply_sqrt_transforms",
        PipelineTransformationType.ZERO_INFLATION_HANDLING: "handle_zero_inflation",
        PipelineTransformationType.CAP_THEN_LOG: "apply_cap_then_log_transforms",
        PipelineTransformationType.YEO_JOHNSON: "apply_power_transforms",
        PipelineTransformationType.FEATURE_SELECT: "apply_feature_selection",
    }

    _DERIVED_ACTION_TO_FUNC = {
        "ratio": "create_ratio_features",
        "interaction": "create_interaction_features",
        "composite": "create_composite_features",
    }

    @classmethod
    def group(cls, steps: List[TransformationStep]) -> List[Tuple[str, List[TransformationStep]]]:
        if not steps:
            return []
        groups: OrderedDict[str, List[TransformationStep]] = OrderedDict()
        for step in steps:
            groups.setdefault(cls._func_name(step), []).append(step)
        return list(groups.items())

    @classmethod
    def _func_name(cls, step: TransformationStep) -> str:
        if step.type == PipelineTransformationType.DERIVED_COLUMN:
            action = step.parameters.get("action", "ratio")
            return cls._DERIVED_ACTION_TO_FUNC.get(action, f"create_{action}_features")
        return cls._TYPE_TO_FUNC.get(step.type, f"apply_{step.type.value}")


group_steps = StepGrouper.group


class InlineLoader(BaseLoader):
    def __init__(self, templates: dict):
        self._templates = templates

    def get_source(self, environment, template):
        if template in self._templates:
            return self._templates[template], template, lambda: True
        raise Exception(f"Template {template} not found")


TEMPLATES = {
    "config.py.j2": """import os
from pathlib import Path

PIPELINE_NAME = "{{ config.name }}"
COMPOSITE_NAME = "{{ config.composite_name or config.name }}"
TARGET_COLUMN = "{{ config.target_column }}"
TIMESTAMP_COLUMN = "event_timestamp"
OUTPUT_DIR = Path("{{ config.output_dir }}")

# Iteration tracking
ITERATION_ID = {{ '"%s"' % config.iteration_id if config.iteration_id else 'None' }}
PARENT_ITERATION_ID = {{ '"%s"' % config.parent_iteration_id if config.parent_iteration_id else 'None' }}

# Recommendations hash for experiment tracking
RECOMMENDATIONS_HASH = {{ '"%s"' % config.recommendations_hash if config.recommendations_hash else 'None' }}


def _find_project_root():
    path = Path(__file__).parent
    for _ in range(10):
        if (path / "pyproject.toml").exists() or (path / ".git").exists():
            return path
        path = path.parent
    return Path(__file__).parent


PROJECT_ROOT = _find_project_root()

# Experiments directory - all artifacts (data, mlruns, feast) go here
# Override with CR_EXPERIMENTS_DIR environment variable for Databricks/custom locations
_default_experiments = {{ '"%s"' % config.experiments_dir if config.experiments_dir else '"experiments"' }}
EXPERIMENTS_DIR = Path(os.environ.get("CR_EXPERIMENTS_DIR", str(PROJECT_ROOT / _default_experiments)))

# Documentation base URL for provenance links in generated code
# Local: file:// URI to HTML docs (from export_tutorial_html.py)
# Databricks: set to workspace notebook path for exploration report
DOCS_BASE_URL = os.environ.get("CR_DOCS_BASE_URL", str(EXPERIMENTS_DIR / "docs"))

# Production output directory - all pipeline writes go here
# Override with CR_PRODUCTION_DIR environment variable
_default_production = {{ '"%s"' % config.production_dir if config.production_dir else 'str(EXPERIMENTS_DIR)' }}
PRODUCTION_DIR = Path(os.environ.get("CR_PRODUCTION_DIR", _default_production))

# MLflow tracking - using SQLite backend (recommended over deprecated file-based backend)
MLFLOW_TRACKING_URI = os.environ.get("MLFLOW_TRACKING_URI", f"sqlite:///{EXPERIMENTS_DIR / 'mlruns.db'}")
MLFLOW_ARTIFACT_ROOT = str(EXPERIMENTS_DIR / "mlruns" / "artifacts")

# Feast feature store configuration - stored in experiments directory
FEAST_REPO_PATH = str(PRODUCTION_DIR / "feature_repo")
FEAST_FEATURE_VIEW = "{{ config.feast.feature_view_name if config.feast else 'featureset_' + (config.composite_name or config.name) }}"
FEAST_ENTITY_NAME = "{{ config.feast.entity_name if config.feast else 'customer' }}"
FEAST_ENTITY_KEY = "{{ config.feast.entity_key if config.feast else config.sources[0].entity_key }}"
FEAST_TIMESTAMP_COL = "{{ config.feast.timestamp_column if config.feast else 'event_timestamp' }}"
FEAST_TTL_DAYS = {{ config.feast.ttl_days if config.feast else 365 }}

# Source paths - findings directory is a subfolder of experiments
FINDINGS_DIR = EXPERIMENTS_DIR / "findings"

SOURCES = {
{% for source in config.sources %}
    "{{ source.name }}": {
        "path": str(FINDINGS_DIR / "{{ source.path }}"),
        "format": "{{ source.format }}",
        "entity_key": "{{ source.entity_key }}",
{% if source.time_column %}
        "time_column": "{{ source.time_column }}",
{% endif %}
        "is_event_level": {{ source.is_event_level }},
    },
{% endfor %}
}


def get_bronze_path(source_name: str) -> Path:
    return PRODUCTION_DIR / "data" / "bronze" / source_name


def get_silver_path() -> Path:
    return PRODUCTION_DIR / "data" / "silver" / f"silver_featureset_{COMPOSITE_NAME}"


def get_gold_path() -> Path:
    return PRODUCTION_DIR / "data" / "gold" / f"gold_features_{COMPOSITE_NAME}"


def get_feast_data_path() -> Path:
    return Path(FEAST_REPO_PATH) / "data" / FEAST_FEATURE_VIEW


# Fit mode configuration for training vs scoring separation
FIT_MODE = {{ 'True' if config.fit_mode else 'False' }}
ARTIFACTS_PATH = {{ '"%s"' % config.artifacts_path if config.artifacts_path else 'str(PRODUCTION_DIR / "artifacts" / (RECOMMENDATIONS_HASH or "default"))' }}

RAW_SOURCES = {
{% for name, landing in config.landing.items() %}
    "{{ name }}": {
        "path": "{{ landing.raw_source_path }}",
        "format": "{{ landing.raw_source_format }}",
        "entity_key": "{{ landing.entity_column }}",
        "time_column": "{{ landing.time_column }}",
    },
{% endfor %}
}

EXCLUDED_SOURCES = [
{% for source in config.sources %}
{% if source.excluded %}
    "{{ source.name }}",
{% endif %}
{% endfor %}
]

EXPLORATION_ARTIFACTS = {
    "bronze": {name: str(EXPERIMENTS_DIR / "data" / "bronze" / name) for name in SOURCES},
    "silver": str(EXPERIMENTS_DIR / "data" / "silver" / f"silver_featureset_{COMPOSITE_NAME}"),
    "gold": str(EXPERIMENTS_DIR / "data" / "gold" / f"gold_features_{COMPOSITE_NAME}"),
    "scoring": str(EXPERIMENTS_DIR / "data" / "scoring" / "predictions"),
}
""",
    "bronze.py.j2": """import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.transformations, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_datetime_column, safe_to_datetime
from config import SOURCES, get_bronze_path{{ ', RAW_SOURCES' if config.lifecycle else '' }}

SOURCE_NAME = "{{ source }}"


def load_{{ source }}():
    source_config = SOURCES[SOURCE_NAME]
    path = Path(source_config["path"])
    if not path.exists():
        raise FileNotFoundError(f"Source file not found: {path}")
    if source_config["format"] == "csv":
        return pd.read_csv(path)
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))


{% set groups = group_steps(config.transformations) %}

def apply_transformations(df: pd.DataFrame) -> pd.DataFrame:
{%- if groups %}
{%- for func_name, steps in groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}

{% if config.lifecycle %}

# --- Lifecycle enrichment (computed on cleaned data) ---

ENTITY_COLUMN = "{{ config.entity_column or config.source.entity_key }}"
TIME_COLUMN = "{{ config.time_column or config.source.time_column }}"


def _load_raw_events():
    source = RAW_SOURCES[SOURCE_NAME]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(path)
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))

{% if config.lifecycle.include_recency_bucket %}

def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_datetime_column(raw_df, TIME_COLUMN)
    reference_date = raw_df[TIME_COLUMN].max()
    entity_stats = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].agg(["min", "max"])
    entity_stats["days_since_last"] = (reference_date - entity_stats["max"]).dt.days
    entity_stats["days_since_first"] = (reference_date - entity_stats["min"]).dt.days
    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df


def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_last" in df.columns:
        df["recency_bucket"] = pd.cut(df["days_since_last"], bins=[0, 7, 30, 90, 180, 365, float("inf")],
                                       labels=["0-7d", "7-30d", "30-90d", "90-180d", "180-365d", "365d+"])
    return df

{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}

def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_first" not in df.columns:
        return df
    tenure = df["days_since_first"]
    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
    if not intensity_col:
        return df
    intensity = df[intensity_col[0]]
    tenure_med = tenure.median()
    intensity_med = intensity.median()
    conditions = [
        (tenure >= tenure_med) & (intensity >= intensity_med),
        (tenure >= tenure_med) & (intensity < intensity_med),
        (tenure < tenure_med) & (intensity >= intensity_med),
        (tenure < tenure_med) & (intensity < intensity_med),
    ]
    labels = ["loyal", "at_risk", "new_active", "new_inactive"]
    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
    return df

{% endif %}
{% if config.lifecycle.include_cyclical_features %}

def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_datetime_column(raw_df, TIME_COLUMN)
    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
    df = df.drop(columns=["mean_dow"], errors="ignore")
    return df

{% endif %}
{% if config.lifecycle.momentum_pairs %}

def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
{% for pair in config.lifecycle.momentum_pairs %}
    short_col = "event_count_{{ pair.short_window }}"
    long_col = "event_count_{{ pair.long_window }}"
    if short_col in df.columns and long_col in df.columns:
        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
{% endfor %}
    return df

{% endif %}

def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
    raw_df = _load_raw_events()
{% if config.raw_time_column %}
    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.lifecycle.include_recency_bucket %}
    df = add_recency_tenure(df, raw_df)
    df = add_recency_buckets(df)
{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}
    df = add_lifecycle_quadrant(df)
{% endif %}
{% if config.lifecycle.include_cyclical_features %}
    df = add_cyclical_features(df, raw_df)
{% endif %}
{% if config.lifecycle.momentum_pairs %}
    df = add_momentum_ratios(df)
{% endif %}
    return df
{% endif %}


def run_bronze_{{ source }}():
    df = load_{{ source }}()
    df = apply_transformations(df)
{% if config.lifecycle %}
    df = enrich_lifecycle(df)
{% endif %}
    output_path = get_bronze_path(SOURCE_NAME)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    get_delta().write(df, str(output_path))
    return df


if __name__ == "__main__":
    run_bronze_{{ source }}()
""",
    "silver.py.j2": '''import pandas as pd
{% set ops, fitted = collect_imports(config.silver.derived_columns, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from config import SOURCES, get_bronze_path, get_silver_path, TARGET_COLUMN


def _load_artifact(path):
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))


def _bronze_output_name(name: str) -> str:
    return f"{name}_aggregated" if SOURCES[name].get("is_event_level") else name


def load_bronze_outputs() -> dict:
    return {name: _load_artifact(get_bronze_path(_bronze_output_name(name)))
            for name in SOURCES if not SOURCES[name].get("excluded")}


def merge_sources(bronze_outputs: dict) -> pd.DataFrame:
    base_source = "{{ config.sources[0].name }}"
    merged = bronze_outputs[base_source]
{% for join in config.silver.joins %}
    merged = merged.merge(
        bronze_outputs["{{ join.right_source }}"],
        left_on="{{ join.left_key }}",
        right_on="{{ join.right_key }}",
        how="{{ join.how }}"
    )
{% endfor %}
    return merged


def create_holdout_mask(df: pd.DataFrame, holdout_fraction: float = 0.1, random_state: int = 42) -> pd.DataFrame:
    """Create holdout set by masking target for a fraction of records.

    IMPORTANT: This must happen in the silver layer (BEFORE gold layer feature computation)
    to prevent temporal leakage. If holdout is created after features are computed,
    the features may contain information derived from the target values that will be masked.

    Args:
        df: DataFrame with TARGET_COLUMN
        holdout_fraction: Fraction of records to use for holdout (default 10%)
        random_state: Random seed for reproducibility

    Returns:
        DataFrame with holdout mask applied (original values stored in original_{TARGET_COLUMN})
    """
    ORIGINAL_COLUMN = f"original_{TARGET_COLUMN}"

    # Skip if holdout already exists
    if ORIGINAL_COLUMN in df.columns:
        print(f"  Holdout already exists ({ORIGINAL_COLUMN}), skipping creation")
        return df

    if TARGET_COLUMN not in df.columns:
        print(f"  Warning: TARGET_COLUMN \\'{TARGET_COLUMN}\\' not found, skipping holdout creation")
        return df

    print(f"Creating holdout set ({holdout_fraction:.0%} of data)...")
    df = df.copy()

    n_holdout = int(len(df) * holdout_fraction)
    holdout_idx = df.sample(n=n_holdout, random_state=random_state).index

    # Store original values for holdout records only
    df[ORIGINAL_COLUMN] = pd.NA
    df.loc[holdout_idx, ORIGINAL_COLUMN] = df.loc[holdout_idx, TARGET_COLUMN]

    # Mask target values for holdout records
    df.loc[holdout_idx, TARGET_COLUMN] = pd.NA

    print(f"  Holdout records: {n_holdout:,} ({holdout_fraction:.0%})")
    print(f"  Training records: {len(df) - n_holdout:,} ({1-holdout_fraction:.0%})")

    return df


{% set derived_groups = group_steps(config.silver.derived_columns) %}

def create_derived_columns(df: pd.DataFrame) -> pd.DataFrame:
{%- if derived_groups %}
{%- for func_name, steps in derived_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in derived_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for dc in steps %}
    # {{ dc.rationale }}
    # {{ action_description(dc) }}
    df = {{ render_step_call(dc) }}
{%- endfor %}
    return df
{% endfor %}


def run_silver_merge(create_holdout: bool = True, holdout_fraction: float = 0.1):
    bronze_outputs = load_bronze_outputs()
    silver = merge_sources(bronze_outputs)
    silver = create_derived_columns(silver)

    if create_holdout:
        silver = create_holdout_mask(silver, holdout_fraction=holdout_fraction)

    output_path = get_silver_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    get_delta().write(silver, str(output_path))
    return silver


if __name__ == "__main__":
    run_silver_merge()
''',
    "gold.py.j2": '''import pandas as pd
import warnings
from datetime import datetime
from pathlib import Path
{% set all_gold_steps = config.gold.transformations + config.gold.encodings + config.gold.scalings %}
{% set ops, fitted = collect_imports(all_gold_steps, True) %}
{% set fs_ops = ['apply_feature_select'] if config.gold.feature_selections else [] %}
from customer_retention.transforms import ArtifactStore{{ (', ' + (ops | sort | join(', '))) if ops }}{{ (', ' + (fs_ops | join(', '))) if fs_ops and 'apply_feature_select' not in ops }}
{% if fitted %}
from customer_retention.transforms.fitted import {{ fitted | sort | join(', ') }}
{% endif %}
from config import (get_silver_path, get_gold_path, get_feast_data_path,
                    TARGET_COLUMN, RECOMMENDATIONS_HASH, FEAST_REPO_PATH,
                    FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, EXPERIMENTS_DIR,
                    ARTIFACTS_PATH, FIT_MODE)

{% if config.fit_mode %}
_store = ArtifactStore(Path(ARTIFACTS_PATH))
{% else %}
_store = ArtifactStore.from_manifest(Path(ARTIFACTS_PATH) / "manifest.yaml")
{% endif %}

from customer_retention.generators.pipeline_generator.models import (
    PipelineTransformationType,
    TransformationStep,
)

ENCODINGS = [
{% for enc in config.gold.encodings %}
    TransformationStep(type=PipelineTransformationType.ENCODE, column="{{ enc.column }}", parameters={{ enc.parameters }}, rationale="{{ enc.rationale }}"),
{% endfor %}
]

SCALINGS = [
{% for scale in config.gold.scalings %}
    TransformationStep(type=PipelineTransformationType.SCALE, column="{{ scale.column }}", parameters={{ scale.parameters }}, rationale="{{ scale.rationale }}"),
{% endfor %}
]


def load_silver() -> pd.DataFrame:
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(get_silver_path()))


def load_gold() -> pd.DataFrame:
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(get_gold_path()))


{% set transform_groups = group_steps(config.gold.transformations) %}

def apply_gold_transformations(df: pd.DataFrame) -> pd.DataFrame:
{%- if transform_groups %}
{%- for func_name, steps in transform_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in transform_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t, config.fit_mode) }}
{%- endfor %}
    return df
{% endfor %}


def apply_encodings(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(config.gold.encodings) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- if config.gold.encodings %}
{%- for enc in config.gold.encodings %}
    # {{ enc.rationale }}
    # {{ action_description(enc) }}
    df = {{ render_step_call(enc, config.fit_mode) }}
{%- endfor %}
{%- endif %}
    return df


def apply_scaling(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(config.gold.scalings) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- if config.gold.scalings %}
{%- for scale in config.gold.scalings %}
    # {{ scale.rationale }}
    # {{ action_description(scale) }}
    df = {{ render_step_call(scale, config.fit_mode) }}
{%- endfor %}
{%- endif %}
    return df


def apply_feature_selection(df: pd.DataFrame) -> pd.DataFrame:
{% if config.gold.feature_selections %}
{% for fs in config.gold.feature_selections %}
    # Feature selection
    # drop {{ fs }} (feature selection)
    df = apply_feature_select(df, '{{ fs }}')
{% endfor %}
{% endif %}
    return df


def get_feature_version_tag() -> str:
    if RECOMMENDATIONS_HASH:
        return f"v1.0.0_{RECOMMENDATIONS_HASH}"
    return "v1.0.0"


def add_feast_timestamp(df: pd.DataFrame, reference_date=None) -> pd.DataFrame:
    if FEAST_TIMESTAMP_COL not in df.columns:
        if "aggregation_reference_date" in df.attrs:
            timestamp = pd.Timestamp(df.attrs["aggregation_reference_date"])
            print(f"  Using aggregation reference_date for Feast timestamp: {timestamp}")
        elif reference_date is not None:
            timestamp = reference_date
            print(f"  Using provided reference_date for Feast timestamp: {timestamp}")
        else:
            timestamp = datetime.now()
            warnings.warn(
                f"No reference_date available for Feast timestamp. Using datetime.now() ({timestamp}). "
                "This may cause temporal leakage - features should use actual aggregation dates. "
                "Set aggregation_reference_date in DataFrame.attrs during aggregation.",
                UserWarning
            )
        df[FEAST_TIMESTAMP_COL] = timestamp
    return df


def materialize_to_feast(df: pd.DataFrame) -> None:
    feast_path = get_feast_data_path()
    feast_path.parent.mkdir(parents=True, exist_ok=True)
    df_feast = df.copy()
    df_feast = add_feast_timestamp(df_feast)
    original_cols = [c for c in df_feast.columns if c.startswith("original_")]
    if original_cols:
        print(f"  Excluding holdout columns from Feast: {original_cols}")
        df_feast = df_feast.drop(columns=original_cols, errors="ignore")
    from customer_retention.integrations.adapters.factory import get_delta
    get_delta().write(df_feast, str(feast_path))
    print(f"Features materialized to Feast: {feast_path}")
    print(f"  Entity key: {FEAST_ENTITY_KEY}")
    print(f"  Feature view: {FEAST_FEATURE_VIEW}")
    print(f"  Rows: {len(df_feast):,}")


def run_gold_features():
    silver = load_silver()
    gold = apply_gold_transformations(silver)
    gold = apply_encodings(gold)
    gold = apply_scaling(gold)
    gold = apply_feature_selection(gold)
{% if config.fit_mode %}
    _store.save_manifest()
    print(f"Fit artifacts saved to: {ARTIFACTS_PATH}")
{% endif %}
    output_path = get_gold_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    gold.attrs["recommendations_hash"] = RECOMMENDATIONS_HASH
    gold.attrs["feature_version"] = get_feature_version_tag()
    from customer_retention.integrations.adapters.factory import get_delta
    get_delta().write(gold, str(output_path))
    print(f"Gold features saved with version: {get_feature_version_tag()}")
    materialize_to_feast(gold)
    return gold


if __name__ == "__main__":
    run_gold_features()
''',
    "training.py.j2": '''import pandas as pd
import mlflow
import mlflow.sklearn
import mlflow.xgboost
import xgboost as xgb
from pathlib import Path
from feast import FeatureStore
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import (roc_auc_score, average_precision_score, f1_score,
                             precision_score, recall_score, accuracy_score)
from config import (TARGET_COLUMN, PIPELINE_NAME, COMPOSITE_NAME, RECOMMENDATIONS_HASH, MLFLOW_TRACKING_URI,
                    MLFLOW_ARTIFACT_ROOT, FEAST_REPO_PATH, FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY,
                    FEAST_TIMESTAMP_COL, get_feast_data_path)

# Set tracking URI immediately to prevent default mlruns directory creation
mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)


def _load_feast_data():
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(get_feast_data_path()))


def get_training_data_from_feast() -> pd.DataFrame:
    """Retrieve training data from Feast for training/serving consistency.

    Uses get_historical_features for point-in-time correct feature retrieval.
    This ensures training uses the exact same feature retrieval path as inference.
    """
    feast_path = Path(FEAST_REPO_PATH)

    # Check if Feast repo is initialized
    if not (feast_path / "feature_store.yaml").exists():
        print("Feast repo not initialized, falling back to data file")
        return _load_feast_data()

    try:
        store = FeatureStore(repo_path=str(feast_path))

        # Read the materialized features to get entity keys and timestamps
        features_df = _load_feast_data()

        # Create entity dataframe for historical feature retrieval
        entity_df = features_df[[FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL]].copy()

        # Get all feature names (excluding entity key, timestamp, target, and holdout ground truth)
        exclude_cols = {FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, TARGET_COLUMN}
        feature_cols = [c for c in features_df.columns
                        if c not in exclude_cols and not c.startswith("original_")]

        # Build feature references
        feature_refs = [f"{FEAST_FEATURE_VIEW}:{col}" for col in feature_cols]

        print(f"Retrieving {len(feature_refs)} features from Feast...")
        print(f"  Feature view: {FEAST_FEATURE_VIEW}")
        print(f"  Entity key: {FEAST_ENTITY_KEY}")

        # Get historical features with point-in-time correctness
        training_df = store.get_historical_features(
            entity_df=entity_df,
            features=feature_refs
        ).to_df()

        # Add target column back
        training_df = training_df.merge(
            features_df[[FEAST_ENTITY_KEY, TARGET_COLUMN]],
            on=FEAST_ENTITY_KEY,
            how="left"
        )

        print(f"  Retrieved {len(training_df):,} rows, {len(training_df.columns)} columns")
        return training_df

    except Exception as e:
        print(f"Feast retrieval failed ({e}), falling back to data file")
        return _load_feast_data()


def prepare_features(df: pd.DataFrame) -> pd.DataFrame:
    """Prepare features for model training.

    Explicitly excludes original_* columns which contain holdout ground truth.
    These columns are reserved for scoring validation and must never be used in training.
    """
    df = df.copy()

    # Drop Feast metadata columns
    drop_cols = [FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL]
    df = df.drop(columns=[c for c in drop_cols if c in df.columns], errors="ignore")

    # Exclude original_* columns (holdout ground truth - prevents data leakage)
    original_cols = [c for c in df.columns if c.startswith("original_")]
    df = df.drop(columns=original_cols, errors="ignore")

    # Encode categorical columns
    for col in df.select_dtypes(include=["object", "category"]).columns:
        df[col] = LabelEncoder().fit_transform(df[col].astype(str))

    return df.select_dtypes(include=["int64", "float64", "int32", "float32"]).fillna(0)


def compute_metrics(y_true, y_proba, y_pred) -> dict:
    return {
        "roc_auc": roc_auc_score(y_true, y_proba),
        "pr_auc": average_precision_score(y_true, y_proba),
        "f1": f1_score(y_true, y_pred),
        "precision": precision_score(y_true, y_pred),
        "recall": recall_score(y_true, y_pred),
        "accuracy": accuracy_score(y_true, y_pred),
    }


def get_feature_importance(model, feature_names) -> pd.DataFrame:
    if hasattr(model, "feature_importances_"):
        importance = model.feature_importances_
    elif hasattr(model, "coef_"):
        importance = abs(model.coef_[0])
    else:
        return None
    df = pd.DataFrame({"feature": feature_names, "importance": importance})
    return df.sort_values("importance", ascending=False).reset_index(drop=True)


def log_feature_importance(model, feature_names):
    fi = get_feature_importance(model, feature_names)
    if fi is None:
        return
    fi.to_csv("feature_importance.csv", index=False)
    mlflow.log_artifact("feature_importance.csv")


def train_xgboost(X_train, y_train, X_test, y_test, feature_names):
    mlflow.xgboost.autolog(log_datasets=False, log_models=False)
    dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=feature_names)
    dtest = xgb.DMatrix(X_test, label=y_test, feature_names=feature_names)
    params = {"objective": "binary:logistic", "eval_metric": ["auc", "logloss"],
              "max_depth": 6, "learning_rate": 0.1, "seed": 42}
    model = xgb.train(params, dtrain, num_boost_round=100,
                      evals=[(dtrain, "train"), (dtest, "eval")], verbose_eval=False)
    return model


def get_model_name_with_hash(base_name: str) -> str:
    if RECOMMENDATIONS_HASH:
        return f"{base_name}_{RECOMMENDATIONS_HASH}"
    return base_name


def run_experiment():
    mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
    experiment = mlflow.get_experiment_by_name(PIPELINE_NAME)
    if experiment is None:
        mlflow.create_experiment(PIPELINE_NAME, artifact_location=MLFLOW_ARTIFACT_ROOT)
    mlflow.set_experiment(PIPELINE_NAME)
    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
    print(f"Artifacts: {MLFLOW_ARTIFACT_ROOT}")

    # Load training data from Feast (ensures training/serving consistency)
    print("\\nLoading training data from Feast...")
    training_data = get_training_data_from_feast()

    y = training_data[TARGET_COLUMN]
    X = prepare_features(training_data.drop(columns=[TARGET_COLUMN]))
    feature_names = list(X.columns)
    train_mask = y.notna()
    X, y = X.loc[train_mask], y.loc[train_mask]
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

    sklearn_models = {
        "logistic_regression": LogisticRegression(max_iter=5000, random_state=42),
        "random_forest": RandomForestClassifier(n_estimators=100, random_state=42),
    }

    run_name = get_model_name_with_hash("pipeline_run")
    with mlflow.start_run(run_name=run_name):
        mlflow.log_params({"train_samples": len(X_train), "test_samples": len(X_test), "n_features": X.shape[1]})
        mlflow.set_tag("feature_source", "feast")
        mlflow.set_tag("feast_feature_view", FEAST_FEATURE_VIEW)
        mlflow.set_tag("composite_name", COMPOSITE_NAME)
        if RECOMMENDATIONS_HASH:
            mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
        best_model, best_auc = None, 0

        for name, model in sklearn_models.items():
            with mlflow.start_run(run_name=name, nested=True):
                if RECOMMENDATIONS_HASH:
                    mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
                mlflow.set_tag("feature_source", "feast")
                model.fit(X_train, y_train)
                y_proba = model.predict_proba(X_test)[:, 1]
                y_pred = model.predict(X_test)
                metrics = compute_metrics(y_test, y_proba, y_pred)
                cv = cross_val_score(model, X_train, y_train, cv=5, scoring="roc_auc")
                mlflow.log_metrics({**metrics, "cv_mean": cv.mean(), "cv_std": cv.std()})
                log_feature_importance(model, feature_names)
                model_artifact_name = get_model_name_with_hash(f"model_{name}")
                mlflow.sklearn.log_model(model, name=model_artifact_name)
                print(f"{name}: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}")
                if metrics["roc_auc"] > best_auc:
                    best_auc, best_model = metrics["roc_auc"], name

        with mlflow.start_run(run_name="xgboost", nested=True):
            if RECOMMENDATIONS_HASH:
                mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
            mlflow.set_tag("feature_source", "feast")
            xgb_model = train_xgboost(X_train, y_train, X_test, y_test, feature_names)
            dtest = xgb.DMatrix(X_test, feature_names=feature_names)
            y_proba = xgb_model.predict(dtest)
            y_pred = (y_proba > 0.5).astype(int)
            metrics = compute_metrics(y_test, y_proba, y_pred)
            mlflow.log_metrics(metrics)
            xgb_model_name = get_model_name_with_hash("model_xgboost")
            mlflow.xgboost.log_model(xgb_model, name=xgb_model_name)
            importance = xgb_model.get_score(importance_type="gain")
            fi = pd.DataFrame({"feature": importance.keys(), "importance": importance.values()})
            fi = fi.sort_values("importance", ascending=False).reset_index(drop=True)
            fi.to_csv("feature_importance.csv", index=False)
            mlflow.log_artifact("feature_importance.csv")
            print(f"xgboost: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}")
            if metrics["roc_auc"] > best_auc:
                best_auc, best_model = metrics["roc_auc"], "xgboost"

        mlflow.set_tag("best_model", best_model)
        mlflow.log_metric("best_roc_auc", best_auc)
        print(f"Best: {best_model} (ROC-AUC={best_auc:.4f})")


if __name__ == "__main__":
    run_experiment()
''',
    "runner.py.j2": '''import argparse
import sys
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, str(Path(__file__).parent))

from config import PIPELINE_NAME, COMPOSITE_NAME, EXPERIMENTS_DIR, PRODUCTION_DIR
{% for name in config.landing %}
from landing.landing_{{ name }} import run_landing_{{ name }}
{% endfor %}
{% for name in config.bronze %}
from bronze.bronze_entity_{{ name }} import run_bronze_entity_{{ name }}
{% endfor %}
{% for name in config.bronze_event %}
from bronze.bronze_event_{{ name }} import run_bronze_event_{{ name }}
from bronze.bronze_entity_{{ name }}_aggregated import run_bronze_entity_{{ name }}_aggregated
{% endfor %}
{% set cn = config.composite_name or config.name %}
from silver.silver_featureset_{{ cn }} import run_silver_merge
from gold.gold_features_{{ cn }} import run_gold_features
from training.ml_experiment import run_experiment


def setup_experiments_dir():
    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)


def run_pipeline(validate=False):
    print(f"Starting pipeline: {PIPELINE_NAME}")
    setup_experiments_dir()
{% if config.landing %}

    print("\\n[1/6] Landing (event sources)...")
{% for name in config.landing %}
    run_landing_{{ name }}()
{% endfor %}
    print("Landing complete")
    if validate:
        from validation.validate_pipeline import validate_landing
        validate_landing()
{% endif %}

    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze event...")
{% for name in config.bronze_event %}
    run_bronze_event_{{ name }}()
{% endfor %}
    print("Bronze event complete")

    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Bronze entity (parallel)...")
    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as executor:
        bronze_futures = [
{% for name in config.bronze %}
            executor.submit(run_bronze_entity_{{ name }}),
{% endfor %}
{% for name in config.bronze_event %}
            executor.submit(run_bronze_entity_{{ name }}_aggregated),
{% endfor %}
        ]
        for f in bronze_futures:
            f.result()
    print("Bronze entity complete")
    if validate:
        from validation.validate_pipeline import validate_bronze
        validate_bronze()

    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Silver...")
    run_silver_merge()
    print("Silver complete")
    if validate:
        from validation.validate_pipeline import validate_silver
        validate_silver()

    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Gold...")
    run_gold_features()
    print("Gold complete")
    if validate:
        from validation.validate_pipeline import validate_gold
        validate_gold()

    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Training...")
    run_experiment()
    print("Training complete")
    if validate:
        from validation.validate_pipeline import validate_training
        validate_training()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--validate", action="store_true")
    args = parser.parse_args()
    run_pipeline(validate=args.validate)
''',
    "run_all.py.j2": '''"""{{ config.name }} - Pipeline Runner with MLflow UI

All artifacts (data, mlruns, feast) are stored in the experiments directory.
Override location with CR_EXPERIMENTS_DIR environment variable.
"""
import os
import sys
import webbrowser
import subprocess
import time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, str(Path(__file__).parent))

from config import PIPELINE_NAME, COMPOSITE_NAME, SOURCES, MLFLOW_TRACKING_URI, EXPERIMENTS_DIR, PRODUCTION_DIR, FINDINGS_DIR
{% for name in config.landing %}
from landing.landing_{{ name }} import run_landing_{{ name }}
{% endfor %}
{% for name in config.bronze %}
from bronze.bronze_entity_{{ name }} import run_bronze_entity_{{ name }}
{% endfor %}
{% for name in config.bronze_event %}
from bronze.bronze_event_{{ name }} import run_bronze_event_{{ name }}
from bronze.bronze_entity_{{ name }}_aggregated import run_bronze_entity_{{ name }}_aggregated
{% endfor %}
{% set cn = config.composite_name or config.name %}
from silver.silver_featureset_{{ cn }} import run_silver_merge
from gold.gold_features_{{ cn }} import run_gold_features
from training.ml_experiment import run_experiment


def setup_experiments_dir():
    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)
    print(f"Experiments directory: {EXPERIMENTS_DIR}")
    print(f"Production directory: {PRODUCTION_DIR}")
    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
    print(f"Findings directory: {FINDINGS_DIR}")


def run_landing():
{% for name in config.landing %}
    run_landing_{{ name }}()
{% endfor %}
    pass


def run_bronze_event():
{% for name in config.bronze_event %}
    run_bronze_event_{{ name }}()
{% endfor %}
    pass


def run_bronze_entity_parallel():
    bronze_funcs = [
{% for name in config.bronze %}
        run_bronze_entity_{{ name }},
{% endfor %}
{% for name in config.bronze_event %}
        run_bronze_entity_{{ name }}_aggregated,
{% endfor %}
    ]
    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as ex:
        list(ex.map(lambda f: f(), bronze_funcs))


def is_port_in_use(port):
    import socket
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        return s.connect_ex(('localhost', port)) == 0


def start_mlflow_ui():
    port = 5050
    if is_port_in_use(port):
        print(f"\\n⚠ Port {port} is already in use.")
        print(f"  Either mlflow is already running, or kill the old process:")
        print(f"  pkill -f 'mlflow ui'")
        print(f"\\n  Opening browser to existing server...")
        webbrowser.open(f"http://localhost:{port}")
        return None

    print(f"\\nStarting MLflow UI (tracking: {MLFLOW_TRACKING_URI})...")
    process = subprocess.Popen(
        ["mlflow", "ui", "--backend-store-uri", MLFLOW_TRACKING_URI, "--port", str(port)],
        stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
    )
    for _ in range(10):
        time.sleep(1)
        if process.poll() is not None:
            err = process.stderr.read().decode() if process.stderr else ""
            print(f"\\n✗ MLflow UI failed to start (exit code {process.returncode})")
            if err:
                print(err)
            return None
        if is_port_in_use(port):
            break
    webbrowser.open(f"http://localhost:{port}")
    print(f"MLflow UI running at http://localhost:{port}")
    print("Press Ctrl+C to stop")
    return process


def run_pipeline():
    print(f"Running {PIPELINE_NAME}")
    print("=" * 50)

    setup_experiments_dir()
{% if config.landing %}

    print("\\n[1/6] Landing (event sources)...")
    run_landing()
    print("Landing complete")
{% endif %}

    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze event...")
    run_bronze_event()
    print("Bronze event complete")

    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Bronze entity (parallel)...")
    run_bronze_entity_parallel()
    print("Bronze entity complete")

    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Silver...")
    run_silver_merge()
    print("Silver complete")

    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Gold...")
    run_gold_features()
    print("Gold complete")

    print("\\n[{{ '6/6' if config.landing else '5/4' }}] Training...")
    run_experiment()
    print("Training complete")

    print("\\n" + "=" * 50)
    print("Pipeline finished!")

    mlflow_process = start_mlflow_ui()
    if mlflow_process:
        try:
            mlflow_process.wait()
        except KeyboardInterrupt:
            mlflow_process.terminate()
            print("\\nMLflow UI stopped")


if __name__ == "__main__":
    run_pipeline()
''',
    "workflow.json.j2": """{
{% set cn = config.composite_name or config.name %}
  "name": "{{ config.name }}_pipeline",
  "tasks": [
{% for name in config.landing %}
    {
      "task_key": "landing_{{ name }}",
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/landing/landing_{{ name }}"
      }
    },
{% endfor %}
{% for name in config.bronze_event %}
    {
      "task_key": "bronze_event_{{ name }}",
{% if config.landing %}
      "depends_on": [
{% for lname in config.landing %}
        {"task_key": "landing_{{ lname }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
{% endif %}
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_event_{{ name }}"
      }
    },
    {
      "task_key": "bronze_entity_{{ name }}_aggregated",
      "depends_on": [{"task_key": "bronze_event_{{ name }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_entity_{{ name }}_aggregated"
      }
    },
{% endfor %}
{% for name in config.bronze %}
    {
      "task_key": "bronze_entity_{{ name }}",
{% if config.landing %}
      "depends_on": [
{% for lname in config.landing %}
        {"task_key": "landing_{{ lname }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
{% endif %}
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_entity_{{ name }}"
      }
    },
{% endfor %}
    {
      "task_key": "silver_featureset_{{ cn }}",
      "depends_on": [
{% for name in config.bronze_event %}
        {"task_key": "bronze_entity_{{ name }}_aggregated"},
{% endfor %}
{% for name in config.bronze %}
        {"task_key": "bronze_entity_{{ name }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/silver/silver_featureset_{{ cn }}"
      }
    },
    {
      "task_key": "gold_features_{{ cn }}",
      "depends_on": [{"task_key": "silver_featureset_{{ cn }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/gold/gold_features_{{ cn }}"
      }
    },
    {
      "task_key": "ml_experiment",
      "depends_on": [{"task_key": "gold_features_{{ cn }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/training/ml_experiment"
      }
    }
  ]
}
""",
    "feature_store.yaml.j2": """project: {{ config.name }}
registry: data/registry.db
provider: local
online_store:
  type: sqlite
  path: data/online_store.db
offline_store:
  type: file
entity_key_serialization_version: 2
""",
    "features.py.j2": '''from datetime import timedelta
from feast import Entity, FeatureView, Field, FileSource
from feast.types import Float32, Float64, Int64, String

{% set fv_name = config.feast.feature_view_name if config.feast else 'featureset_' + (config.composite_name or config.name) %}
{% set entity_name = config.feast.entity_name if config.feast else 'customer' %}
{% set entity_key = config.feast.entity_key if config.feast else config.sources[0].entity_key %}
{% set ts_col = config.feast.timestamp_column if config.feast else 'event_timestamp' %}
{% set ttl = config.feast.ttl_days if config.feast else 365 %}

{{ entity_name }} = Entity(
    name="{{ entity_name }}",
    join_keys=["{{ entity_key }}"],
)

{{ fv_name }}_source = FileSource(
    path="data/{{ fv_name }}",
    timestamp_field="{{ ts_col }}"
)

{{ fv_name }} = FeatureView(
    name="{{ fv_name }}",
    entities=[{{ entity_name }}],
    ttl=timedelta(days={{ ttl }}),
    source={{ fv_name }}_source,
    tags={
        "pipeline": "{{ config.name }}",
        "composite_name": "{{ config.composite_name or '' }}",
        "recommendations_hash": "{{ config.recommendations_hash or 'none' }}",
    }
)
''',
    "landing.py.j2": '''import pandas as pd
import numpy as np
from pathlib import Path
from config import RAW_SOURCES, PRODUCTION_DIR

SOURCE_NAME = "{{ name }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"
TARGET_COLUMN = "{{ config.target_column }}"


def load_raw_data() -> pd.DataFrame:
    source = RAW_SOURCES[SOURCE_NAME]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(path)
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))


def derive_feature_timestamp(df: pd.DataFrame) -> pd.DataFrame:
{% if config.timestamp_coalesce %}
{% set cols = config.timestamp_coalesce.datetime_columns_ordered %}
    df["feature_timestamp"] = pd.to_datetime(df["{{ cols[-1] }}"], errors="coerce")
{% for col in cols[:-1] | reverse %}
    df["feature_timestamp"] = df["feature_timestamp"].fillna(pd.to_datetime(df["{{ col }}"], errors="coerce"))
{% endfor %}
{% else %}
    df["feature_timestamp"] = pd.to_datetime(df[TIME_COLUMN], errors="coerce")
{% endif %}
    return df


def derive_label_timestamp(df: pd.DataFrame) -> pd.DataFrame:
{% if config.label_timestamp %}
{% set lt = config.label_timestamp %}
{% if lt.label_column %}
    df["label_timestamp"] = pd.to_datetime(df["{{ lt.label_column }}"], errors="coerce")
    df["label_timestamp"] = df["label_timestamp"].fillna(
        df["feature_timestamp"] + pd.Timedelta(days={{ lt.fallback_window_days }})
    )
{% else %}
    df["label_timestamp"] = df["feature_timestamp"] + pd.Timedelta(days={{ lt.fallback_window_days }})
{% endif %}
{% else %}
    df["label_timestamp"] = df["feature_timestamp"] + pd.Timedelta(days=180)
{% endif %}
    return df


def derive_label_available_flag(df: pd.DataFrame) -> pd.DataFrame:
    df["label_available_flag"] = df[TARGET_COLUMN].notna() if TARGET_COLUMN in df.columns else False
    return df


def derive_temporal_columns(df: pd.DataFrame) -> pd.DataFrame:
    df = derive_feature_timestamp(df)
    df = derive_label_timestamp(df)
    df = derive_label_available_flag(df)
    return df


def get_landing_output_path() -> Path:
    return PRODUCTION_DIR / "data" / "landing" / SOURCE_NAME


def run_landing_{{ name }}():
    print(f"Landing: {SOURCE_NAME}")
    df = load_raw_data()
    print(f"  Raw records: {len(df):,}")
{% if config.raw_time_column %}
    df = df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.original_target_column %}
    df = df.rename(columns={"{{ config.original_target_column }}": TARGET_COLUMN})
{% endif %}
    df = derive_temporal_columns(df)
    output_path = get_landing_output_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    get_delta().write(df, str(output_path))
    print(f"  Records: {len(df):,}")
    print(f"  Output: {output_path}")
    return df


if __name__ == "__main__":
    run_landing_{{ name }}()
''',
    "bronze_event.py.j2": '''import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.pre_shaping, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_datetime_column, safe_to_datetime
from config import PRODUCTION_DIR, TARGET_COLUMN

SOURCE_NAME = "{{ source }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"

{% set pre_groups = group_steps(config.pre_shaping) %}

def apply_pre_shaping(df: pd.DataFrame) -> pd.DataFrame:
{% if config.deduplicate %}
    df = df.drop_duplicates(subset=[ENTITY_COLUMN, TIME_COLUMN], keep="first")
{% endif %}
{%- if pre_groups %}
{%- for func_name, steps in pre_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in pre_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}

{% if config.aggregation %}
def _parse_window(window_str):
    if window_str == "all_time":
        return None
    if window_str.endswith("d"):
        return pd.Timedelta(days=int(window_str[:-1]))
    if window_str.endswith("h"):
        return pd.Timedelta(hours=int(window_str[:-1]))
    if window_str.endswith("w"):
        return pd.Timedelta(weeks=int(window_str[:-1]))
    return pd.Timedelta(days=int(window_str))


AGGREGATION_WINDOWS = {{ config.aggregation.windows }}
VALUE_COLUMNS = {{ config.aggregation.value_columns }}
AGG_FUNCS = {{ config.aggregation.agg_funcs }}
{% endif %}


def apply_event_aggregation(df: pd.DataFrame) -> pd.DataFrame:
{% if config.aggregation %}
    ensure_datetime_column(df, TIME_COLUMN)
    reference_date = df[TIME_COLUMN].max()
    base = df.groupby(ENTITY_COLUMN).agg("first")[[]]
    parts = []
    if TARGET_COLUMN in df.columns:
        parts.append(df.groupby(ENTITY_COLUMN)[TARGET_COLUMN].first())
    for window in AGGREGATION_WINDOWS:
        td = _parse_window(window)
        window_df = df if td is None else df[df[TIME_COLUMN] >= (reference_date - td)]
        for col in VALUE_COLUMNS:
            for func in AGG_FUNCS:
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].agg(func).rename(f"{col}_{func}_{window}"))
        parts.append(window_df.groupby(ENTITY_COLUMN).size().rename(f"event_count_{window}"))
    df = pd.concat([base] + parts, axis=1).reset_index()
    df.attrs["aggregation_reference_date"] = str(reference_date)
{% endif %}
    return df


def run_bronze_event_{{ source }}():
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta()
    landing_path = str(PRODUCTION_DIR / "data" / "landing" / SOURCE_NAME)
    if not storage.exists(landing_path):
        raise FileNotFoundError(f"Landing output not found: {landing_path}")
    df = storage.read(landing_path)
    df = apply_pre_shaping(df)
    df = apply_event_aggregation(df)
    output_name = f"{SOURCE_NAME}_aggregated"
    bronze_dir = PRODUCTION_DIR / "data" / "bronze"
    bronze_dir.mkdir(parents=True, exist_ok=True)
    storage.write(df, str(bronze_dir / output_name))
    return df


if __name__ == "__main__":
    run_bronze_event_{{ source }}()
''',
    "bronze_entity.py.j2": '''import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.post_shaping, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_datetime_column, safe_to_datetime
from config import PRODUCTION_DIR, RAW_SOURCES, get_bronze_path

SOURCE_NAME = "{{ source }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"

{% if config.lifecycle %}

def _load_raw_events():
    source = RAW_SOURCES["{{ raw_source }}"]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(path)
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))

{% if config.lifecycle.include_recency_bucket %}

def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_datetime_column(raw_df, TIME_COLUMN)
    reference_date = raw_df[TIME_COLUMN].max()
    entity_stats = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].agg(["min", "max"])
    entity_stats["days_since_last"] = (reference_date - entity_stats["max"]).dt.days
    entity_stats["days_since_first"] = (reference_date - entity_stats["min"]).dt.days
    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df


def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_last" in df.columns:
        df["recency_bucket"] = pd.cut(df["days_since_last"], bins=[0, 7, 30, 90, 180, 365, float("inf")],
                                       labels=["0-7d", "7-30d", "30-90d", "90-180d", "180-365d", "365d+"])
    return df

{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}

def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_first" not in df.columns:
        return df
    tenure = df["days_since_first"]
    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
    if not intensity_col:
        return df
    intensity = df[intensity_col[0]]
    tenure_med = tenure.median()
    intensity_med = intensity.median()
    conditions = [
        (tenure >= tenure_med) & (intensity >= intensity_med),
        (tenure >= tenure_med) & (intensity < intensity_med),
        (tenure < tenure_med) & (intensity >= intensity_med),
        (tenure < tenure_med) & (intensity < intensity_med),
    ]
    labels = ["loyal", "at_risk", "new_active", "new_inactive"]
    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
    return df

{% endif %}
{% if config.lifecycle.include_cyclical_features %}

def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_datetime_column(raw_df, TIME_COLUMN)
    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
    df = df.drop(columns=["mean_dow"], errors="ignore")
    return df

{% endif %}
{% if config.lifecycle.momentum_pairs %}

def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
{% for pair in config.lifecycle.momentum_pairs %}
    short_col = "event_count_{{ pair.short_window }}"
    long_col = "event_count_{{ pair.long_window }}"
    if short_col in df.columns and long_col in df.columns:
        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
{% endfor %}
    return df

{% endif %}

def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
    raw_df = _load_raw_events()
{% if config.raw_time_column %}
    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.lifecycle.include_recency_bucket %}
    df = add_recency_tenure(df, raw_df)
    df = add_recency_buckets(df)
{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}
    df = add_lifecycle_quadrant(df)
{% endif %}
{% if config.lifecycle.include_cyclical_features %}
    df = add_cyclical_features(df, raw_df)
{% endif %}
{% if config.lifecycle.momentum_pairs %}
    df = add_momentum_ratios(df)
{% endif %}
    return df
{% endif %}

{% set post_groups = group_steps(config.post_shaping) %}

def apply_quality_transforms(df: pd.DataFrame) -> pd.DataFrame:
{% if config.lifecycle %}
    df = enrich_lifecycle(df)
{% endif %}
{%- if post_groups %}
{%- for func_name, steps in post_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in post_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}


def run_bronze_entity_{{ source }}():
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta()
    bronze_path = get_bronze_path("{{ bronze_input_name }}")
    if not storage.exists(str(bronze_path)):
        raise FileNotFoundError(f"Bronze input not found: {bronze_path}")
    df = storage.read(str(bronze_path))
    df = apply_quality_transforms(df)
    bronze_path.parent.mkdir(parents=True, exist_ok=True)
    storage.write(df, str(bronze_path))
    return df


if __name__ == "__main__":
    run_bronze_entity_{{ source }}()
''',
    "validate.py.j2": '''import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

import pandas as pd
import numpy as np
from config import (SOURCES, EXPLORATION_ARTIFACTS, EXPERIMENTS_DIR, PRODUCTION_DIR,
                    TARGET_COLUMN, get_silver_path, get_gold_path)


def _load_artifact(path):
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta().read(str(path))


def _compare_dataframes(stage, production_path, exploration_path, entity_key=None, tolerance=1e-5):
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta()
    if not storage.exists(str(production_path)):
        raise FileNotFoundError(f"[{stage}] Production output not found: {production_path}")
    if not storage.exists(str(exploration_path)):
        print(f"[{stage}] SKIP - exploration artifact not found: {exploration_path}")
        return True

    prod = _load_artifact(production_path)
    expl = _load_artifact(exploration_path)

    if entity_key and entity_key in prod.columns and entity_key in expl.columns:
        prod = prod.sort_values(entity_key).reset_index(drop=True)
        expl = expl.sort_values(entity_key).reset_index(drop=True)

    if prod.shape[0] != expl.shape[0]:
        raise AssertionError(f"[{stage}] Row count: production={prod.shape[0]} vs exploration={expl.shape[0]}")

    prod_cols = set(prod.columns)
    expl_cols = set(expl.columns)
    missing = expl_cols - prod_cols
    extra = prod_cols - expl_cols
    if missing:
        print(f"[{stage}] WARNING: missing columns: {missing}")
    if extra:
        print(f"[{stage}] INFO: extra columns: {extra}")

    common = sorted(prod_cols & expl_cols)
    for col in common:
        if pd.api.types.is_numeric_dtype(prod[col]) and pd.api.types.is_numeric_dtype(expl[col]):
            try:
                pd.testing.assert_series_equal(prod[col], expl[col], check_exact=False, rtol=tolerance, check_names=False)
            except AssertionError as e:
                delta = (prod[col].astype(float) - expl[col].astype(float)).abs()
                max_idx = delta.idxmax()
                raise AssertionError(
                    f"[{stage}] Column '{col}' diverges at row {max_idx}: "
                    f"production={prod[col].iloc[max_idx]} vs exploration={expl[col].iloc[max_idx]} "
                    f"(max delta={delta.max():.2e})"
                ) from None

    print(f"[{stage}] PASS - {prod.shape[0]} rows, {len(common)} common cols, tolerance={tolerance}")
    return True


def validate_landing(tolerance=1e-5):
    landing_dir = PRODUCTION_DIR / "data" / "landing"
    if not landing_dir.exists():
        print("[Landing] SKIP - no landing directory")
        return True
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta()
    for path in landing_dir.iterdir():
        if storage.exists(str(path)):
            name = path.name
            expl_key = f"landing_{name}" if f"landing_{name}" in EXPLORATION_ARTIFACTS else "landing"
            if expl_key in EXPLORATION_ARTIFACTS:
                _compare_dataframes(f"Landing/{name}", str(path), EXPLORATION_ARTIFACTS[expl_key])
    return True


def validate_bronze(tolerance=1e-5):
    bronze_artifacts = EXPLORATION_ARTIFACTS.get("bronze", {})
    for name, expl_path in bronze_artifacts.items():
        prod_path = PRODUCTION_DIR / "data" / "bronze" / name
        _compare_dataframes(f"Bronze/{name}", str(prod_path), expl_path, tolerance=tolerance)
    return True


def validate_silver(tolerance=1e-5):
    prod_path = get_silver_path()
    expl_path = EXPLORATION_ARTIFACTS.get("silver", "")
    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
    _compare_dataframes("Silver", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
    return True


def validate_gold(tolerance=1e-5):
    prod_path = get_gold_path()
    expl_path = EXPLORATION_ARTIFACTS.get("gold", "")
    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
    _compare_dataframes("Gold", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
    return True


def validate_training():
    print("[Training] PASS - training validation requires MLflow comparison (not yet implemented)")
    return True


def validate_scoring(tolerance=1e-5):
    prod_path = PRODUCTION_DIR / "data" / "scoring" / "predictions"
    expl_path = EXPLORATION_ARTIFACTS.get("scoring", "")
    _compare_dataframes("Scoring", str(prod_path), expl_path, tolerance=tolerance)
    return True


def run_all_validations(tolerance=1e-5):
    stages = [
        ("Landing", lambda: validate_landing(tolerance)),
        ("Bronze", lambda: validate_bronze(tolerance)),
        ("Silver", lambda: validate_silver(tolerance)),
        ("Gold", lambda: validate_gold(tolerance)),
        ("Training", validate_training),
        ("Scoring", lambda: validate_scoring(tolerance)),
    ]
    results = []
    for name, fn in stages:
        try:
            fn()
            results.append((name, "PASS"))
        except Exception as e:
            results.append((name, f"FAIL: {e}"))
            break

    print("\\nStage Validation Report")
    print("=" * 50)
    for name, status in results:
        print(f"[{status.split(':')[0]:4s}] {name}")
    return results
''',
    "run_validation.py.j2": '''"""{{ config.name }} - Standalone Validation Runner

Compares pipeline outputs against exploration artifacts.
Run after pipeline completes to verify correctness.
"""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

from validation.validate_pipeline import run_all_validations


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Validate pipeline outputs")
    parser.add_argument("--tolerance", type=float, default=1e-5)
    args = parser.parse_args()

    results = run_all_validations(tolerance=args.tolerance)
    failures = [r for r in results if not r[1].startswith("PASS")]
    sys.exit(1 if failures else 0)
''',
    "exploration_report.py.j2": '''"""Exploration Report Viewer

Opens HTML documentation for the exploration notebooks that informed
the pipeline transformations. Works both locally (file:// URI) and
on Databricks (displayHTML with scroll-to-anchor injection).
"""
import os
import webbrowser
from pathlib import Path

# Known notebooks referenced by pipeline provenance comments
KNOWN_NOTEBOOKS = [
{% for nb in notebooks %}
    "{{ nb }}",
{% endfor %}
]

DOCS_DIR = Path(os.environ.get("CR_DOCS_BASE_URL", str(Path(__file__).parent)))


def _is_databricks():
    return "DATABRICKS_RUNTIME_VERSION" in os.environ


def list_reports():
    for nb in KNOWN_NOTEBOOKS:
        html_path = DOCS_DIR / f"{nb}.html"
        status = "available" if html_path.exists() else "missing"
        print(f"  {nb}: {status}")


if __name__ == "__main__":
    print("Available exploration reports:")
    list_reports()
''',
}


class CodeRenderer:
    _TEMPLATE_MAP = {
        "config": "config.py.j2",
        "silver": "silver.py.j2",
        "gold": "gold.py.j2",
        "training": "training.py.j2",
        "runner": "runner.py.j2",
        "workflow": "workflow.json.j2",
        "run_all": "run_all.py.j2",
        "feast_config": "feature_store.yaml.j2",
        "feast_features": "features.py.j2",

        "landing": "landing.py.j2",
        "bronze_event": "bronze_event.py.j2",
        "validation": "validate.py.j2",
        "run_validation": "run_validation.py.j2",
        "exploration_report": "exploration_report.py.j2",
    }

    def __init__(self):
        self._env = Environment(loader=InlineLoader(TEMPLATES))
        self._env.globals["action_description"] = action_description
        self._env.globals["render_step_call"] = render_step_call
        self._env.globals["collect_imports"] = collect_imports
        self._env.globals["group_steps"] = group_steps
        self._env.globals["provenance_docstring_block"] = provenance_docstring_block
        self._env.globals["provenance_key"] = provenance_key

    def set_docs_base(self, experiments_dir: str | None) -> None:
        global _docs_base
        if experiments_dir:
            _docs_base = f"file://{Path(experiments_dir).resolve() / 'docs'}"
        else:
            _docs_base = "docs"

    def _render(self, template_key: str, **context) -> str:
        return self._env.get_template(self._TEMPLATE_MAP[template_key]).render(**context)

    def render_config(self, config: PipelineConfig) -> str:
        return self._render("config", config=config)

    def render_bronze(self, source_name: str, bronze_config: BronzeLayerConfig) -> str:
        return self._env.get_template("bronze.py.j2").render(source=source_name, config=bronze_config)

    def render_silver(self, config: PipelineConfig) -> str:
        return self._render("silver", config=config)

    def render_gold(self, config: PipelineConfig) -> str:
        return self._render("gold", config=config)

    def render_training(self, config: PipelineConfig) -> str:
        return self._render("training", config=config)

    def render_runner(self, config: PipelineConfig) -> str:
        return self._render("runner", config=config)

    def render_workflow(self, config: PipelineConfig) -> str:
        return self._render("workflow", config=config)

    def render_run_all(self, config: PipelineConfig) -> str:
        return self._render("run_all", config=config)

    def render_feast_config(self, config: PipelineConfig) -> str:
        return self._render("feast_config", config=config)

    def render_feast_features(self, config: PipelineConfig) -> str:
        return self._render("feast_features", config=config)


    def render_landing(self, name: str, config: LandingLayerConfig) -> str:
        return self._env.get_template("landing.py.j2").render(name=name, config=config)

    def render_bronze_event(self, source_name: str, config: BronzeEventConfig) -> str:
        return self._env.get_template("bronze_event.py.j2").render(source=source_name, config=config)

    def render_bronze_entity(self, source_name: str, config: BronzeEventConfig, bronze_input_name: str, raw_source_name: str = "") -> str:
        return self._env.get_template("bronze_entity.py.j2").render(
            source=source_name, config=config, bronze_input_name=bronze_input_name,
            raw_source=raw_source_name or source_name,
        )

    def render_validation(self, config: PipelineConfig) -> str:
        return self._render("validation", config=config)

    def render_run_validation(self, config: PipelineConfig) -> str:
        return self._render("run_validation", config=config)

    def render_exploration_report(self, config: PipelineConfig) -> str:
        notebooks = set()
        for bronze in config.bronze.values():
            for step in bronze.transformations:
                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
                if nb:
                    notebooks.add(nb)
        for step in config.gold.transformations + config.gold.encodings + config.gold.scalings:
            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
            if nb:
                notebooks.add(nb)
        for step in config.silver.derived_columns:
            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
            if nb:
                notebooks.add(nb)
        for be in config.bronze_event.values():
            for step in be.pre_shaping + be.post_shaping:
                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
                if nb:
                    notebooks.add(nb)
        return self._render("exploration_report", notebooks=sorted(notebooks))


_StepMeta = namedtuple("_StepMeta", ["desc_tpl", "call_tpl", "import_name", "param_defaults"])

_STATELESS_REGISTRY = {
    PipelineTransformationType.IMPUTE_NULL: _StepMeta(
        "impute nulls in {col} with {value}",
        "apply_impute_null(df, '{col}', value='{value}')",
        "apply_impute_null", {"value": 0}),
    PipelineTransformationType.CAP_OUTLIER: _StepMeta(
        "cap outliers in {col} to [{lower}, {upper}]",
        "apply_cap_outlier(df, '{col}', lower={lower}, upper={upper})",
        "apply_cap_outlier", {"lower": 0, "upper": 1000000}),
    PipelineTransformationType.TYPE_CAST: _StepMeta(
        "cast {col} to {dtype}",
        "apply_type_cast(df, '{col}', dtype='{dtype}')",
        "apply_type_cast", {"dtype": "float"}),
    PipelineTransformationType.DROP_COLUMN: _StepMeta(
        "drop column {col}",
        "apply_drop_column(df, '{col}')",
        "apply_drop_column", {}),
    PipelineTransformationType.WINSORIZE: _StepMeta(
        "winsorize {col} to [{lower_bound}, {upper_bound}]",
        "apply_winsorize(df, '{col}', lower_bound={lower_bound}, upper_bound={upper_bound})",
        "apply_winsorize", {"lower_bound": 0, "upper_bound": 1000000}),
    PipelineTransformationType.SEGMENT_AWARE_CAP: _StepMeta(
        "segment-aware outlier cap on {col} ({n_segments} segments)",
        "apply_segment_aware_cap(df, '{col}', n_segments={n_segments})",
        "apply_segment_aware_cap", {"n_segments": 2}),
    PipelineTransformationType.LOG_TRANSFORM: _StepMeta(
        "log-transform {col}",
        "apply_log_transform(df, '{col}')",
        "apply_log_transform", {}),
    PipelineTransformationType.SQRT_TRANSFORM: _StepMeta(
        "sqrt-transform {col}",
        "apply_sqrt_transform(df, '{col}')",
        "apply_sqrt_transform", {}),
    PipelineTransformationType.ZERO_INFLATION_HANDLING: _StepMeta(
        "handle zero-inflation in {col}",
        "apply_zero_inflation_handling(df, '{col}')",
        "apply_zero_inflation_handling", {}),
    PipelineTransformationType.CAP_THEN_LOG: _StepMeta(
        "cap at p99 then log-transform {col}",
        "apply_cap_then_log(df, '{col}')",
        "apply_cap_then_log", {}),
    PipelineTransformationType.FEATURE_SELECT: _StepMeta(
        "drop {col} (feature selection)",
        "apply_feature_select(df, '{col}')",
        "apply_feature_select", {}),
}


def _extract_params(step, meta):
    return {k: step.parameters.get(k, v) for k, v in meta.param_defaults.items()}


def action_description(step: TransformationStep) -> str:
    t, col, p = step.type, step.column, step.parameters
    meta = _STATELESS_REGISTRY.get(t)
    if meta is not None:
        return meta.desc_tpl.format(col=col, **_extract_params(step, meta))
    if t == PipelineTransformationType.YEO_JOHNSON:
        return f"yeo-johnson transform {col}"
    if t == PipelineTransformationType.ENCODE:
        method = p.get("method", "one_hot")
        if method in ("one_hot", "onehot"):
            return f"one-hot encode {col}"
        return f"label-encode {col}"
    if t == PipelineTransformationType.SCALE:
        method = p.get("method", "standard")
        if method == "minmax":
            return f"min-max scale {col}"
        return f"standard-scale {col}"
    if t == PipelineTransformationType.DERIVED_COLUMN:
        action = p.get("action", "ratio")
        if action == "ratio":
            return f"create {col} = {p.get('numerator', '?')} / {p.get('denominator', '?')}"
        if action == "interaction":
            features = p.get("features", [])
            col_a = features[0] if len(features) > 0 else p.get("col_a", "?")
            col_b = features[1] if len(features) > 1 else p.get("col_b", "?")
            return f"create {col} = {col_a} * {col_b}"
        if action == "composite":
            return f"create {col} = mean({', '.join(p.get('columns', []))})"
    return f"transform {col}"


def render_step_call(step: TransformationStep, fit_mode: bool = True) -> str:
    t, col, p = step.type, step.column, step.parameters
    meta = _STATELESS_REGISTRY.get(t)
    if meta is not None:
        return meta.call_tpl.format(col=col, **_extract_params(step, meta))
    if t == PipelineTransformationType.YEO_JOHNSON:
        method = "fit_transform" if fit_mode else "transform"
        return f"FittedPowerTransform().{method}(df, '{col}', _store)"
    if t == PipelineTransformationType.ENCODE:
        method = p.get("method", "one_hot")
        if method in ("one_hot", "onehot"):
            return f"apply_one_hot_encode(df, '{col}')"
        fit_method = "fit_transform" if fit_mode else "transform"
        return f"FittedEncoder().{fit_method}(df, '{col}', _store)"
    if t == PipelineTransformationType.SCALE:
        method = p.get("method", "standard")
        fit_method = "fit_transform" if fit_mode else "transform"
        return f"FittedScaler('{method}').{fit_method}(df, '{col}', _store)"
    if t == PipelineTransformationType.DERIVED_COLUMN:
        action = p.get("action", "ratio")
        if action == "ratio":
            return f"apply_derived_ratio(df, '{col}', numerator='{p.get('numerator', '')}', denominator='{p.get('denominator', '')}')"
        if action == "interaction":
            features = p.get("features", [])
            col_a = features[0] if len(features) > 0 else p.get("col_a", "")
            col_b = features[1] if len(features) > 1 else p.get("col_b", "")
            return f"apply_derived_interaction(df, '{col}', col_a='{col_a}', col_b='{col_b}')"
        if action == "composite":
            return f"apply_derived_composite(df, '{col}', columns={p.get('columns', [])})"
    raise ValueError(f"Unknown transformation type: {step.type}")


def collect_imports(steps, include_fitted):
    ops = set()
    fitted = set()
    _OPS_MAP = {k: v.import_name for k, v in _STATELESS_REGISTRY.items()}
    for step in steps:
        t, p = step.type, step.parameters
        if t in _OPS_MAP:
            ops.add(_OPS_MAP[t])
        elif t == PipelineTransformationType.ENCODE:
            method = p.get("method", "one_hot")
            if method in ("one_hot", "onehot"):
                ops.add("apply_one_hot_encode")
            elif include_fitted:
                fitted.add("FittedEncoder")
        elif t == PipelineTransformationType.SCALE:
            if include_fitted:
                fitted.add("FittedScaler")
        elif t == PipelineTransformationType.YEO_JOHNSON:
            if include_fitted:
                fitted.add("FittedPowerTransform")
        elif t == PipelineTransformationType.DERIVED_COLUMN:
            action = p.get("action", "ratio")
            if action == "ratio":
                ops.add("apply_derived_ratio")
            elif action == "interaction":
                ops.add("apply_derived_interaction")
            elif action == "composite":
                ops.add("apply_derived_composite")
    return ops, fitted
