from collections import OrderedDict, namedtuple
from pathlib import Path
from typing import List, Tuple

from jinja2 import BaseLoader, Environment

from .models import (
    BronzeEventConfig,
    BronzeLayerConfig,
    LandingLayerConfig,
    PipelineConfig,
    PipelineTransformationType,
    TransformationStep,
)

SECTION_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "Missing Value Analysis",
    PipelineTransformationType.DROP_COLUMN: "Missing Value Analysis",
    PipelineTransformationType.CAP_OUTLIER: "Global Outlier Detection",
    PipelineTransformationType.WINSORIZE: "Global Outlier Detection",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "Segment-Aware Outlier Analysis",
    PipelineTransformationType.LOG_TRANSFORM: "Feature Distributions",
    PipelineTransformationType.SQRT_TRANSFORM: "Feature Distributions",
    PipelineTransformationType.YEO_JOHNSON: "Feature Distributions",
    PipelineTransformationType.CAP_THEN_LOG: "Feature Distributions",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "Feature Distributions",
    PipelineTransformationType.ENCODE: "Categorical Feature Analysis",
    PipelineTransformationType.SCALE: "Feature-Target Correlations",
    PipelineTransformationType.FEATURE_SELECT: "Feature Selection Recommendations",
    PipelineTransformationType.DERIVED_COLUMN: "Feature Engineering Recommendations",
    PipelineTransformationType.TYPE_CAST: "Data Consistency Checks",
    PipelineTransformationType.FILTER: "Data Quality Filters",
}

ANCHOR_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "2.5-Missing-Value-Analysis",
    PipelineTransformationType.DROP_COLUMN: "2.5-Missing-Value-Analysis",
    PipelineTransformationType.CAP_OUTLIER: "2.8-Global-Outlier-Detection",
    PipelineTransformationType.WINSORIZE: "2.8-Global-Outlier-Detection",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "2.7-Segment-Aware-Outlier-Analysis",
    PipelineTransformationType.LOG_TRANSFORM: "5.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.SQRT_TRANSFORM: "5.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.YEO_JOHNSON: "5.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.CAP_THEN_LOG: "5.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "5.4-Feature-Distributions-by-Retention-Status",
    PipelineTransformationType.ENCODE: "5.6-Categorical-Feature-Analysis",
    PipelineTransformationType.SCALE: "5.5-Feature-Target-Correlations",
    PipelineTransformationType.FEATURE_SELECT: "5.9.1-Feature-Selection-Recommendations",
    PipelineTransformationType.DERIVED_COLUMN: "5.9.4-Feature-Engineering-Recommendations",
    PipelineTransformationType.TYPE_CAST: "2.11-Data-Consistency-Checks",
}

DEFAULT_NOTEBOOK_MAP = {
    PipelineTransformationType.IMPUTE_NULL: "02_source_integrity",
    PipelineTransformationType.DROP_COLUMN: "02_source_integrity",
    PipelineTransformationType.CAP_OUTLIER: "02_source_integrity",
    PipelineTransformationType.WINSORIZE: "02_source_integrity",
    PipelineTransformationType.SEGMENT_AWARE_CAP: "02_source_integrity",
    PipelineTransformationType.TYPE_CAST: "02_source_integrity",
    PipelineTransformationType.LOG_TRANSFORM: "05_relationship_analysis",
    PipelineTransformationType.SQRT_TRANSFORM: "05_relationship_analysis",
    PipelineTransformationType.YEO_JOHNSON: "05_relationship_analysis",
    PipelineTransformationType.CAP_THEN_LOG: "05_relationship_analysis",
    PipelineTransformationType.ZERO_INFLATION_HANDLING: "05_relationship_analysis",
    PipelineTransformationType.ENCODE: "05_relationship_analysis",
    PipelineTransformationType.SCALE: "05_relationship_analysis",
    PipelineTransformationType.FEATURE_SELECT: "05_relationship_analysis",
    PipelineTransformationType.DERIVED_COLUMN: "05_relationship_analysis",
    PipelineTransformationType.FILTER: "04_column_deep_dive",
}


_docs_base: str = "docs"


def _notebook_title(notebook: str) -> str:
    name = notebook.split("_", 1)[1] if "_" in notebook else notebook
    return name.replace("_", " ").title()


def provenance_docstring(step: TransformationStep) -> str:
    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
    if not notebook:
        return ""
    title = _notebook_title(notebook)
    anchor = ANCHOR_MAP.get(step.type)
    section = SECTION_MAP.get(step.type)
    base = _docs_base
    if anchor:
        return f"{title} {section}\n    {base}/{notebook}.html#{anchor}"
    return f"{title}\n    {base}/{notebook}.html"


def provenance_docstring_block(steps) -> str:
    seen = set()
    entries = []
    for step in steps:
        key = provenance_key(step)
        if not key or key in seen:
            continue
        seen.add(key)
        entry = provenance_docstring(step)
        if entry:
            entries.append(entry)
    if not entries:
        return ""
    body = "\n    ".join(entries)
    return f'    """\n    {body}\n    """'


def provenance_key(step: TransformationStep) -> str:
    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
    section = SECTION_MAP.get(step.type, "")
    return f"{notebook}:{section}" if notebook else ""


class StepGrouper:
    _TYPE_TO_FUNC = {
        PipelineTransformationType.DROP_COLUMN: "drop_unusable_columns",
        PipelineTransformationType.IMPUTE_NULL: "impute_remaining_nulls",
        PipelineTransformationType.CAP_OUTLIER: "cap_outliers",
        PipelineTransformationType.TYPE_CAST: "apply_type_casts",
        PipelineTransformationType.WINSORIZE: "winsorize_outliers",
        PipelineTransformationType.SEGMENT_AWARE_CAP: "cap_segment_aware_outliers",
        PipelineTransformationType.LOG_TRANSFORM: "apply_log_transforms",
        PipelineTransformationType.SQRT_TRANSFORM: "apply_sqrt_transforms",
        PipelineTransformationType.ZERO_INFLATION_HANDLING: "handle_zero_inflation",
        PipelineTransformationType.CAP_THEN_LOG: "apply_cap_then_log_transforms",
        PipelineTransformationType.YEO_JOHNSON: "apply_power_transforms",
        PipelineTransformationType.FEATURE_SELECT: "apply_feature_selection",
        PipelineTransformationType.FILTER: "apply_filters",
    }

    _DERIVED_ACTION_TO_FUNC = {
        "ratio": "create_ratio_features",
        "interaction": "create_interaction_features",
        "composite": "create_composite_features",
    }

    @classmethod
    def group(cls, steps: List[TransformationStep]) -> List[Tuple[str, List[TransformationStep]]]:
        if not steps:
            return []
        groups: OrderedDict[str, List[TransformationStep]] = OrderedDict()
        for step in steps:
            groups.setdefault(cls._func_name(step), []).append(step)
        return list(groups.items())

    @classmethod
    def _func_name(cls, step: TransformationStep) -> str:
        if step.type == PipelineTransformationType.DERIVED_COLUMN:
            action = step.parameters.get("action", "ratio")
            return cls._DERIVED_ACTION_TO_FUNC.get(action, f"create_{action}_features")
        return cls._TYPE_TO_FUNC.get(step.type, f"apply_{step.type.value}")


group_steps = StepGrouper.group


class InlineLoader(BaseLoader):
    def __init__(self, templates: dict):
        self._templates = templates

    def get_source(self, environment, template):
        if template in self._templates:
            return self._templates[template], template, lambda: True
        raise Exception(f"Template {template} not found")


TEMPLATES = {
    "config.py.j2": """import os
from pathlib import Path

PIPELINE_NAME = "{{ config.name }}"
COMPOSITE_NAME = "{{ config.composite_name or config.name }}"
TARGET_COLUMN = "{{ config.target_column }}"
TIMESTAMP_COLUMN = "event_timestamp"
OUTPUT_DIR = Path("{{ config.output_dir }}")

# Iteration tracking
ITERATION_ID = {{ '"%s"' % config.iteration_id if config.iteration_id else 'None' }}
PARENT_ITERATION_ID = {{ '"%s"' % config.parent_iteration_id if config.parent_iteration_id else 'None' }}

# Recommendations hash for experiment tracking
RECOMMENDATIONS_HASH = {{ '"%s"' % config.recommendations_hash if config.recommendations_hash else 'None' }}


def _find_project_root():
    path = Path(__file__).parent
    for _ in range(10):
        if (path / "pyproject.toml").exists() or (path / ".git").exists():
            return path
        path = path.parent
    return Path(__file__).parent


PROJECT_ROOT = _find_project_root()

# Experiments directory - all artifacts (data, mlruns, feast) go here
# Override with CR_EXPERIMENTS_DIR environment variable for Databricks/custom locations
_default_experiments = {{ '"%s"' % config.experiments_dir if config.experiments_dir else '"experiments"' }}
EXPERIMENTS_DIR = Path(os.environ.get("CR_EXPERIMENTS_DIR", str(PROJECT_ROOT / _default_experiments)))

# Documentation base URL for provenance links in generated code
# Local: file:// URI to HTML docs (from export_tutorial_html.py)
# Databricks: set to workspace notebook path for exploration report
DOCS_BASE_URL = os.environ.get("CR_DOCS_BASE_URL", str(EXPERIMENTS_DIR / "docs"))

# Production output directory - all pipeline writes go here
# Override with CR_PRODUCTION_DIR environment variable
_default_production = {{ '"%s"' % config.production_dir if config.production_dir else 'str(EXPERIMENTS_DIR)' }}
PRODUCTION_DIR = Path(os.environ.get("CR_PRODUCTION_DIR", _default_production))

# MLflow tracking - using SQLite backend (recommended over deprecated file-based backend)
MLFLOW_TRACKING_URI = os.environ.get("MLFLOW_TRACKING_URI", f"sqlite:///{EXPERIMENTS_DIR / 'mlruns.db'}")
MLFLOW_ARTIFACT_ROOT = str(EXPERIMENTS_DIR / "mlruns" / "artifacts")

# Feast feature store configuration - stored in experiments directory
FEAST_REPO_PATH = str(PRODUCTION_DIR / "feature_repo")
FEAST_FEATURE_VIEW = "{{ config.feast.feature_view_name if config.feast else 'featureset_' + (config.composite_name or config.name) }}"
FEAST_ENTITY_NAME = "{{ config.feast.entity_name if config.feast else 'customer' }}"
FEAST_ENTITY_KEY = "{{ config.feast.entity_key if config.feast else 'entity_id' }}"
FEAST_TIMESTAMP_COL = "{{ config.feast.timestamp_column if config.feast else 'event_timestamp' }}"
FEAST_TTL_DAYS = {{ config.feast.ttl_days if config.feast else 365 }}

# Source paths - findings directory is a subfolder of experiments
FINDINGS_DIR = EXPERIMENTS_DIR / "findings"

SOURCES = {
{% for source in config.sources %}
    "{{ source.name }}": {
        "path": "{{ source.raw_source_path }}",
        "format": "{{ source.format }}",
        "entity_key": "{{ source.entity_key }}",
{% if source.time_column %}
        "time_column": "{{ source.time_column }}",
{% endif %}
        "is_event_level": {{ source.is_event_level }},
    },
{% endfor %}
}


def get_bronze_path(source_name: str) -> Path:
    return PRODUCTION_DIR / "data" / "bronze" / source_name


def get_silver_path() -> Path:
    return PRODUCTION_DIR / "data" / "silver" / f"silver_featureset_{COMPOSITE_NAME}"


def get_gold_path() -> Path:
    return PRODUCTION_DIR / "data" / "gold" / f"gold_features_{COMPOSITE_NAME}"


get_feast_data_path = get_gold_path


# Fit mode configuration for training vs scoring separation
FIT_MODE = {{ 'True' if config.fit_mode else 'False' }}
ARTIFACTS_PATH = {{ '"%s"' % config.artifacts_path if config.artifacts_path else 'str(PRODUCTION_DIR / "artifacts" / (RECOMMENDATIONS_HASH or "default"))' }}

RAW_SOURCES = {
{% for name, landing in config.landing.items() %}
    "{{ name }}": {
        "path": "{{ landing.raw_source_path }}",
        "format": "{{ landing.raw_source_format }}",
        "entity_key": "{{ landing.entity_column }}",
        "time_column": "{{ landing.time_column }}",
    },
{% endfor %}
}

EXCLUDED_SOURCES = [
{% for source in config.sources %}
{% if source.excluded %}
    "{{ source.name }}",
{% endif %}
{% endfor %}
]

EXPLORATION_ARTIFACTS = {
    "bronze": {name: str(EXPERIMENTS_DIR / "data" / "bronze" / name) for name in SOURCES},
    "silver": str(EXPERIMENTS_DIR / "data" / "silver" / f"silver_featureset_{COMPOSITE_NAME}"),
    "gold": str(EXPERIMENTS_DIR / "data" / "gold" / f"gold_features_{COMPOSITE_NAME}"),
    "scoring": str(EXPERIMENTS_DIR / "data" / "scoring" / "predictions"),
}
""",
    "bronze.py.j2": """import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.transformations, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_timestamp, safe_to_datetime, timedelta_to_days
from config import SOURCES, get_bronze_path{{ ', RAW_SOURCES' if config.lifecycle else '' }}

SOURCE_NAME = "{{ source }}"


def load_{{ source }}():
    source_config = SOURCES[SOURCE_NAME]
    path = Path(source_config["path"])
    if not path.exists():
        raise FileNotFoundError(f"Source file not found: {path}")
    if source_config["format"] == "csv":
        return pd.read_csv(str(path))
    if source_config["format"] == "parquet":
        return pd.read_parquet(str(path))
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(path))


{% set groups = group_steps(config.transformations) %}

def apply_transformations(df: pd.DataFrame) -> pd.DataFrame:
{%- if groups %}
{%- for func_name, steps in groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}

{% if config.lifecycle %}

ENTITY_COLUMN = "{{ config.entity_column or config.source.entity_key }}"
TIME_COLUMN = "{{ config.time_column or config.source.time_column }}"


def _load_raw_events():
    source = RAW_SOURCES[SOURCE_NAME]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(str(path))
    if source["format"] == "parquet":
        return pd.read_parquet(str(path))
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(path))

{% if config.lifecycle.include_recency_bucket %}

def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    reference_date = raw_df[TIME_COLUMN].max()
    _grp = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN]
    entity_stats = _grp.min().to_frame("_time_min")
    entity_stats["_time_max"] = _grp.max()
    entity_stats["days_since_last"] = timedelta_to_days(reference_date - entity_stats["_time_max"])
    entity_stats["days_since_first"] = timedelta_to_days(reference_date - entity_stats["_time_min"])
    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df


def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_last" in df.columns:
        df["recency_bucket"] = pd.cut(df["days_since_last"], bins={{ config.lifecycle.recency_bucket_edges }} + [float("inf")],
                                       labels={{ config.lifecycle.recency_bucket_labels }})
    return df

{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}

def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_first" not in df.columns:
        return df
    tenure = df["days_since_first"]
    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
    if not intensity_col:
        return df
    intensity = df[intensity_col[0]]
    tenure_med = tenure.median()
    intensity_med = intensity.median()
    conditions = [
        (tenure >= tenure_med) & (intensity >= intensity_med),
        (tenure >= tenure_med) & (intensity < intensity_med),
        (tenure < tenure_med) & (intensity >= intensity_med),
        (tenure < tenure_med) & (intensity < intensity_med),
    ]
    labels = ["steady_loyal_lifecycle", "occasional_loyal_lifecycle", "intense_brief_lifecycle", "one_shot_lifecycle"]
    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
    return df

{% endif %}
{% if config.lifecycle.include_cyclical_features %}

def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
    df = df.drop(columns=["mean_dow"], errors="ignore")
    return df

{% endif %}
{% if config.lifecycle.include_month_cyclical %}

def add_month_quarter_cyclical(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    mean_month = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.month.mean())
    df = df.merge(mean_month.rename("mean_month"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["month_sin"] = np.sin(2 * np.pi * df["mean_month"] / 12)
    df["month_cos"] = np.cos(2 * np.pi * df["mean_month"] / 12)
{% if config.lifecycle.include_quarter_cyclical %}
    mean_quarter = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: ((x.dt.month - 1) // 3).mean())
    df = df.merge(mean_quarter.rename("mean_quarter"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["quarter_sin"] = np.sin(2 * np.pi * df["mean_quarter"] / 4)
    df["quarter_cos"] = np.cos(2 * np.pi * df["mean_quarter"] / 4)
    df = df.drop(columns=["mean_month", "mean_quarter"], errors="ignore")
{% else %}
    df = df.drop(columns=["mean_month"], errors="ignore")
{% endif %}
    return df

{% endif %}
{% if config.lifecycle.include_trend_features %}

def add_trend_features(df: pd.DataFrame) -> pd.DataFrame:
    window_cols = sorted([c for c in df.columns if c.startswith("event_count_") and c != "event_count_all_time"])
    all_time_col = "event_count_all_time" if "event_count_all_time" in df.columns else None
    if window_cols and all_time_col:
        df["recent_vs_overall_ratio"] = df[window_cols[0]] / df[all_time_col].replace(0, float("nan"))
    if len(window_cols) >= 2:
        window_values = df[window_cols].values
        x = np.arange(len(window_cols), dtype=float)
        slopes = np.array([np.polyfit(x, row, 1)[0] if not np.any(np.isnan(row)) else 0.0 for row in window_values])
        df["entity_trend_slope"] = slopes
    return df

{% endif %}
{% if config.lifecycle.include_cohort_features %}

def add_cohort_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    first_event = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].min()
    cohort_data = pd.DataFrame({"first_event": first_event})
    cohort_data["cohort_year"] = cohort_data["first_event"].dt.year
    cohort_data["cohort_quarter"] = ((cohort_data["first_event"].dt.month - 1) // 3 + 1)
    df = df.merge(cohort_data[["cohort_year", "cohort_quarter"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df

{% endif %}
{% if config.lifecycle.momentum_pairs %}

def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
{% for pair in config.lifecycle.momentum_pairs %}
    short_col = "event_count_{{ pair.short_window }}"
    long_col = "event_count_{{ pair.long_window }}"
    if short_col in df.columns and long_col in df.columns:
        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
{% endfor %}
    return df

{% endif %}

def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
    raw_df = _load_raw_events()
{% if config.raw_time_column %}
    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.lifecycle.include_recency_bucket %}
    df = add_recency_tenure(df, raw_df)
    df = add_recency_buckets(df)
{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}
    df = add_lifecycle_quadrant(df)
{% endif %}
{% if config.lifecycle.include_cyclical_features %}
    df = add_cyclical_features(df, raw_df)
{% endif %}
{% if config.lifecycle.include_month_cyclical %}
    df = add_month_quarter_cyclical(df, raw_df)
{% endif %}
{% if config.lifecycle.include_trend_features %}
    df = add_trend_features(df)
{% endif %}
{% if config.lifecycle.include_cohort_features %}
    df = add_cohort_features(df, raw_df)
{% endif %}
{% if config.lifecycle.momentum_pairs %}
    df = add_momentum_ratios(df)
{% endif %}
    return df
{% endif %}

{% if config.text_features %}

def compute_text_features_entity(df):
    from customer_retention.stages.profiling.text_processor import TextColumnProcessor, TextProcessingConfig
{% for tf in config.text_features %}
    if "{{ tf.column }}" in df.columns:
        processor = TextColumnProcessor(TextProcessingConfig(embedding_model="{{ tf.embedding_model }}"), registry=None)
        df, result = processor.process_column(df, "{{ tf.column }}", fit=True)
{% endfor %}
    return df
{% endif %}


def run_bronze_entity_{{ source }}():
    df = load_{{ source }}()
    df = apply_transformations(df)
{% if config.lifecycle %}
    df = enrich_lifecycle(df)
{% endif %}
{% if config.text_features %}
    df = compute_text_features_entity(df)
{% endif %}
    output_path = get_bronze_path(SOURCE_NAME)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    _delta = get_delta(force_local=True)
    _delta.write(df, str(output_path))
    _z_cols = [c for c in ["{{ config.source.entity_key }}"] if c in df.columns]
    if _z_cols:
        _delta.optimize(str(output_path), _z_cols)
    else:
        _delta.optimize(str(output_path))
    return df


if __name__ == "__main__":
    run_bronze_entity_{{ source }}()
""",
    "silver.py.j2": '''import pandas as pd
from pathlib import Path
{% set ops, fitted = collect_imports(config.silver.derived_columns, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
{% if config.silver.grid_dates %}
from customer_retention.stages.temporal.temporal_merger import TemporalMerger, MergeConfig, DatasetMergeInput
from customer_retention.core.config.column_config import DatasetGranularity
{% endif %}
{% set has_key_resolution = config.silver.merge_sources | selectattr('key_resolution_steps') | list | length > 0 %}
{% if has_key_resolution %}
from customer_retention.analysis.auto_explorer.key_resolver import resolve_entity_keys
from customer_retention.analysis.auto_explorer.project_context import KeyResolutionStep
{% endif %}
from config import SOURCES, get_bronze_path, get_silver_path, TARGET_COLUMN

{% if config.silver.grid_dates %}
GRID_DATES = {{ config.silver.grid_dates }}

MERGE_SOURCE_META = [
{% for src in config.silver.merge_sources %}
    {"name": "{{ src.name }}", "granularity": "{{ src.granularity }}"{{ ', "feature_timestamp_column": "' + src.feature_timestamp_column + '"' if src.feature_timestamp_column else '' }}, "key_resolution_steps": [{% for kr in src.key_resolution_steps %}{"bridge_dataset": "{{ kr.bridge_dataset }}", "source_key": "{{ kr.source_key }}", "bridge_key": "{{ kr.bridge_key }}", "resolve_column": "{{ kr.resolve_column }}"}{{ ", " if not loop.last else "" }}{% endfor %}]},
{% endfor %}
]
{% endif %}


def _load_artifact(path):
    from customer_retention.integrations.adapters.factory import get_delta
    df = get_delta(force_local=True).read(str(path))
    ref_path = Path(str(path) + ".reference_date")
    if ref_path.exists():
        df.attrs["aggregation_reference_date"] = ref_path.read_text().strip()
    return df


def _bronze_output_name(name: str) -> str:
    return f"{name}_aggregated" if SOURCES[name].get("is_event_level") else name


def load_bronze_outputs() -> dict:
    return {name: _load_artifact(get_bronze_path(_bronze_output_name(name)))
            for name in SOURCES if not SOURCES[name].get("excluded")}


{% if config.silver.grid_dates %}
def merge_sources(bronze_outputs: dict) -> pd.DataFrame:
    raw_entity_key = "{{ config.silver.entity_key or config.sources[0].entity_key }}"
    base_source = "{{ config.sources[0].name }}"
    entity_ids = bronze_outputs[base_source][raw_entity_key].unique()
{% if has_key_resolution %}
    resolutions = {}
    for meta in MERGE_SOURCE_META:
        kr_steps = meta.get("key_resolution_steps", [])
        if kr_steps:
            resolutions[meta["name"]] = [
                KeyResolutionStep(**s) for s in kr_steps
            ]
    if resolutions:
        bronze_outputs = resolve_entity_keys(bronze_outputs, resolutions)
{% endif %}
    for name, df in bronze_outputs.items():
        if raw_entity_key in df.columns and raw_entity_key != "entity_id":
            bronze_outputs[name] = df.rename(columns={raw_entity_key: "entity_id"})
    merger = TemporalMerger(MergeConfig(entity_key="entity_id"))
    spine = merger.build_spine(entity_ids, GRID_DATES)
    inputs = []
    for meta in MERGE_SOURCE_META:
        name = meta["name"]
        if name not in bronze_outputs:
            continue
        granularity = DatasetGranularity(meta["granularity"])
        inputs.append(DatasetMergeInput(
            name=name,
            df=bronze_outputs[name],
            granularity=granularity,
            feature_timestamp_column=meta.get("feature_timestamp_column"),
        ))
    merged, _report = merger.merge_all(spine, inputs)
    return merged
{% else %}
def merge_sources(bronze_outputs: dict) -> pd.DataFrame:
    raw_entity_key = "{{ config.silver.entity_key or config.sources[0].entity_key }}"
    base_source = "{{ config.sources[0].name }}"
    merged = bronze_outputs[base_source]
{% for join in config.silver.joins %}
    merged = merged.merge(
        bronze_outputs["{{ join.right_source }}"],
        left_on={{ join.left_keys }},
        right_on={{ join.right_keys }},
        how="{{ join.how }}"
    )
{% endfor %}
    if raw_entity_key in merged.columns and raw_entity_key != "entity_id":
        merged = merged.rename(columns={raw_entity_key: "entity_id"})
    return merged
{% endif %}


def create_holdout_mask(df: pd.DataFrame, holdout_fraction: float = 0.1, random_state: int = 42) -> pd.DataFrame:
    """Create entity-level holdout by masking target for a fraction of entities.

    Samples entities (not rows) to avoid capturing nearly all entities when
    temporal data has multiple rows per entity. All rows for holdout entities
    get their target masked. This is entity-aware per Architecture Decision #6.

    IMPORTANT: This must happen in the silver layer (BEFORE gold layer feature
    computation) to prevent temporal leakage.
    """
    ORIGINAL_COLUMN = f"original_{TARGET_COLUMN}"

    if ORIGINAL_COLUMN in df.columns:
        print(f"  Holdout already exists ({ORIGINAL_COLUMN}), skipping creation")
        return df

    if TARGET_COLUMN not in df.columns:
        print(f"  Warning: TARGET_COLUMN \\'{TARGET_COLUMN}\\' not found, skipping holdout creation")
        return df

    print(f"Creating holdout set ({holdout_fraction:.0%} of entities)...")
    df = df.copy()

    entity_ids = df["entity_id"].drop_duplicates()
    n_holdout_entities = max(1, int(len(entity_ids) * holdout_fraction))
    holdout_entities = entity_ids.sample(n=n_holdout_entities, random_state=random_state)
    holdout_mask = df["entity_id"].isin(holdout_entities)

    df[ORIGINAL_COLUMN] = pd.NA
    df.loc[holdout_mask, ORIGINAL_COLUMN] = df.loc[holdout_mask, TARGET_COLUMN]
    df.loc[holdout_mask, TARGET_COLUMN] = pd.NA

    n_holdout_rows = int(holdout_mask.sum())
    print(f"  Holdout entities: {n_holdout_entities:,} / {len(entity_ids):,} ({holdout_fraction:.0%})")
    print(f"  Holdout rows: {n_holdout_rows:,}, Training rows: {len(df) - n_holdout_rows:,}")

    return df


{% set derived_groups = group_steps(config.silver.derived_columns) %}

def create_derived_columns(df: pd.DataFrame) -> pd.DataFrame:
{%- if derived_groups %}
{%- for func_name, steps in derived_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in derived_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for dc in steps %}
    # {{ dc.rationale }}
    # {{ action_description(dc) }}
    df = {{ render_step_call(dc) }}
{%- endfor %}
    return df
{% endfor %}


def run_silver_merge(create_holdout: bool = True, holdout_fraction: float = 0.1):
    bronze_outputs = load_bronze_outputs()
    ref_date = next(
        (v.attrs["aggregation_reference_date"] for v in bronze_outputs.values()
         if "aggregation_reference_date" in v.attrs),
        None,
    )
    silver = merge_sources(bronze_outputs)
    silver = create_derived_columns(silver)

    if create_holdout:
        silver = create_holdout_mask(silver, holdout_fraction=holdout_fraction)

    if ref_date:
        silver.attrs["aggregation_reference_date"] = ref_date

    output_path = get_silver_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    _delta = get_delta(force_local=True)
    _delta.write(silver, str(output_path))
    _z_cols = [c for c in ["entity_id", "as_of_date"] if c in silver.columns]
    if _z_cols:
        _delta.optimize(str(output_path), _z_cols)
    else:
        _delta.optimize(str(output_path))
    if ref_date:
        Path(str(output_path) + ".reference_date").write_text(ref_date)
    return silver


if __name__ == "__main__":
    run_silver_merge()
''',
    "gold.py.j2": """import pandas as pd
import warnings
from datetime import datetime
from pathlib import Path
{% set parts = partition_gold_steps(config.gold) %}
{% set fitted_steps = parts['fitted_transforms'] + parts['fitted_encodings'] + parts['fitted_scalings'] %}
{% set stateless_steps = parts['stateless_transforms'] + parts['stateless_encodings'] %}
{% set all_gold_steps = config.gold.transformations + config.gold.encodings + config.gold.scalings %}
{% set ops, fitted = collect_imports(all_gold_steps, True) %}
{% set fs_ops = ['apply_feature_select'] if config.gold.feature_selections else [] %}
from customer_retention.transforms import ArtifactStore{{ (', ' + (ops | sort | join(', '))) if ops }}{{ (', ' + (fs_ops | join(', '))) if fs_ops and 'apply_feature_select' not in ops }}
{% if fitted %}
from customer_retention.transforms.fitted import {{ fitted | sort | join(', ') }}
{% endif %}
from config import (get_silver_path, get_gold_path, get_feast_data_path,
                    TARGET_COLUMN, TIMESTAMP_COLUMN, RECOMMENDATIONS_HASH, FEAST_REPO_PATH,
                    FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, EXPERIMENTS_DIR,
                    ARTIFACTS_PATH, FIT_MODE)

{% if not fitted_steps %}
{% if config.fit_mode %}
_store = ArtifactStore(Path(ARTIFACTS_PATH))
{% else %}
_store = ArtifactStore.from_manifest(Path(ARTIFACTS_PATH) / "manifest.yaml")
{% endif %}
{% endif %}

from customer_retention.generators.pipeline_generator.models import (
    PipelineTransformationType,
    TransformationStep,
)

ENCODINGS = [
{% for enc in config.gold.encodings %}
    TransformationStep(type=PipelineTransformationType.ENCODE, column="{{ enc.column }}", parameters={{ enc.parameters }}, rationale="{{ enc.rationale }}"),
{% endfor %}
]

SCALINGS = [
{% for scale in config.gold.scalings %}
    TransformationStep(type=PipelineTransformationType.SCALE, column="{{ scale.column }}", parameters={{ scale.parameters }}, rationale="{{ scale.rationale }}"),
{% endfor %}
]


def load_silver() -> pd.DataFrame:
    from customer_retention.integrations.adapters.factory import get_delta
    path = get_silver_path()
    df = get_delta(force_local=True).read(str(path))
    ref_path = Path(str(path) + ".reference_date")
    if ref_path.exists():
        df.attrs["aggregation_reference_date"] = ref_path.read_text().strip()
    return df


def load_gold() -> pd.DataFrame:
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(get_gold_path()))


{% set transform_groups = group_steps(parts['stateless_transforms']) %}

def apply_gold_transformations(df: pd.DataFrame) -> pd.DataFrame:
{%- if transform_groups %}
{%- for func_name, steps in transform_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in transform_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t, config.fit_mode) }}
{%- endfor %}
    return df
{% endfor %}


def apply_encodings(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(parts['stateless_encodings']) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- if parts['stateless_encodings'] %}
{%- for enc in parts['stateless_encodings'] %}
    # {{ enc.rationale }}
    # {{ action_description(enc) }}
    df = {{ render_step_call(enc, config.fit_mode) }}
{%- endfor %}
{%- endif %}
    return df


def apply_scaling(df: pd.DataFrame) -> pd.DataFrame:
    return df


def apply_feature_selection(df: pd.DataFrame) -> pd.DataFrame:
{% if config.gold.feature_selections %}
{% for fs in config.gold.feature_selections %}
    df = apply_feature_select(df, '{{ fs }}')
{% endfor %}
{% endif %}
    return df

{% if fitted_steps %}

def apply_fitted_transforms(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(fitted_steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{% if config.fit_mode %}
    _store = ArtifactStore(Path(ARTIFACTS_PATH))
    from customer_retention.core.compat import temporal_quantile
    from datetime import timedelta
    _ts_col = FEAST_TIMESTAMP_COL
    if _ts_col not in df.columns:
        for _candidate in ["as_of_date", "feature_timestamp", "event_timestamp"]:
            if _candidate in df.columns:
                _ts_col = _candidate
                break
    _cutoff = temporal_quantile(df[_ts_col], 1 - {{ config.training.test_size if config.training else 0.2 }})
{% if config.training and config.training.purge_gap_days %}
    _train_mask = df[_ts_col] < (_cutoff - timedelta(days={{ config.training.purge_gap_days }}))
{% else %}
    _train_mask = df[_ts_col] < _cutoff
{% endif %}
    _fit_subset = df[_train_mask].copy()
{%- for step in fitted_steps %}
    # {{ step.rationale }}
    # {{ action_description(step) }}
    _fit_subset = {{ render_step_call(step, fit_mode=True) }}
{%- endfor %}
    del _fit_subset
{%- for step in fitted_steps %}
    df = {{ render_step_call(step, fit_mode=False) }}
{%- endfor %}
    _store.save_manifest()
    print(f"Fit artifacts saved to: {ARTIFACTS_PATH} (fitted on {_train_mask.sum():,} train rows, transforming {len(df):,} total)")
{% else %}
    _store = ArtifactStore.from_manifest(Path(ARTIFACTS_PATH) / "manifest.yaml")
{%- for step in fitted_steps %}
    # {{ step.rationale }}
    # {{ action_description(step) }}
    df = {{ render_step_call(step, fit_mode=False) }}
{%- endfor %}
{% endif %}
    return df
{% endif %}


def get_feature_version_tag() -> str:
    if RECOMMENDATIONS_HASH:
        return f"v1.0.0_{RECOMMENDATIONS_HASH}"
    return "v1.0.0"


def add_feast_timestamp(df: pd.DataFrame, reference_date=None) -> pd.DataFrame:
    if FEAST_TIMESTAMP_COL in df.columns:
        return df
    if "as_of_date" in df.columns:
        return df.rename(columns={"as_of_date": FEAST_TIMESTAMP_COL})
    if "feature_timestamp" in df.columns:
        return df.rename(columns={"feature_timestamp": FEAST_TIMESTAMP_COL})
    if "aggregation_reference_date" in df.attrs:
        timestamp = pd.to_datetime(df.attrs["aggregation_reference_date"])
        print(f"  Using aggregation reference_date for Feast timestamp: {timestamp}")
    elif reference_date is not None:
        timestamp = reference_date
        print(f"  Using provided reference_date for Feast timestamp: {timestamp}")
    else:
        timestamp = datetime.now()
        warnings.warn(
            f"No reference_date available for Feast timestamp. Using datetime.now() ({timestamp}). "
            "This may cause temporal leakage - features should use actual aggregation dates. "
            "Set aggregation_reference_date in DataFrame.attrs during aggregation.",
            UserWarning
        )
    df[FEAST_TIMESTAMP_COL] = timestamp
    return df


def run_gold_features():
    silver = load_silver()
    gold = apply_gold_transformations(silver)
    gold = apply_encodings(gold)
    gold = apply_scaling(gold)
    gold = apply_feature_selection(gold)
{% if fitted_steps %}
    gold = apply_fitted_transforms(gold)
{% endif %}
    gold = add_feast_timestamp(gold)
{% if not fitted_steps and config.fit_mode %}
    _store.save_manifest()
    print(f"Fit artifacts saved to: {ARTIFACTS_PATH}")
{% endif %}
    output_path = get_gold_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    gold.attrs["recommendations_hash"] = RECOMMENDATIONS_HASH
    gold.attrs["feature_version"] = get_feature_version_tag()
    from customer_retention.integrations.adapters.factory import get_delta
    _delta = get_delta(force_local=True)
    _delta.write(gold, str(output_path))
    _z_cols = [c for c in ["entity_id", TIMESTAMP_COLUMN] if c in gold.columns]
    if _z_cols:
        _delta.optimize(str(output_path), _z_cols)
    else:
        _delta.optimize(str(output_path))
    print(f"Gold features saved with version: {get_feature_version_tag()}")
    return gold


if __name__ == "__main__":
    run_gold_features()
""",
    "training.py.j2": '''import logging
import numpy as np
import pandas as pd
from datetime import datetime
import mlflow
import mlflow.sklearn
import mlflow.xgboost
import xgboost as xgb
from pathlib import Path
from feast import FeatureStore
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import (roc_auc_score, average_precision_score, f1_score,
                             precision_score, recall_score, accuracy_score)
from customer_retention.stages.modeling.data_splitter import DataSplitter, SplitStrategy
from customer_retention.stages.modeling.cross_validator import CrossValidator, CVStrategy
from customer_retention.stages.modeling.feature_profile import FeatureProfile, ColumnProfile, build_feature_profile, compare_feature_profiles
from customer_retention.analysis.auto_explorer.run_namespace import RunNamespace
from customer_retention.core.compat.timing import log_timing
{% if config.training and config.training.imbalance_strategy == "smote" %}
from customer_retention.stages.modeling.imbalance_handler import ImbalanceHandler, ImbalanceStrategy
{% endif %}
from config import (TARGET_COLUMN, PIPELINE_NAME, COMPOSITE_NAME, RECOMMENDATIONS_HASH, MLFLOW_TRACKING_URI,
                    MLFLOW_ARTIFACT_ROOT, FEAST_REPO_PATH, FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY,
                    FEAST_TIMESTAMP_COL, EXPERIMENTS_DIR, get_feast_data_path)

mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
logger = logging.getLogger("training")
_EXCLUDE_COLS = {TARGET_COLUMN, FEAST_TIMESTAMP_COL, FEAST_ENTITY_KEY, "as_of_date", "feature_timestamp", "label_timestamp", "label_available_flag"}
{% if config.training and config.training.exploration_feature_profile %}
_EXPLORATION_PROFILE = {{ config.training.exploration_feature_profile }}
{% else %}
_EXPLORATION_PROFILE = None
{% endif %}
_NAMESPACE = RunNamespace.from_env_or_latest(EXPERIMENTS_DIR)

def _assert_rows(count, stage):
    if count == 0:
        raise ValueError(f"[TRAINING] {stage}: 0 rows remaining — cannot proceed")
    return count


def _load_feast_data():
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(get_feast_data_path()))


def get_training_data_from_feast() -> pd.DataFrame:
    """Retrieve training data from Feast for training/serving consistency.

    Uses get_historical_features for point-in-time correct feature retrieval.
    This ensures training uses the exact same feature retrieval path as inference.
    """
    feast_path = Path(FEAST_REPO_PATH)

    if not (feast_path / "feature_store.yaml").exists():
        print("Feast repo not initialized, falling back to data file")
        return _load_feast_data()

    try:
        store = FeatureStore(repo_path=str(feast_path))

        features_df = _load_feast_data()

        entity_df = features_df[[FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL]].copy()

        exclude_cols = {FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, TARGET_COLUMN}
        feature_cols = [c for c in features_df.columns
                        if c not in exclude_cols and not c.startswith("original_")]

        feature_refs = [f"{FEAST_FEATURE_VIEW}:{col}" for col in feature_cols]

        print(f"Retrieving {len(feature_refs)} features from Feast...")
        print(f"  Feature view: {FEAST_FEATURE_VIEW}")
        print(f"  Entity key: {FEAST_ENTITY_KEY}")

        training_df = store.get_historical_features(
            entity_df=entity_df,
            features=feature_refs
        ).to_df()

        training_df = training_df.merge(
            features_df[[FEAST_ENTITY_KEY, TARGET_COLUMN]],
            on=FEAST_ENTITY_KEY,
            how="left"
        )

        print(f"  Retrieved {len(training_df):,} rows, {len(training_df.columns)} columns")
        return training_df

    except (ImportError, ConnectionError, RuntimeError, KeyError) as e:
        import warnings
        warnings.warn(f"Feast retrieval failed ({e}), loading directly from gold data")
        return _load_feast_data()


def prepare_features(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()
    drop_cols = [c for c in df.columns if c in _EXCLUDE_COLS or c.startswith("original_")]
    df = df.drop(columns=[c for c in drop_cols if c in df.columns], errors="ignore")
    for col in df.select_dtypes(include=["object", "category"]).columns:
        df[col] = LabelEncoder().fit_transform(df[col].astype(str))
    return df.select_dtypes(include=["int64", "float64", "int32", "float32", "bool"]).fillna(0)


def compute_metrics(y_true, y_proba, y_pred) -> dict:
    both_classes = len(np.unique(y_true)) > 1
    return {
        "roc_auc": roc_auc_score(y_true, y_proba) if both_classes else 0.0,
        "pr_auc": average_precision_score(y_true, y_proba) if both_classes else 0.0,
        "f1": f1_score(y_true, y_pred, zero_division=0),
        "precision": precision_score(y_true, y_pred, zero_division=0),
        "recall": recall_score(y_true, y_pred, zero_division=0),
        "accuracy": accuracy_score(y_true, y_pred),
    }


def get_feature_importance(model, feature_names) -> pd.DataFrame:
    if hasattr(model, "feature_importances_"):
        importance = model.feature_importances_
    elif hasattr(model, "coef_"):
        importance = abs(model.coef_[0])
    else:
        return None
    df = pd.DataFrame({"feature": feature_names, "importance": importance})
    return df.sort_values("importance", ascending=False).reset_index(drop=True)


def log_feature_importance(model, feature_names):
    fi = get_feature_importance(model, feature_names)
    if fi is None:
        return
    fi.to_csv("feature_importance.csv", index=False)
    mlflow.log_artifact("feature_importance.csv")


def train_xgboost(X_train, y_train, X_test, y_test, feature_names):
    mlflow.xgboost.autolog(disable=True)
    dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=feature_names)
    dtest = xgb.DMatrix(X_test, label=y_test, feature_names=feature_names)
    params = {"objective": "binary:logistic", "eval_metric": ["auc", "logloss"],
              "max_depth": 6, "learning_rate": 0.1, "seed": 42}
    model = xgb.train(params, dtrain, num_boost_round=100,
                      evals=[(dtrain, "train"), (dtest, "eval")], verbose_eval=False)
    return model


def _on_fold(detail, fold_num, total_folds):
    print(f"  CV fold {fold_num}/{total_folds}: roc_auc={detail['score']:.4f} ({detail['elapsed_seconds']:.0f}s)", flush=True)


def get_model_name_with_hash(base_name: str) -> str:
    if RECOMMENDATIONS_HASH:
        return f"{base_name}_{RECOMMENDATIONS_HASH}"
    return base_name


def run_experiment():
    mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
    _experiment_name = f"training_{COMPOSITE_NAME}"
    experiment = mlflow.get_experiment_by_name(_experiment_name)
    if experiment is None:
        mlflow.create_experiment(_experiment_name, artifact_location=MLFLOW_ARTIFACT_ROOT)
    mlflow.set_experiment(_experiment_name)
    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
    print(f"Artifacts: {MLFLOW_ARTIFACT_ROOT}")

    with log_timing("load_gold_data", logger):
        training_data = get_training_data_from_feast()
    raw_count = _assert_rows(len(training_data), "gold_data")
    type_summary = dict(training_data.dtypes.value_counts())
    print(f"[TRAINING] Gold data: {raw_count:,} rows, {len(training_data.columns)} columns")
    print(f"[TRAINING] Column types: {type_summary}")

    with log_timing("prepare_features", logger):
        y = training_data[TARGET_COLUMN]
        X = prepare_features(training_data.drop(columns=[TARGET_COLUMN]))
        feature_names = list(X.columns)
    print(f"[TRAINING] Features: {len(feature_names)} columns after preparation")
    train_mask = y.notna()
    X, y = X.loc[train_mask], y.loc[train_mask]
    filtered_count = _assert_rows(len(X), "after_null_label_filter")
    print(f"[TRAINING] After null-label filter: {filtered_count:,} rows")

    with log_timing("feature_profile", logger):
        excluded_cols = {}
        for c in training_data.columns:
            if c in _EXCLUDE_COLS:
                excluded_cols[c] = "metadata"
            elif c.startswith("original_"):
                excluded_cols[c] = "original_prefix"
        feature_stats = {}
        for c in feature_names:
            null_count = int(X[c].isna().sum())
            feature_stats[c] = ColumnProfile(dtype=str(X[c].dtype), non_null_count=filtered_count - null_count, null_count=null_count)
        prod_profile = build_feature_profile("production", TARGET_COLUMN, filtered_count, feature_stats, excluded_cols)
        print(f"[TRAINING] Production profile: {prod_profile.feature_count} features, {filtered_count:,} rows")
        if _NAMESPACE is not None:
            prod_profile.save(_NAMESPACE.production_feature_profile_path)
            print(f"[TRAINING] Production profile saved to {_NAMESPACE.production_feature_profile_path}")
        if _EXPLORATION_PROFILE is not None:
            exp_profile = FeatureProfile.from_dict(_EXPLORATION_PROFILE)
            discrepancies = compare_feature_profiles(exp_profile, prod_profile)
            if discrepancies:
                print(f"[TRAINING] WARNING: {len(discrepancies)} feature discrepancies vs exploration:")
                for d in discrepancies:
                    print(f"[TRAINING]   {d}")
            else:
                print("[TRAINING] Feature profile matches exploration")
        else:
            print("[TRAINING] WARNING: No exploration feature profile available for comparison")
{% if config.training and config.training.recommended_training_start %}
    if FEAST_TIMESTAMP_COL in training_data.columns:
        time_mask = training_data.loc[train_mask, FEAST_TIMESTAMP_COL] >= pd.to_datetime("{{ config.training.recommended_training_start }}")
        X, y = X.loc[time_mask], y.loc[time_mask]
{% endif %}
{% if config.training and config.training.filter_future_dates %}
    if FEAST_TIMESTAMP_COL in training_data.columns:
        future_mask = training_data.loc[X.index, FEAST_TIMESTAMP_COL] <= datetime.now()
        X, y = X.loc[future_mask], y.loc[future_mask]
{% endif %}
    with log_timing("temporal_split", logger):
        splitter = DataSplitter(
            target_column=TARGET_COLUMN,
            strategy=SplitStrategy.TEMPORAL,
            temporal_column=FEAST_TIMESTAMP_COL,
{% if config.training and config.training.purge_gap_days %}
            purge_gap_days={{ config.training.purge_gap_days }},
{% endif %}
            test_size={{ config.training.test_size if config.training else 0.2 }},
        )
        split_df = training_data.loc[X.index].copy()
        split_df[TARGET_COLUMN] = y
        for col in X.columns:
            split_df[col] = X[col]
        splits = splitter.split(split_df)
    X_train, X_test = splits.X_train[feature_names], splits.X_test[feature_names]
    y_train, y_test = splits.y_train, splits.y_test
    _assert_rows(len(X_train), "train_set_after_split")
    _assert_rows(len(X_test), "test_set_after_split")
    print(f"[TRAINING] Split: train={len(X_train):,}, test={len(X_test):,}")
    print(f"[TRAINING] Split info: {splits.split_info}")
    label_dist = dict(y_train.value_counts())
    print(f"[TRAINING] Label distribution: {label_dist}")
    if y_train.nunique() < 2:
        raise ValueError(f"[TRAINING] Only {y_train.nunique()} class(es) — Need at least 2 for binary classification")
{% if config.training and config.training.imbalance_strategy == "smote" %}
    handler = ImbalanceHandler(strategy=ImbalanceStrategy.SMOTE)
    _imb_result = handler.fit_transform(X_train, y_train)
    X_train, y_train = _imb_result.X_resampled, _imb_result.y_resampled
{% endif %}

{% set class_weight_param = ', class_weight="balanced"' if config.training and config.training.imbalance_strategy == "class_weight" else '' %}
    sklearn_models = {
        "logistic_regression": LogisticRegression(max_iter=5000, random_state=42{{ class_weight_param }}),
        "random_forest": RandomForestClassifier(n_estimators=100, random_state=42{{ class_weight_param }}),
    }

    run_name = get_model_name_with_hash(f"training_{COMPOSITE_NAME}")
    with mlflow.start_run(run_name=run_name):
        mlflow.log_params({"train_samples": len(X_train), "test_samples": len(X_test), "n_features": X.shape[1]})
        mlflow.set_tag("feature_source", "feast")
        mlflow.set_tag("feast_feature_view", FEAST_FEATURE_VIEW)
        mlflow.set_tag("composite_name", COMPOSITE_NAME)
        mlflow.set_tag("pipeline_name", PIPELINE_NAME)
        if RECOMMENDATIONS_HASH:
            mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
        best_model, best_auc = None, -1

        for name, model in sklearn_models.items():
            with mlflow.start_run(run_name=name, nested=True):
                if RECOMMENDATIONS_HASH:
                    mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
                mlflow.set_tag("feature_source", "feast")
                with log_timing(f"fit_{name}", logger):
                    model.fit(X_train, y_train)
                y_proba = model.predict_proba(X_test)[:, 1]
                y_pred = model.predict(X_test)
                metrics = compute_metrics(y_test, y_proba, y_pred)
                model_artifact_name = get_model_name_with_hash(f"model_{name}")
                mlflow.sklearn.log_model(model, model_artifact_name)
                _cv = CrossValidator(strategy=CVStrategy.STRATIFIED_KFOLD, n_splits=5, scoring="roc_auc")
                _cv_result = _cv.run(model, X_train, y_train, on_fold_complete=_on_fold)
                print(f"  CV: {_cv_result.cv_mean:.4f} +/- {_cv_result.cv_std:.4f}", flush=True)
                mlflow.log_metrics({**metrics, "cv_mean": _cv_result.cv_mean, "cv_std": _cv_result.cv_std})
                log_feature_importance(model, feature_names)
                print(f"{name}: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}", flush=True)
                if metrics["roc_auc"] > best_auc:
                    best_auc, best_model = metrics["roc_auc"], name

        with mlflow.start_run(run_name="xgboost", nested=True):
            if RECOMMENDATIONS_HASH:
                mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
            mlflow.set_tag("feature_source", "feast")
            xgb_model = train_xgboost(X_train, y_train, X_test, y_test, feature_names)
            dtest = xgb.DMatrix(X_test, feature_names=feature_names)
            y_proba = xgb_model.predict(dtest)
            y_pred = (y_proba > 0.5).astype(int)
            metrics = compute_metrics(y_test, y_proba, y_pred)
            xgb_model_name = get_model_name_with_hash("model_xgboost")
            mlflow.xgboost.log_model(xgb_model, xgb_model_name)
            mlflow.log_metrics(metrics)
            importance = xgb_model.get_score(importance_type="gain")
            fi = pd.DataFrame({"feature": importance.keys(), "importance": importance.values()})
            fi = fi.sort_values("importance", ascending=False).reset_index(drop=True)
            fi.to_csv("feature_importance.csv", index=False)
            mlflow.log_artifact("feature_importance.csv")
            print(f"xgboost: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}")
            if metrics["roc_auc"] > best_auc:
                best_auc, best_model = metrics["roc_auc"], "xgboost"

        mlflow.set_tag("best_model", best_model)
        mlflow.log_metric("best_roc_auc", best_auc)
        print(f"Best: {best_model} (ROC-AUC={best_auc:.4f})")


if __name__ == "__main__":
    run_experiment()
''',
    "runner.py.j2": """import argparse
import sys
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, str(Path(__file__).parent))

from config import PIPELINE_NAME, COMPOSITE_NAME, EXPERIMENTS_DIR, PRODUCTION_DIR
{% for name in sorted_landing_names(config.landing) %}
from landing.landing_{{ name }} import run_landing_{{ name }}
{% endfor %}
{% for name in config.bronze %}
from bronze.bronze_entity_{{ name }} import run_bronze_entity_{{ name }}
{% endfor %}
{% for name in config.bronze_event %}
from bronze.bronze_event_{{ name }} import run_bronze_event_{{ name }}
from bronze.bronze_entity_{{ name }}_aggregated import run_bronze_entity_{{ name }}_aggregated
{% endfor %}
{% set cn = config.composite_name or config.name %}
from silver.silver_featureset_{{ cn }} import run_silver_merge
from gold.gold_features_{{ cn }} import run_gold_features
from training.ml_experiment import run_experiment


def setup_experiments_dir():
    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)


def run_pipeline(validate=False):
    print(f"Starting pipeline: {PIPELINE_NAME}")
    setup_experiments_dir()
{% if config.landing %}

    print("\\n[1/6] Landing (event sources)...")
{% for name in sorted_landing_names(config.landing) %}
    run_landing_{{ name }}()
{% endfor %}
    print("Landing complete")
    if validate:
        from validation.validate_pipeline import validate_landing
        validate_landing()
{% endif %}

    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze event...")
{% for name in config.bronze_event %}
    run_bronze_event_{{ name }}()
{% endfor %}
    print("Bronze event complete")

    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Bronze entity (parallel)...")
    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as executor:
        bronze_futures = [
{% for name in config.bronze %}
            executor.submit(run_bronze_entity_{{ name }}),
{% endfor %}
{% for name in config.bronze_event %}
            executor.submit(run_bronze_entity_{{ name }}_aggregated),
{% endfor %}
        ]
        for f in bronze_futures:
            f.result()
    print("Bronze entity complete")
    if validate:
        from validation.validate_pipeline import validate_bronze
        validate_bronze()

    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Silver...")
    run_silver_merge()
    print("Silver complete")
    if validate:
        from validation.validate_pipeline import validate_silver
        validate_silver()

    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Gold...")
    run_gold_features()
    print("Gold complete")
    if validate:
        from validation.validate_pipeline import validate_gold
        validate_gold()

    print("\\n[{{ '6/6' if config.landing else '4/4' }}] Training...")
    run_experiment()
    print("Training complete")
    if validate:
        from validation.validate_pipeline import validate_training
        validate_training()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--validate", action="store_true")
    args = parser.parse_args()
    run_pipeline(validate=args.validate)
""",
    "run_all.py.j2": '''"""{{ config.name }} - Pipeline Runner with MLflow UI

All artifacts (data, mlruns, feast) are stored in the experiments directory.
Override location with CR_EXPERIMENTS_DIR environment variable.
"""
import os
import sys
import webbrowser
import subprocess
import time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, str(Path(__file__).parent))

from config import PIPELINE_NAME, COMPOSITE_NAME, SOURCES, MLFLOW_TRACKING_URI, EXPERIMENTS_DIR, PRODUCTION_DIR, FINDINGS_DIR
{% for name in sorted_landing_names(config.landing) %}
from landing.landing_{{ name }} import run_landing_{{ name }}
{% endfor %}
{% for name in config.bronze %}
from bronze.bronze_entity_{{ name }} import run_bronze_entity_{{ name }}
{% endfor %}
{% for name in config.bronze_event %}
from bronze.bronze_event_{{ name }} import run_bronze_event_{{ name }}
from bronze.bronze_entity_{{ name }}_aggregated import run_bronze_entity_{{ name }}_aggregated
{% endfor %}
{% set cn = config.composite_name or config.name %}
from silver.silver_featureset_{{ cn }} import run_silver_merge
from gold.gold_features_{{ cn }} import run_gold_features
from training.ml_experiment import run_experiment


def setup_experiments_dir():
    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)
    print(f"Experiments directory: {EXPERIMENTS_DIR}")
    print(f"Production directory: {PRODUCTION_DIR}")
    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
    print(f"Findings directory: {FINDINGS_DIR}")


def run_landing():
{% for name in sorted_landing_names(config.landing) %}
    run_landing_{{ name }}()
{% endfor %}
    pass


def run_bronze_event():
{% for name in config.bronze_event %}
    run_bronze_event_{{ name }}()
{% endfor %}
    pass


def run_bronze_entity_parallel():
    bronze_funcs = [
{% for name in config.bronze %}
        run_bronze_entity_{{ name }},
{% endfor %}
{% for name in config.bronze_event %}
        run_bronze_entity_{{ name }}_aggregated,
{% endfor %}
    ]
    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as ex:
        list(ex.map(lambda f: f(), bronze_funcs))


def is_port_in_use(port):
    import socket
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        return s.connect_ex(('localhost', port)) == 0


def start_mlflow_ui():
    port = 5050
    if is_port_in_use(port):
        print(f"\\n⚠ Port {port} is already in use.")
        print(f"  Either mlflow is already running, or kill the old process:")
        print(f"  pkill -f 'mlflow ui'")
        print(f"\\n  Opening browser to existing server...")
        webbrowser.open(f"http://localhost:{port}")
        return None

    print(f"\\nStarting MLflow UI (tracking: {MLFLOW_TRACKING_URI})...")
    process = subprocess.Popen(
        ["mlflow", "ui", "--backend-store-uri", MLFLOW_TRACKING_URI, "--port", str(port)],
        stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
    )
    for _ in range(10):
        time.sleep(1)
        if process.poll() is not None:
            err = process.stderr.read().decode() if process.stderr else ""
            print(f"\\n✗ MLflow UI failed to start (exit code {process.returncode})")
            if err:
                print(err)
            return None
        if is_port_in_use(port):
            break
    webbrowser.open(f"http://localhost:{port}")
    print(f"MLflow UI running at http://localhost:{port}")
    print("Press Ctrl+C to stop")
    return process


def run_pipeline():
    print(f"Running {PIPELINE_NAME}")
    print("=" * 50)

    setup_experiments_dir()
{% if config.landing %}

    print("\\n[1/6] Landing (event sources)...")
    run_landing()
    print("Landing complete")
{% endif %}

    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze event...")
    run_bronze_event()
    print("Bronze event complete")

    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Bronze entity (parallel)...")
    run_bronze_entity_parallel()
    print("Bronze entity complete")

    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Silver...")
    run_silver_merge()
    print("Silver complete")

    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Gold...")
    run_gold_features()
    print("Gold complete")

    print("\\n[{{ '6/6' if config.landing else '5/4' }}] Training...")
    run_experiment()
    print("Training complete")

    print("\\n" + "=" * 50)
    print("Pipeline finished!")

    mlflow_process = start_mlflow_ui()
    if mlflow_process:
        try:
            mlflow_process.wait()
        except KeyboardInterrupt:
            mlflow_process.terminate()
            print("\\nMLflow UI stopped")


if __name__ == "__main__":
    run_pipeline()
''',
    "workflow.json.j2": """{
{% set cn = config.composite_name or config.name %}
  "name": "{{ config.name }}_pipeline",
  "tasks": [
{% for name, lcfg in config.landing.items() %}
    {
      "task_key": "landing_{{ name }}",
{% if lcfg.key_resolution_steps %}
      "depends_on": [
{% for step in lcfg.key_resolution_steps %}
{% if step.bridge_dataset in config.landing %}
        {"task_key": "landing_{{ step.bridge_dataset }}"}{{ "," if not loop.last else "" }}
{% endif %}
{% endfor %}
      ],
{% endif %}
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/landing/landing_{{ name }}"
      }
    },
{% endfor %}
{% for name in config.bronze_event %}
    {
      "task_key": "bronze_event_{{ name }}",
{% if config.landing %}
      "depends_on": [
{% for lname in config.landing %}
        {"task_key": "landing_{{ lname }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
{% endif %}
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_event_{{ name }}"
      }
    },
    {
      "task_key": "bronze_entity_{{ name }}_aggregated",
      "depends_on": [{"task_key": "bronze_event_{{ name }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_entity_{{ name }}_aggregated"
      }
    },
{% endfor %}
{% for name in config.bronze %}
    {
      "task_key": "bronze_entity_{{ name }}",
{% if config.landing %}
      "depends_on": [
{% for lname in config.landing %}
        {"task_key": "landing_{{ lname }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
{% endif %}
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_entity_{{ name }}"
      }
    },
{% endfor %}
    {
      "task_key": "silver_featureset_{{ cn }}",
      "depends_on": [
{% for name in config.bronze_event %}
        {"task_key": "bronze_entity_{{ name }}_aggregated"},
{% endfor %}
{% for name in config.bronze %}
        {"task_key": "bronze_entity_{{ name }}"}{{ "," if not loop.last else "" }}
{% endfor %}
      ],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/silver/silver_featureset_{{ cn }}"
      }
    },
    {
      "task_key": "gold_features_{{ cn }}",
      "depends_on": [{"task_key": "silver_featureset_{{ cn }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/gold/gold_features_{{ cn }}"
      }
    },
    {
      "task_key": "ml_experiment",
      "depends_on": [{"task_key": "gold_features_{{ cn }}"}],
      "notebook_task": {
        "notebook_path": "/Workspace/orchestration/{{ config.name }}/training/ml_experiment"
      }
    }
  ]
}
""",
    "feature_store.yaml.j2": """project: {{ config.name }}
registry: data/registry.db
provider: local
online_store:
  type: sqlite
  path: data/online_store.db
offline_store:
  type: file
entity_key_serialization_version: 2
""",
    "features.py.j2": """from datetime import timedelta
from feast import Entity, FeatureView, Field, FileSource
from feast.types import Float32, Float64, Int64, String

{% set fv_name = config.feast.feature_view_name if config.feast else 'featureset_' + (config.composite_name or config.name) %}
{% set entity_name = config.feast.entity_name if config.feast else 'customer' %}
{% set entity_key = config.feast.entity_key if config.feast else config.sources[0].entity_key %}
{% set ts_col = config.feast.timestamp_column if config.feast else 'event_timestamp' %}
{% set ttl = config.feast.ttl_days if config.feast else 365 %}

{{ entity_name }} = Entity(
    name="{{ entity_name }}",
    join_keys=["{{ entity_key }}"],
)

{{ fv_name }}_source = FileSource(
    path="../data/gold/gold_features_{{ config.composite_name or config.name }}",
    timestamp_field="{{ ts_col }}"
)

{{ fv_name }} = FeatureView(
    name="{{ fv_name }}",
    entities=[{{ entity_name }}],
    ttl=timedelta(days={{ ttl }}),
    source={{ fv_name }}_source,
    tags={
        "pipeline": "{{ config.name }}",
        "composite_name": "{{ config.composite_name or '' }}",
        "recommendations_hash": "{{ config.recommendations_hash or 'none' }}",
    }
)
""",
    "landing.py.j2": """import pandas as pd
import numpy as np
from pathlib import Path
from customer_retention.core.compat import safe_to_datetime
from config import RAW_SOURCES, PRODUCTION_DIR

SOURCE_NAME = "{{ name }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"
TARGET_COLUMN = "{{ config.target_column }}"


def load_raw_data() -> pd.DataFrame:
    source = RAW_SOURCES[SOURCE_NAME]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(str(path))
    if source["format"] == "parquet":
        return pd.read_parquet(str(path))
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(path))


def derive_feature_timestamp(df: pd.DataFrame) -> pd.DataFrame:
{% if config.timestamp_coalesce %}
{% set cols = config.timestamp_coalesce.datetime_columns_ordered %}
    df["feature_timestamp"] = safe_to_datetime(df["{{ cols[-1] }}"], errors="coerce")
{% for col in cols[:-1] | reverse %}
    df["feature_timestamp"] = df["feature_timestamp"].fillna(safe_to_datetime(df["{{ col }}"], errors="coerce"))
{% endfor %}
{% else %}
    if TIME_COLUMN in df.columns:
        df["feature_timestamp"] = safe_to_datetime(df[TIME_COLUMN], errors="coerce")
    elif "feature_timestamp" not in df.columns:
        raise KeyError(f"Time column '{TIME_COLUMN}' not found in columns: {list(df.columns)}")
{% endif %}
    return df


def derive_label_timestamp(df: pd.DataFrame) -> pd.DataFrame:
{% if config.label_timestamp %}
{% set lt = config.label_timestamp %}
{% if lt.label_column %}
    df["label_timestamp"] = safe_to_datetime(df["{{ lt.label_column }}"], errors="coerce")
    df["label_timestamp"] = df["label_timestamp"].fillna(
        df["feature_timestamp"] + pd.Timedelta(days={{ lt.fallback_window_days }})
    )
{% else %}
    df["label_timestamp"] = df["feature_timestamp"] + pd.Timedelta(days={{ lt.fallback_window_days }})
{% endif %}
{% else %}
    df["label_timestamp"] = df["feature_timestamp"] + pd.Timedelta(days=180)
{% endif %}
    return df


def derive_label_available_flag(df: pd.DataFrame) -> pd.DataFrame:
    df["label_available_flag"] = df[TARGET_COLUMN].notna() if TARGET_COLUMN in df.columns else False
    return df

{% if config.datetime_derivation %}

def derive_datetime_features(df: pd.DataFrame) -> pd.DataFrame:
    from customer_retention.stages.profiling import derive_extra_datetime_features
    source_columns = {{ config.datetime_derivation.source_columns }}
    mask_future_columns = {{ config.datetime_derivation.mask_future_columns }}
    existing = [c for c in source_columns if c in df.columns]
    if existing:
        df, _ = derive_extra_datetime_features(
            df, "{{ config.datetime_derivation.reference_column }}", existing,
            mask_future_columns=[c for c in mask_future_columns if c in existing],
        )
    return df
{% endif %}


def derive_temporal_columns(df: pd.DataFrame) -> pd.DataFrame:
    df = derive_feature_timestamp(df)
    df = derive_label_timestamp(df)
    df = derive_label_available_flag(df)
{% if config.datetime_derivation %}
    df = derive_datetime_features(df)
{% endif %}
    return df


{% if config.history_window %}

def apply_history_window(df: pd.DataFrame) -> pd.DataFrame:
    time_col = "feature_timestamp"
{% if config.history_window.upper_limit %}
    upper = safe_to_datetime("{{ config.history_window.upper_limit }}")
{% else %}
    upper = df[time_col].max()
{% endif %}
{% if config.history_window.lookback_periods %}
    lookback_days = {{ config.history_window.lookback_periods }} * {{ config.history_window.cadence_days }}
    lower = upper - pd.Timedelta(days=lookback_days)
    df = df[df[time_col].isna() | (df[time_col] >= lower)]
{% endif %}
{% if config.history_window.upper_limit %}
    df = df[df[time_col].isna() | (df[time_col] <= upper)]
{% endif %}
    print(f"  History window: {len(df):,} records after filtering")
    return df
{% endif %}


{% if config.key_resolution_steps %}

def resolve_entity_key(df: pd.DataFrame) -> pd.DataFrame:
    from customer_retention.integrations.adapters.factory import get_delta
{% for step in config.key_resolution_steps %}
    _bridge_path = PRODUCTION_DIR / "data" / "landing" / "{{ step.bridge_dataset }}"
    _bridge = get_delta(force_local=True).read(str(_bridge_path))
    _bridge = _bridge[["{{ step.bridge_key }}", "{{ step.resolve_column }}"]].drop_duplicates(subset=["{{ step.bridge_key }}"])
    df = df.merge(_bridge, left_on="{{ step.source_key }}", right_on="{{ step.bridge_key }}", how="inner")
{% if step.source_key != step.bridge_key %}
    df = df.drop(columns=["{{ step.bridge_key }}"])
{% endif %}
{% endfor %}
    return df
{% endif %}


def get_landing_output_path() -> Path:
    return PRODUCTION_DIR / "data" / "landing" / SOURCE_NAME


def run_landing_{{ name }}():
    print(f"Landing: {SOURCE_NAME}")
    df = load_raw_data()
    print(f"  Raw records: {len(df):,}")
{% if config.raw_time_column %}
    df = df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.original_target_column %}
    df = df.rename(columns={"{{ config.original_target_column }}": TARGET_COLUMN})
{% endif %}
{% if config.key_resolution_steps %}
    df = resolve_entity_key(df)
    print(f"  After key resolution: {len(df):,}")
{% endif %}
    df = derive_temporal_columns(df)
{% if config.history_window %}
    df = apply_history_window(df)
{% endif %}
    output_path = get_landing_output_path()
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from customer_retention.integrations.adapters.factory import get_delta
    _delta = get_delta(force_local=True)
    _delta.write(df, str(output_path))
    _z_cols = [c for c in [ENTITY_COLUMN, TIME_COLUMN] if c in df.columns]
    if _z_cols:
        _delta.optimize(str(output_path), _z_cols)
    else:
        _delta.optimize(str(output_path))
    print(f"  Records: {len(df):,}")
    print(f"  Output: {output_path}")
    return df


if __name__ == "__main__":
    run_landing_{{ name }}()
""",
    "bronze_event.py.j2": """import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.pre_shaping, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_timestamp, safe_to_datetime, as_tz_naive
from pandas.api.types import is_numeric_dtype
{% if config.text_features %}
from config import PRODUCTION_DIR, TARGET_COLUMN, FIT_MODE
{% else %}
from config import PRODUCTION_DIR, TARGET_COLUMN
{% endif %}

SOURCE_NAME = "{{ source }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"

{% set pre_groups = group_steps(config.pre_shaping) %}

def apply_pre_shaping(df: pd.DataFrame) -> pd.DataFrame:
{% if config.deduplicate %}
{% if config.deduplicate is not true and config.deduplicate.strategy is defined and config.deduplicate.strategy == "keep_most_complete" %}
{% if config.deduplicate.conflict_columns %}
    _subset = {{ config.deduplicate.conflict_columns }}
{% else %}
    _subset = [ENTITY_COLUMN, TIME_COLUMN]
{% endif %}
    _null_counts = df[df.columns.difference(_subset)].isnull().sum(axis=1)
    df = df.assign(_null_count=_null_counts).sort_values("_null_count").drop_duplicates(subset=_subset, keep="first").drop(columns=["_null_count"])
{% elif config.deduplicate is not true and config.deduplicate.conflict_columns is defined and config.deduplicate.conflict_columns %}
    df = df.drop_duplicates(subset={{ config.deduplicate.conflict_columns }}, keep="first")
{% else %}
    df = df.drop_duplicates(subset=[ENTITY_COLUMN, TIME_COLUMN], keep="first")
{% endif %}
{% endif %}
{%- if pre_groups %}
{%- for func_name, steps in pre_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in pre_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}

{% if config.datetime_derivation %}

def derive_datetime_features(df: pd.DataFrame) -> pd.DataFrame:
    from customer_retention.stages.profiling import derive_extra_datetime_features
    source_columns = {{ config.datetime_derivation.source_columns }}
    existing = [c for c in source_columns if c in df.columns]
    if existing:
        df, _ = derive_extra_datetime_features(df, TIME_COLUMN, existing)
    return df
{% endif %}

{% if config.aggregation %}
def _parse_window(window_str):
    if window_str == "all_time":
        return None
    if window_str.endswith("d"):
        return pd.Timedelta(days=int(window_str[:-1]))
    if window_str.endswith("h"):
        return pd.Timedelta(hours=int(window_str[:-1]))
    if window_str.endswith("w"):
        return pd.Timedelta(weeks=int(window_str[:-1]))
    return pd.Timedelta(days=int(window_str))


def _safe_mode(x):
    if len(x) == 0:
        return None
    return x.value_counts().idxmax()


AGGREGATION_WINDOWS = {{ config.aggregation.windows }}
VALUE_COLUMNS = {{ config.aggregation.value_columns }}
AGG_FUNCS = {{ config.aggregation.agg_funcs | reject("equalto", "count") | list }}
CATEGORICAL_COLUMNS = {{ config.aggregation.categorical_columns }}
CATEGORICAL_AGG_FUNCS = {{ config.aggregation.categorical_agg_funcs }}
BINARY_COLUMNS = {{ config.aggregation.binary_columns }}
{% endif %}


def apply_event_aggregation(df: pd.DataFrame) -> pd.DataFrame:
{% if config.aggregation %}
    ensure_timestamp(df, TIME_COLUMN)
    df[TIME_COLUMN] = as_tz_naive(df[TIME_COLUMN])
    reference_date = df[TIME_COLUMN].max()
    numeric_value_columns = [c for c in VALUE_COLUMNS if c in df.columns and is_numeric_dtype(df[c])]
    base = df.groupby(ENTITY_COLUMN).agg("first")[[]]
    parts = []
    if TARGET_COLUMN in df.columns:
        parts.append(df.groupby(ENTITY_COLUMN)[TARGET_COLUMN].first())
    for window in AGGREGATION_WINDOWS:
        td = _parse_window(window)
        window_df = df if td is None else df[df[TIME_COLUMN] >= (reference_date - td)]
        for col in numeric_value_columns:
            for func in AGG_FUNCS:
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].agg(func).rename(f"{col}_{func}_{window}"))
        for col in CATEGORICAL_COLUMNS:
            if col in window_df.columns:
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].nunique().rename(f"{col}_nunique_{window}"))
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].agg(_safe_mode).rename(f"{col}_mode_{window}"))
        for col in BINARY_COLUMNS:
            if col in window_df.columns:
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].mean().rename(f"{col}_rate_{window}"))
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].sum().rename(f"{col}_count_{window}"))
                parts.append(window_df.groupby(ENTITY_COLUMN)[col].max().rename(f"{col}_any_{window}"))
        parts.append(window_df.groupby(ENTITY_COLUMN).size().rename(f"event_count_{window}"))
    if "feature_timestamp" in df.columns:
        parts.append(df.groupby(ENTITY_COLUMN)["feature_timestamp"].max().rename("feature_timestamp"))
    df = pd.concat([base] + parts, axis=1).reset_index()
    _fill_cols = [c for c in df.columns if any(c.endswith(s) for s in ("_count", "_sum", "_rate")) or c.startswith("event_count_")]
    if _fill_cols:
        df[_fill_cols] = df[_fill_cols].fillna(0)
    df.attrs["aggregation_reference_date"] = str(reference_date)
{% endif %}
    return df


{% if config.temporal_features %}

def compute_temporal_features(agg_df, raw_df):
    from customer_retention.stages.profiling.temporal_feature_engineer import (
        TemporalAggregationConfig, TemporalFeatureEngineer,
    )
    from customer_retention.core.compat import ensure_timestamp
    ensure_timestamp(raw_df, TIME_COLUMN)
    value_cols = {{ config.temporal_features.lag_columns or config.aggregation.value_columns if config.aggregation else [] }}
    numeric_vals = [c for c in value_cols if c in raw_df.columns]
    eng_config = TemporalAggregationConfig(
        lag_window_days={{ config.temporal_features.lag_window_days }},
        num_lags={{ config.temporal_features.num_lags }},
        lag_aggregations={{ config.temporal_features.lag_agg_funcs }},
    )
    engineer = TemporalFeatureEngineer(eng_config)
    result = engineer.compute(raw_df, ENTITY_COLUMN, TIME_COLUMN, numeric_vals)
    temporal_df = result.features_df
    merge_cols = [c for c in temporal_df.columns if c != ENTITY_COLUMN]
    agg_df = agg_df.merge(temporal_df[[ENTITY_COLUMN] + merge_cols], on=ENTITY_COLUMN, how="left")
    return agg_df
{% endif %}

{% if config.text_features %}

def compute_text_features(df, raw_df):
    from customer_retention.stages.profiling.text_processor import TextColumnProcessor, TextProcessingConfig
    from customer_retention.transforms import ArtifactStore
    from pathlib import Path
    store = ArtifactStore(Path(PRODUCTION_DIR / "artifacts" / "text"))
{% for tf in config.text_features %}
    if "{{ tf.column }}" in raw_df.columns:
        processor = TextColumnProcessor(TextProcessingConfig(embedding_model="{{ tf.embedding_model }}"), registry=None)
        text_data = raw_df.groupby(ENTITY_COLUMN)["{{ tf.column }}"].first().reset_index()
        if FIT_MODE:
            text_data, result = processor.process_column(text_data, "{{ tf.column }}", fit=True)
        else:
            text_data, result = processor.process_column(text_data, "{{ tf.column }}", fit=False)
        component_cols = result.component_columns
        df = df.merge(text_data[[ENTITY_COLUMN] + component_cols], on=ENTITY_COLUMN, how="left")
{% endfor %}
    return df
{% endif %}

def run_bronze_event_{{ source }}():
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta(force_local=True)
    landing_path = str(PRODUCTION_DIR / "data" / "landing" / SOURCE_NAME)
    if not storage.exists(landing_path):
        raise FileNotFoundError(f"Landing output not found: {landing_path}")
    raw_df = storage.read(landing_path)
    df = apply_pre_shaping(raw_df.copy())
{% if config.datetime_derivation %}
    df = derive_datetime_features(df)
{% endif %}
    df = apply_event_aggregation(df)
{% if config.temporal_features %}
    df = compute_temporal_features(df, raw_df)
{% endif %}
{% if config.text_features %}
    df = compute_text_features(df, raw_df)
{% endif %}
    output_name = f"{SOURCE_NAME}_aggregated"
    bronze_dir = PRODUCTION_DIR / "data" / "bronze"
    bronze_dir.mkdir(parents=True, exist_ok=True)
    _output_path = str(bronze_dir / output_name)
    storage.write(df, _output_path)
    _z_cols = [c for c in [ENTITY_COLUMN, "as_of_date"] if c in df.columns]
    if _z_cols:
        storage.optimize(_output_path, _z_cols)
    else:
        storage.optimize(_output_path)
    if "aggregation_reference_date" in df.attrs:
        (bronze_dir / f"{output_name}.reference_date").write_text(df.attrs["aggregation_reference_date"])
    return df


if __name__ == "__main__":
    run_bronze_event_{{ source }}()
""",
    "bronze_entity.py.j2": """import pandas as pd
import numpy as np
from pathlib import Path
{% set ops, fitted = collect_imports(config.post_shaping, False) %}
{% if ops %}
from customer_retention.transforms import {{ ops | sort | join(', ') }}
{% endif %}
from customer_retention.core.compat import ensure_timestamp, safe_to_datetime, timedelta_to_days
from config import PRODUCTION_DIR, RAW_SOURCES, get_bronze_path

SOURCE_NAME = "{{ source }}"
ENTITY_COLUMN = "{{ config.entity_column }}"
TIME_COLUMN = "{{ config.time_column }}"

{% if config.lifecycle %}

def _load_raw_events():
    source = RAW_SOURCES["{{ raw_source }}"]
    path = Path(source["path"])
    if not path.exists():
        raise FileNotFoundError(f"Raw source not found: {path}")
    if source["format"] == "csv":
        return pd.read_csv(str(path))
    if source["format"] == "parquet":
        return pd.read_parquet(str(path))
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(path))

{% if config.lifecycle.include_recency_bucket %}

def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    reference_date = raw_df[TIME_COLUMN].max()
    _grp = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN]
    entity_stats = _grp.min().to_frame("_time_min")
    entity_stats["_time_max"] = _grp.max()
    entity_stats["days_since_last"] = timedelta_to_days(reference_date - entity_stats["_time_max"])
    entity_stats["days_since_first"] = timedelta_to_days(reference_date - entity_stats["_time_min"])
    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df


def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_last" in df.columns:
        df["recency_bucket"] = pd.cut(df["days_since_last"], bins={{ config.lifecycle.recency_bucket_edges }} + [float("inf")],
                                       labels={{ config.lifecycle.recency_bucket_labels }})
    return df

{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}

def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
    if "days_since_first" not in df.columns:
        return df
    tenure = df["days_since_first"]
    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
    if not intensity_col:
        return df
    intensity = df[intensity_col[0]]
    tenure_med = tenure.median()
    intensity_med = intensity.median()
    conditions = [
        (tenure >= tenure_med) & (intensity >= intensity_med),
        (tenure >= tenure_med) & (intensity < intensity_med),
        (tenure < tenure_med) & (intensity >= intensity_med),
        (tenure < tenure_med) & (intensity < intensity_med),
    ]
    labels = ["steady_loyal_lifecycle", "occasional_loyal_lifecycle", "intense_brief_lifecycle", "one_shot_lifecycle"]
    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
    return df

{% endif %}
{% if config.lifecycle.include_cyclical_features %}

def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
    df = df.drop(columns=["mean_dow"], errors="ignore")
    return df

{% endif %}
{% if config.lifecycle.include_month_cyclical %}

def add_month_quarter_cyclical(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    mean_month = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.month.mean())
    df = df.merge(mean_month.rename("mean_month"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["month_sin"] = np.sin(2 * np.pi * df["mean_month"] / 12)
    df["month_cos"] = np.cos(2 * np.pi * df["mean_month"] / 12)
{% if config.lifecycle.include_quarter_cyclical %}
    mean_quarter = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: ((x.dt.month - 1) // 3).mean())
    df = df.merge(mean_quarter.rename("mean_quarter"), left_on=ENTITY_COLUMN, right_index=True, how="left")
    df["quarter_sin"] = np.sin(2 * np.pi * df["mean_quarter"] / 4)
    df["quarter_cos"] = np.cos(2 * np.pi * df["mean_quarter"] / 4)
    df = df.drop(columns=["mean_month", "mean_quarter"], errors="ignore")
{% else %}
    df = df.drop(columns=["mean_month"], errors="ignore")
{% endif %}
    return df

{% endif %}
{% if config.lifecycle.include_trend_features %}

def add_trend_features(df: pd.DataFrame) -> pd.DataFrame:
    window_cols = sorted([c for c in df.columns if c.startswith("event_count_") and c != "event_count_all_time"])
    all_time_col = "event_count_all_time" if "event_count_all_time" in df.columns else None
    if window_cols and all_time_col:
        df["recent_vs_overall_ratio"] = df[window_cols[0]] / df[all_time_col].replace(0, float("nan"))
    if len(window_cols) >= 2:
        window_values = df[window_cols].values
        x = np.arange(len(window_cols), dtype=float)
        slopes = np.array([np.polyfit(x, row, 1)[0] if not np.any(np.isnan(row)) else 0.0 for row in window_values])
        df["entity_trend_slope"] = slopes
    return df

{% endif %}
{% if config.lifecycle.include_cohort_features %}

def add_cohort_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
    ensure_timestamp(raw_df, TIME_COLUMN)
    first_event = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].min()
    cohort_data = pd.DataFrame({"first_event": first_event})
    cohort_data["cohort_year"] = cohort_data["first_event"].dt.year
    cohort_data["cohort_quarter"] = ((cohort_data["first_event"].dt.month - 1) // 3 + 1)
    df = df.merge(cohort_data[["cohort_year", "cohort_quarter"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
    return df

{% endif %}
{% if config.lifecycle.momentum_pairs %}

def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
{% for pair in config.lifecycle.momentum_pairs %}
    short_col = "event_count_{{ pair.short_window }}"
    long_col = "event_count_{{ pair.long_window }}"
    if short_col in df.columns and long_col in df.columns:
        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
{% endfor %}
    return df

{% endif %}

def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
    raw_df = _load_raw_events()
{% if config.raw_time_column %}
    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
{% endif %}
{% if config.lifecycle.include_recency_bucket %}
    df = add_recency_tenure(df, raw_df)
    df = add_recency_buckets(df)
{% endif %}
{% if config.lifecycle.include_lifecycle_quadrant %}
    df = add_lifecycle_quadrant(df)
{% endif %}
{% if config.lifecycle.include_cyclical_features %}
    df = add_cyclical_features(df, raw_df)
{% endif %}
{% if config.lifecycle.include_month_cyclical %}
    df = add_month_quarter_cyclical(df, raw_df)
{% endif %}
{% if config.lifecycle.include_trend_features %}
    df = add_trend_features(df)
{% endif %}
{% if config.lifecycle.include_cohort_features %}
    df = add_cohort_features(df, raw_df)
{% endif %}
{% if config.lifecycle.momentum_pairs %}
    df = add_momentum_ratios(df)
{% endif %}
    return df
{% endif %}

{% set post_groups = group_steps(config.post_shaping) %}

def apply_quality_transforms(df: pd.DataFrame) -> pd.DataFrame:
{% if config.lifecycle %}
    df = enrich_lifecycle(df)
{% endif %}
{%- if post_groups %}
{%- for func_name, steps in post_groups %}
    df = {{ func_name }}(df)
{%- endfor %}
{%- endif %}
    return df

{% for func_name, steps in post_groups %}

def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
{%- set _prov = provenance_docstring_block(steps) %}
{%- if _prov %}
{{ _prov }}
{%- endif %}
{%- for t in steps %}
    # {{ t.rationale }}
    # {{ action_description(t) }}
    df = {{ render_step_call(t) }}
{%- endfor %}
    return df
{% endfor %}


def run_bronze_entity_{{ source }}():
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta(force_local=True)
    bronze_path = get_bronze_path("{{ bronze_input_name }}")
    if not storage.exists(str(bronze_path)):
        raise FileNotFoundError(f"Bronze input not found: {bronze_path}")
    df = storage.read(str(bronze_path))
    df = apply_quality_transforms(df)
    bronze_path.parent.mkdir(parents=True, exist_ok=True)
    storage.write(df, str(bronze_path))
    return df


if __name__ == "__main__":
    run_bronze_entity_{{ source }}()
""",
    "validate.py.j2": """import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

import pandas as pd
import numpy as np
from config import (SOURCES, EXPLORATION_ARTIFACTS, EXPERIMENTS_DIR, PRODUCTION_DIR,
                    TARGET_COLUMN, get_silver_path, get_gold_path)


def _load_artifact(path):
    from customer_retention.integrations.adapters.factory import get_delta
    return get_delta(force_local=True).read(str(path))


def _categorize_missing(columns):
    categories = {}
    for col in columns:
        if any(col.startswith(p) for p in ["pca_", "text_pca_"]) or col.endswith("_pca"):
            categories.setdefault("text_embedding_pca", set()).add(col)
        elif any(k in col for k in ["_lag", "velocity", "acceleration"]):
            categories.setdefault("lag_velocity", set()).add(col)
        elif col in ("month_sin", "month_cos", "quarter_sin", "quarter_cos"):
            categories.setdefault("cyclical_month_quarter", set()).add(col)
        elif col in ("cohort_year", "cohort_quarter"):
            categories.setdefault("cohort", set()).add(col)
        elif col in ("recent_vs_overall_ratio", "entity_trend_slope"):
            categories.setdefault("trend", set()).add(col)
        elif col in ("dow_sin", "dow_cos"):
            categories.setdefault("cyclical_dow", set()).add(col)
        else:
            categories.setdefault("other", set()).add(col)
    return categories


def _compare_dataframes(stage, production_path, exploration_path, entity_key=None, tolerance=1e-5):
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta(force_local=True)
    if not storage.exists(str(production_path)):
        raise FileNotFoundError(f"[{stage}] Production output not found: {production_path}")
    if not storage.exists(str(exploration_path)):
        print(f"[{stage}] SKIP - exploration artifact not found: {exploration_path}")
        return True

    prod = _load_artifact(production_path)
    expl = _load_artifact(exploration_path)

    if entity_key and entity_key in prod.columns and entity_key in expl.columns:
        prod = prod.sort_values(entity_key).reset_index(drop=True)
        expl = expl.sort_values(entity_key).reset_index(drop=True)

    if prod.shape[0] != expl.shape[0]:
        raise AssertionError(f"[{stage}] Row count: production={prod.shape[0]} vs exploration={expl.shape[0]}")

    prod_cols = set(prod.columns)
    expl_cols = set(expl.columns)
    missing = expl_cols - prod_cols
    extra = prod_cols - expl_cols
    if missing:
        categories = _categorize_missing(missing)
        for category, cols in categories.items():
            print(f"[{stage}] MISSING ({category}): {sorted(cols)}")
    if extra:
        print(f"[{stage}] INFO: extra columns: {extra}")

    common = sorted(prod_cols & expl_cols)
    for col in common:
        if pd.api.types.is_numeric_dtype(prod[col]) and pd.api.types.is_numeric_dtype(expl[col]):
            try:
                pd.testing.assert_series_equal(prod[col], expl[col], check_exact=False, rtol=tolerance, check_names=False)
            except AssertionError as e:
                delta = (prod[col].astype(float) - expl[col].astype(float)).abs()
                max_idx = delta.idxmax()
                raise AssertionError(
                    f"[{stage}] Column '{col}' diverges at row {max_idx}: "
                    f"production={prod[col].iloc[max_idx]} vs exploration={expl[col].iloc[max_idx]} "
                    f"(max delta={delta.max():.2e})"
                ) from None

    print(f"[{stage}] PASS - {prod.shape[0]} rows, {len(common)} common cols, tolerance={tolerance}")
    return True


def validate_landing(tolerance=1e-5):
    landing_dir = PRODUCTION_DIR / "data" / "landing"
    if not landing_dir.exists():
        print("[Landing] SKIP - no landing directory")
        return True
    from customer_retention.integrations.adapters.factory import get_delta
    storage = get_delta(force_local=True)
    for path in landing_dir.iterdir():
        if storage.exists(str(path)):
            name = path.name
            expl_key = f"landing_{name}" if f"landing_{name}" in EXPLORATION_ARTIFACTS else "landing"
            if expl_key in EXPLORATION_ARTIFACTS:
                _compare_dataframes(f"Landing/{name}", str(path), EXPLORATION_ARTIFACTS[expl_key])
    return True


def validate_bronze(tolerance=1e-5):
    bronze_artifacts = EXPLORATION_ARTIFACTS.get("bronze", {})
    for name, expl_path in bronze_artifacts.items():
        prod_path = PRODUCTION_DIR / "data" / "bronze" / name
        _compare_dataframes(f"Bronze/{name}", str(prod_path), expl_path, tolerance=tolerance)
    return True


def validate_silver(tolerance=1e-5):
    prod_path = get_silver_path()
    expl_path = EXPLORATION_ARTIFACTS.get("silver", "")
    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
    _compare_dataframes("Silver", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
    return True


def validate_gold(tolerance=1e-5):
    prod_path = get_gold_path()
    expl_path = EXPLORATION_ARTIFACTS.get("gold", "")
    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
    _compare_dataframes("Gold", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
    return True


def validate_training():
    print("[Training] Split strategy: temporal (with DataSplitter)")
    print("[Training] PASS - training validation requires MLflow comparison (not yet implemented)")
    return True


def validate_scoring(tolerance=1e-5):
    prod_path = PRODUCTION_DIR / "data" / "scoring" / "predictions"
    expl_path = EXPLORATION_ARTIFACTS.get("scoring", "")
    _compare_dataframes("Scoring", str(prod_path), expl_path, tolerance=tolerance)
    return True


def run_all_validations(tolerance=1e-5):
    stages = [
        ("Landing", lambda: validate_landing(tolerance)),
        ("Bronze", lambda: validate_bronze(tolerance)),
        ("Silver", lambda: validate_silver(tolerance)),
        ("Gold", lambda: validate_gold(tolerance)),
        ("Training", validate_training),
        ("Scoring", lambda: validate_scoring(tolerance)),
    ]
    results = []
    for name, fn in stages:
        try:
            fn()
            results.append((name, "PASS"))
        except Exception as e:
            results.append((name, f"FAIL: {e}"))
            break

    print("\\nStage Validation Report")
    print("=" * 50)
    for name, status in results:
        print(f"[{status.split(':')[0]:4s}] {name}")
    return results
""",
    "run_validation.py.j2": '''"""{{ config.name }} - Standalone Validation Runner

Compares pipeline outputs against exploration artifacts.
Run after pipeline completes to verify correctness.
"""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

from validation.validate_pipeline import run_all_validations


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Validate pipeline outputs")
    parser.add_argument("--tolerance", type=float, default=1e-5)
    args = parser.parse_args()

    results = run_all_validations(tolerance=args.tolerance)
    failures = [r for r in results if not r[1].startswith("PASS")]
    sys.exit(1 if failures else 0)
''',
    "exploration_report.py.j2": '''"""Exploration Report Viewer

Opens HTML documentation for the exploration notebooks that informed
the pipeline transformations. Works both locally (file:// URI) and
on Databricks (displayHTML with scroll-to-anchor injection).
"""
import os
import webbrowser
from pathlib import Path

# Known notebooks referenced by pipeline provenance comments
KNOWN_NOTEBOOKS = [
{% for nb in notebooks %}
    "{{ nb }}",
{% endfor %}
]

DOCS_DIR = Path(os.environ.get("CR_DOCS_BASE_URL", str(Path(__file__).parent)))


def _is_databricks():
    return "DATABRICKS_RUNTIME_VERSION" in os.environ


def list_reports():
    for nb in KNOWN_NOTEBOOKS:
        html_path = DOCS_DIR / f"{nb}.html"
        status = "available" if html_path.exists() else "missing"
        print(f"  {nb}: {status}")


if __name__ == "__main__":
    print("Available exploration reports:")
    list_reports()
''',
}


def _sorted_landing_names(landing_dict):
    bridge_deps = {}
    for name, cfg in landing_dict.items():
        deps = set()
        for step in getattr(cfg, "key_resolution_steps", []) or []:
            if step.bridge_dataset in landing_dict:
                deps.add(step.bridge_dataset)
        bridge_deps[name] = deps
    ordered = []
    remaining = set(landing_dict)
    while remaining:
        ready = [n for n in remaining if not bridge_deps[n] - set(ordered)]
        if not ready:
            ordered.extend(sorted(remaining))
            break
        for n in sorted(ready):
            ordered.append(n)
            remaining.discard(n)
    return ordered


class CodeRenderer:
    _TEMPLATE_MAP = {
        "config": "config.py.j2",
        "silver": "silver.py.j2",
        "gold": "gold.py.j2",
        "training": "training.py.j2",
        "runner": "runner.py.j2",
        "workflow": "workflow.json.j2",
        "run_all": "run_all.py.j2",
        "feast_config": "feature_store.yaml.j2",
        "feast_features": "features.py.j2",
        "landing": "landing.py.j2",
        "bronze_event": "bronze_event.py.j2",
        "validation": "validate.py.j2",
        "run_validation": "run_validation.py.j2",
        "exploration_report": "exploration_report.py.j2",
    }

    def __init__(self):
        self._env = Environment(loader=InlineLoader(TEMPLATES))
        self._env.globals["action_description"] = action_description
        self._env.globals["render_step_call"] = render_step_call
        self._env.globals["collect_imports"] = collect_imports
        self._env.globals["group_steps"] = group_steps
        self._env.globals["provenance_docstring_block"] = provenance_docstring_block
        self._env.globals["provenance_key"] = provenance_key
        self._env.globals["partition_gold_steps"] = partition_gold_steps
        self._env.globals["sorted_landing_names"] = _sorted_landing_names

    def set_docs_base(self, experiments_dir: str | None) -> None:
        global _docs_base
        if experiments_dir:
            _docs_base = f"file://{Path(experiments_dir).resolve() / 'docs'}"
        else:
            _docs_base = "docs"

    def _render(self, template_key: str, **context) -> str:
        return self._env.get_template(self._TEMPLATE_MAP[template_key]).render(**context)

    def render_config(self, config: PipelineConfig) -> str:
        return self._render("config", config=config)

    def render_bronze(self, source_name: str, bronze_config: BronzeLayerConfig) -> str:
        return self._env.get_template("bronze.py.j2").render(source=source_name, config=bronze_config)

    def render_silver(self, config: PipelineConfig) -> str:
        return self._render("silver", config=config)

    def render_gold(self, config: PipelineConfig) -> str:
        return self._render("gold", config=config)

    def render_training(self, config: PipelineConfig) -> str:
        return self._render("training", config=config)

    def render_runner(self, config: PipelineConfig) -> str:
        return self._render("runner", config=config)

    def render_workflow(self, config: PipelineConfig) -> str:
        return self._render("workflow", config=config)

    def render_run_all(self, config: PipelineConfig) -> str:
        return self._render("run_all", config=config)

    def render_feast_config(self, config: PipelineConfig) -> str:
        return self._render("feast_config", config=config)

    def render_feast_features(self, config: PipelineConfig) -> str:
        return self._render("feast_features", config=config)

    def render_landing(self, name: str, config: LandingLayerConfig) -> str:
        return self._env.get_template("landing.py.j2").render(name=name, config=config)

    def render_bronze_event(self, source_name: str, config: BronzeEventConfig) -> str:
        return self._env.get_template("bronze_event.py.j2").render(source=source_name, config=config)

    def render_bronze_entity(
        self, source_name: str, config: BronzeEventConfig, bronze_input_name: str, raw_source_name: str = ""
    ) -> str:
        return self._env.get_template("bronze_entity.py.j2").render(
            source=source_name,
            config=config,
            bronze_input_name=bronze_input_name,
            raw_source=raw_source_name or source_name,
        )

    def render_validation(self, config: PipelineConfig) -> str:
        return self._render("validation", config=config)

    def render_run_validation(self, config: PipelineConfig) -> str:
        return self._render("run_validation", config=config)

    def render_exploration_report(self, config: PipelineConfig) -> str:
        notebooks = set()
        for bronze in config.bronze.values():
            for step in bronze.transformations:
                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
                if nb:
                    notebooks.add(nb)
        for step in config.gold.transformations + config.gold.encodings + config.gold.scalings:
            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
            if nb:
                notebooks.add(nb)
        for step in config.silver.derived_columns:
            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
            if nb:
                notebooks.add(nb)
        for be in config.bronze_event.values():
            for step in be.pre_shaping + be.post_shaping:
                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
                if nb:
                    notebooks.add(nb)
        return self._render("exploration_report", notebooks=sorted(notebooks))


_StepMeta = namedtuple("_StepMeta", ["desc_tpl", "call_tpl", "import_name", "param_defaults"])

_STATELESS_REGISTRY = {
    PipelineTransformationType.IMPUTE_NULL: _StepMeta(
        "impute nulls in {col} with {value}",
        "apply_impute_null(df, '{col}', value='{value}')",
        "apply_impute_null",
        {"value": 0},
    ),
    PipelineTransformationType.CAP_OUTLIER: _StepMeta(
        "cap outliers in {col} to [{lower}, {upper}]",
        "apply_cap_outlier(df, '{col}', lower={lower}, upper={upper})",
        "apply_cap_outlier",
        {"lower": 0, "upper": 1000000},
    ),
    PipelineTransformationType.TYPE_CAST: _StepMeta(
        "cast {col} to {dtype}", "apply_type_cast(df, '{col}', dtype='{dtype}')", "apply_type_cast", {"dtype": "float"}
    ),
    PipelineTransformationType.DROP_COLUMN: _StepMeta(
        "drop column {col}", "apply_drop_column(df, '{col}')", "apply_drop_column", {}
    ),
    PipelineTransformationType.WINSORIZE: _StepMeta(
        "winsorize {col} to [{lower_bound}, {upper_bound}]",
        "apply_winsorize(df, '{col}', lower_bound={lower_bound}, upper_bound={upper_bound})",
        "apply_winsorize",
        {"lower_bound": 0, "upper_bound": 1000000},
    ),
    PipelineTransformationType.SEGMENT_AWARE_CAP: _StepMeta(
        "segment-aware outlier cap on {col} ({n_segments} segments)",
        "apply_segment_aware_cap(df, '{col}', n_segments={n_segments})",
        "apply_segment_aware_cap",
        {"n_segments": 2},
    ),
    PipelineTransformationType.LOG_TRANSFORM: _StepMeta(
        "log-transform {col}", "apply_log_transform(df, '{col}')", "apply_log_transform", {}
    ),
    PipelineTransformationType.SQRT_TRANSFORM: _StepMeta(
        "sqrt-transform {col}", "apply_sqrt_transform(df, '{col}')", "apply_sqrt_transform", {}
    ),
    PipelineTransformationType.ZERO_INFLATION_HANDLING: _StepMeta(
        "handle zero-inflation in {col}",
        "apply_zero_inflation_handling(df, '{col}')",
        "apply_zero_inflation_handling",
        {},
    ),
    PipelineTransformationType.CAP_THEN_LOG: _StepMeta(
        "cap at p99 then log-transform {col}", "apply_cap_then_log(df, '{col}')", "apply_cap_then_log", {}
    ),
    PipelineTransformationType.FEATURE_SELECT: _StepMeta(
        "drop {col} (feature selection)", "apply_feature_select(df, '{col}')", "apply_feature_select", {}
    ),
}


def _extract_params(step, meta):
    return {k: step.parameters.get(k, v) for k, v in meta.param_defaults.items()}


def action_description(step: TransformationStep) -> str:
    t, col, p = step.type, step.column, step.parameters
    meta = _STATELESS_REGISTRY.get(t)
    if meta is not None:
        return meta.desc_tpl.format(col=col, **_extract_params(step, meta))
    if t == PipelineTransformationType.YEO_JOHNSON:
        return f"yeo-johnson transform {col}"
    if t == PipelineTransformationType.ENCODE:
        method = p.get("method", "one_hot")
        if method in ("one_hot", "onehot"):
            return f"one-hot encode {col}"
        return f"label-encode {col}"
    if t == PipelineTransformationType.SCALE:
        method = p.get("method", "standard")
        if method == "minmax":
            return f"min-max scale {col}"
        return f"standard-scale {col}"
    if t == PipelineTransformationType.DERIVED_COLUMN:
        action = p.get("action", "ratio")
        if action == "ratio":
            return f"create {col} = {p.get('numerator', '?')} / {p.get('denominator', '?')}"
        if action == "interaction":
            features = p.get("features", [])
            col_a = features[0] if len(features) > 0 else p.get("col_a", "?")
            col_b = features[1] if len(features) > 1 else p.get("col_b", "?")
            return f"create {col} = {col_a} * {col_b}"
        if action == "composite":
            return f"create {col} = mean({', '.join(p.get('columns', []))})"
    return f"transform {col}"


def render_step_call(step: TransformationStep, fit_mode: bool = True) -> str:
    t, col, p = step.type, step.column, step.parameters
    meta = _STATELESS_REGISTRY.get(t)
    if meta is not None:
        return meta.call_tpl.format(col=col, **_extract_params(step, meta))
    if t == PipelineTransformationType.YEO_JOHNSON:
        method = "fit_transform" if fit_mode else "transform"
        return f"FittedPowerTransform().{method}(df, '{col}', _store)"
    if t == PipelineTransformationType.ENCODE:
        method = p.get("method", "one_hot")
        if method in ("one_hot", "onehot"):
            return f"apply_one_hot_encode(df, '{col}')"
        fit_method = "fit_transform" if fit_mode else "transform"
        return f"FittedEncoder().{fit_method}(df, '{col}', _store)"
    if t == PipelineTransformationType.SCALE:
        method = p.get("method", "standard")
        fit_method = "fit_transform" if fit_mode else "transform"
        return f"FittedScaler('{method}').{fit_method}(df, '{col}', _store)"
    if t == PipelineTransformationType.DERIVED_COLUMN:
        action = p.get("action", "ratio")
        if action == "ratio":
            return f"apply_derived_ratio(df, '{col}', numerator='{p.get('numerator', '')}', denominator='{p.get('denominator', '')}')"
        if action == "interaction":
            features = p.get("features", [])
            col_a = features[0] if len(features) > 0 else p.get("col_a", "")
            col_b = features[1] if len(features) > 1 else p.get("col_b", "")
            return f"apply_derived_interaction(df, '{col}', col_a='{col_a}', col_b='{col_b}')"
        if action == "composite":
            columns = p.get("columns", [])
            if not columns:
                raise ValueError(f"Composite derived column '{col}' requires non-empty 'columns' parameter")
            return f"apply_derived_composite(df, '{col}', columns={columns})"
    if t == PipelineTransformationType.FILTER:
        condition = p.get("condition", "non_negative")
        if condition == "non_negative":
            return f"df[df['{col}'] >= 0]"
        if condition == "range":
            min_val = p.get("min_value", 0)
            max_val = p.get("max_value", 1000000)
            return f"df[(df['{col}'] >= {min_val}) & (df['{col}'] <= {max_val})]"
        if condition == "valid_values":
            valid = p.get("valid_values", [])
            return f"df[df['{col}'].isin({valid})]"
        return f"df[df['{col}'].notna()]"
    raise ValueError(f"Unknown transformation type: {step.type}")


def collect_imports(steps, include_fitted):
    ops = set()
    fitted = set()
    _OPS_MAP = {k: v.import_name for k, v in _STATELESS_REGISTRY.items()}
    for step in steps:
        t, p = step.type, step.parameters
        if t in _OPS_MAP:
            ops.add(_OPS_MAP[t])
        elif t == PipelineTransformationType.ENCODE:
            method = p.get("method", "one_hot")
            if method in ("one_hot", "onehot"):
                ops.add("apply_one_hot_encode")
            elif include_fitted:
                fitted.add("FittedEncoder")
        elif t == PipelineTransformationType.SCALE:
            if include_fitted:
                fitted.add("FittedScaler")
        elif t == PipelineTransformationType.YEO_JOHNSON:
            if include_fitted:
                fitted.add("FittedPowerTransform")
        elif t == PipelineTransformationType.DERIVED_COLUMN:
            action = p.get("action", "ratio")
            if action == "ratio":
                ops.add("apply_derived_ratio")
            elif action == "interaction":
                ops.add("apply_derived_interaction")
            elif action == "composite":
                ops.add("apply_derived_composite")
    return ops, fitted


def partition_gold_steps(gold):
    FITTED_TYPES = {PipelineTransformationType.SCALE, PipelineTransformationType.YEO_JOHNSON}

    stateless_transforms, fitted_transforms = [], []
    for t in gold.transformations:
        (fitted_transforms if t.type in FITTED_TYPES else stateless_transforms).append(t)

    stateless_encodings, fitted_encodings = [], []
    for e in gold.encodings:
        method = e.parameters.get("method", "one_hot")
        if method in ("one_hot", "onehot"):
            stateless_encodings.append(e)
        else:
            fitted_encodings.append(e)

    fitted_scalings = list(gold.scalings)

    return {
        "stateless_transforms": stateless_transforms,
        "stateless_encodings": stateless_encodings,
        "fitted_transforms": fitted_transforms,
        "fitted_encodings": fitted_encodings,
        "fitted_scalings": fitted_scalings,
    }
