from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable, Iterable, Sequence, TYPE_CHECKING, cast

from relationalai.semantics.backends.sql.sql_emitter import SQLEmitter, SQLDialect, DuckDBDialect
from relationalai.semantics.metamodel.rewriter import NO_WALK, Walker
from relationalai.semantics.metamodel.typer import Typer
from relationalai.util.structures import OrderedSet

from ...frontend.base import Model, Fragment
from ...metamodel.builtins import is_entity_type, is_value_type, get_primitive_supertype
from ...metamodel.metamodel import Aggregate, Construct, Data, Field, Logical, Lookup, Model as mmModel, Relation, ScalarType, Task, Type, Update, Var, Table, Match, Union, Not, Literal, Value, NumberType
from ...metamodel.metamodel_analyzer import Analyzer as MMAnalyzer, Normalize
from . import sql_ir as sql

if TYPE_CHECKING:
    from relationalai.config.config import Config

#------------------------------------------------------
# SQL Analaysis and Compiler
#------------------------------------------------------
"""

We need to take the DSL expression tree and compile it to SQL.
The metamodel represents the semantic model similar to datalog in that we have lookup operations on 6nf relations that
represent concepts, relationships, and properties and are unified by typed variables. Rules are represented by Logical
tasks in the metamodel and can derive multiple relations at once (multi-headed). Logicals nest and unify downward lexically.

The SQL compiler has to first look at all the relations and create a mapping from 6nf to 3nf tables. This involves grouping properties
into a table per Concept (represented in the metamodel as a Type that we check via is_entity_type). Relationships need to be mapped
to their own tables since they can be many to many, this translation requires going from fields of the relation to columns in the table,
whereas properties are relation to column. We can determine relationship vs property by using the analysis attached to the metamodel model.

Once we have a final table layout for the relations we need to understand the dependencies between the tables/columns. We need to determine
what the base shapes of each type are (this is determined by looking at what properties are present when deriving into the Type itself), which
properties are derived and from what, as well as what properties/relationships are recursive. From this dependency information we'll need to
create a set of intermediate tables that we can use to stratify the computation graph as needed.

"""

#------------------------------------------------------
# Helpers
#------------------------------------------------------

def is_function(r: Relation) -> bool:
    return any(f.input for f in r.fields)

def is_filter(r: Relation) -> bool:
    return all(f.input for f in r.fields)

def merge_contexts(contexts: Sequence[RuleContext]) -> sql.Fragment:
    """Merge multiple RuleContexts into a single Fragment."""
    seen = set()
    working = []
    for ctx in contexts:
        if id(ctx) in seen:
            continue
        for c in ctx.stack:
            if id(c) not in seen:
                working.append(c.fragment)
                seen.add(id(c))
        working.append(ctx.fragment)
        seen.add(id(ctx))

    return merge_fragments(working)

def merge_fragments(fragments: Sequence[sql.Fragment]) -> sql.Fragment:
    """Merge multiple Fragments into a single Fragment."""
    selects = OrderedSet()
    filters = OrderedSet()
    scans = OrderedSet()
    joins = OrderedSet()
    aggregates = OrderedSet()

    for frag in fragments:
        selects.update(frag.selects)
        filters.update(frag.filters)
        scans.update(frag.scans)
        joins.update(frag.joins)
        aggregates.update(frag.aggregates)

    return sql.Fragment(
        scans=list(scans),
        joins=list(joins),
        filters=list(filters),
        aggregates=list(aggregates),
    )

def first(seq: Iterable, default):
    for item in seq:
        return item
    return default


def _incoming_table_refs(
    nodes: Iterable[sql.IRNode],
    alias_of: Callable[[sql.IRNode], str | None],
) -> set[sql.TableRef]:
    incoming: set[sql.TableRef] = set()
    for node in nodes:
        if isinstance(node, sql.TableRef):
            incoming.add(node)
            continue
        alias = alias_of(node)
        if alias:
            incoming.add(sql.TableRef(name=alias, alias=alias))
    return incoming

#------------------------------------------------------
# Fragment Normalizer
#------------------------------------------------------

class FragmentNormalizer(sql.IRRewriter):

    def __init__(self, root_keys: set[sql.ColumnRef|sql.RowId]):
        self._root_keys = root_keys
        self._at_root = True
        self._agg_cache: dict[int, sql.AggregatePlaceholder] = {}

    #------------------------------------------------------
    # Helpers
    #------------------------------------------------------

    def _collect_table_refs(self, node: sql.IRNode) -> set[sql.TableRef]:
        """Recursively collect all TableRefs referenced in an IR node."""
        refs: set[sql.TableRef] = set()
        if isinstance(node, sql.ColumnRef):
            refs.add(node.table)
        elif isinstance(node, sql.RowId):
            refs.add(node.table)
        elif isinstance(node, sql.Expression):
            for arg in node.args:
                refs.update(self._collect_table_refs(arg))
        elif isinstance(node, sql.WindowExpression):
            for arg in node.args:
                refs.update(self._collect_table_refs(arg))
            for item in node.partition_by:
                refs.update(self._collect_table_refs(item))
            for item in node.order_by:
                refs.update(self._collect_table_refs(item))
        elif isinstance(node, sql.Alias):
            refs.update(self._collect_table_refs(node.node))
        elif isinstance(node, sql.Filter):
            refs.update(self._collect_table_refs(node.left))
            refs.update(self._collect_table_refs(node.right))
        elif isinstance(node, sql.CaseWhen):
            for cond, result in node.branches:
                refs.update(self._collect_table_refs(cond))
                refs.update(self._collect_table_refs(result))
            if node.else_result:
                refs.update(self._collect_table_refs(node.else_result))
        elif isinstance(node, sql.And):
            for cond in node.conditions:
                refs.update(self._collect_table_refs(cond))
        elif isinstance(node, sql.Or):
            for cond in node.conditions:
                refs.update(self._collect_table_refs(cond))
        elif isinstance(node, sql.AggregatePlaceholder):
            refs.update(self._collect_table_refs(node.ref))
        return refs

    def _ref_name(self, ref: sql.TableRef) -> str:
        return ref.alias or ref.name

    def _alias_name(self, node: sql.IRNode) -> str | None:
        if isinstance(node, sql.Alias):
            return node.alias
        if isinstance(node, sql.Lateral) and isinstance(node.node, sql.Alias):
            return node.node.alias
        if isinstance(node, sql.TableRef):
            return self._ref_name(node)
        return None

    def _table_ref(self, node: sql.IRNode) -> sql.TableRef | None:
        if isinstance(node, sql.TableRef):
            return node
        if isinstance(node, sql.Alias) and isinstance(node.node, sql.TableRef):
            return node.node
        return None

    #------------------------------------------------------
    # Existing wrapping graph walk
    #------------------------------------------------------

    def _compute_live_aliases(self, frag: sql.Fragment, keys: set[sql.ColumnRef|sql.RowId]) -> set[str]:
        live_aliases: set[str] = set()

        def add_refs(refs: Iterable[sql.IRNode]) -> bool:
            changed = False
            for ref in refs:
                if isinstance(ref, sql.TableRef):
                    name = self._ref_name(ref)
                else:
                    name = self._alias_name(ref)
                    if name is None:
                        continue
                if name not in live_aliases:
                    live_aliases.add(name)
                    changed = True
            return changed

        for select in frag.selects:
            add_refs(self._collect_table_refs(select))
        for group in frag.group_by:
            add_refs(self._collect_table_refs(group))
        add_refs({key.table for key in keys})

        # Expand live set based on filters/joins that touch live tables
        changed = True
        while changed:
            changed = False
            for f in frag.filters:
                refs = self._collect_table_refs(f)
                if any(self._ref_name(ref) in live_aliases for ref in refs):
                    if add_refs(refs):
                        changed = True
            for join in frag.joins:
                if self._is_live_node(join.target, live_aliases):
                    if add_refs(join.incoming):
                        changed = True
                    for f in join.filters:
                        refs = self._collect_table_refs(f)
                        if add_refs(refs):
                            changed = True

        return live_aliases

    def _is_live_ref(self, ref: sql.TableRef, live_aliases: set[str]) -> bool:
        return self._ref_name(ref) in live_aliases

    def _is_live_node(self, node: sql.IRNode, live_aliases: set[str]) -> bool:
        ref = self._table_ref(node)
        if ref and self._is_live_ref(ref, live_aliases):
            return True
        alias = self._alias_name(node)
        if alias and alias in live_aliases:
            return True
        return False

    def _split_scans(self, scans: list[sql.IRNode], live_aliases: set[str]) -> tuple[list[sql.IRNode], list[sql.IRNode]]:
        kept: list[sql.IRNode] = []
        moved: list[sql.IRNode] = []
        for scan in scans:
            if self._is_live_node(scan, live_aliases):
                kept.append(scan)
            else:
                moved.append(scan)
        return kept, moved

    def _split_joins(self, joins: list[sql.Join], live_aliases: set[str]) -> tuple[list[sql.Join], list[sql.Join], list[sql.Filter]]:
        kept: list[sql.Join] = []
        moved: list[sql.Join] = []
        moved_filters: list[sql.Filter] = []
        for join in joins:
            target_live = self._is_live_node(join.target, live_aliases)
            if target_live:
                # Keep join if target is live, but remove filters that reference non-live tables
                kept_filters = []
                for f in join.filters:
                    filter_refs = self._collect_table_refs(f)
                    if all(self._is_live_ref(ref, live_aliases) for ref in filter_refs):
                        kept_filters.append(f)
                    else:
                        moved_filters.append(f)
                # Create new join with only kept filters
                new_join = sql.Join(
                    target=join.target,
                    incoming=join.incoming,
                    filters=kept_filters,
                    type=join.type,
                )
                kept.append(new_join)
            else:
                moved.append(join)
                moved_filters.extend(join.filters)
        return kept, moved, moved_filters

    def _split_filters(self, filters: list[sql.IRNode], live_aliases: set[str], moved_join_filters: list[sql.Filter]) -> tuple[list[sql.IRNode], list[sql.IRNode]]:
        filters_to_keep: list[sql.IRNode] = []
        exists_filters: list[sql.IRNode] = list(moved_join_filters)
        for f in filters:
            if isinstance(f, sql.Exists):
                filters_to_keep.append(f)
                continue
            filter_refs = self._collect_table_refs(f)
            if any(not self._is_live_ref(ref, live_aliases) for ref in filter_refs):
                exists_filters.append(f)
            else:
                filters_to_keep.append(f)
        return filters_to_keep, exists_filters

    def _collect_exists_scans(self, moved_scans: list[sql.IRNode], moved_joins: list[sql.Join], live_aliases: set[str]) -> list[sql.IRNode]:
        exists_scans = OrderedSet()
        for scan in moved_scans:
            exists_scans.add(scan)
        for join in moved_joins:
            if not self._is_live_node(join.target, live_aliases):
                exists_scans.add(join.target)
        return list(exists_scans)

    def _wrap_exists(self, frag: sql.Fragment, keys: set[sql.ColumnRef|sql.RowId]) -> sql.Fragment:
        live_aliases = self._compute_live_aliases(frag, keys)
        kept_scans, moved_scans = self._split_scans(frag.scans, live_aliases)
        kept_joins, moved_joins, moved_join_filters = self._split_joins(frag.joins, live_aliases)
        filters_to_keep, exists_filters = self._split_filters(frag.filters, live_aliases, moved_join_filters)
        exists_scans = self._collect_exists_scans(moved_scans, moved_joins, live_aliases)

        if exists_scans:
            exists_frag = sql.Fragment(
                selects=[sql.Literal(value=1, type="Number")],
                scans=exists_scans,
                filters=exists_filters,
            )
            filters_to_keep.append(sql.Exists(subquery=exists_frag))

        frag.scans = kept_scans
        frag.joins = kept_joins
        frag.filters = filters_to_keep

        # If no scans remain but there are CROSS JOINs, convert them to scans
        if not frag.scans and frag.joins:
            new_scans = []
            new_joins = []
            for join in frag.joins:
                if not join.filters:
                    new_scans.append(join.target)
                else:
                    new_joins.append(join)
            if new_scans:
                frag.scans = new_scans
                frag.joins = new_joins

        return frag

    #------------------------------------------------------
    # Aggregate rewriting
    #------------------------------------------------------

    def _needs_distinct(self, agg: sql.AggregatePlaceholder) -> bool:
        """Check if aggregate needs DISTINCT based on projection.

        The frontend signals distinct by including/excluding entity IDs in the projection.

        For bag semantics: projection includes all entity IDs involved
        For set semantics: projection excludes some entity IDs

        We detect distinct by tracking entity types:
        1. Collect entity types that have their IDs in projection (via _id column or entity-typed field)
        2. Collect entity types that are referenced (have properties used) in projection expressions
        3. If any referenced entity type's ID is not in projection, need DISTINCT
        """
        # Track entity types whose IDs are in the projection
        covered_entity_types: set[Type | Relation] = set()

        # Track entity types that contribute data to the projection
        referenced_entity_types: set[Type] = set()

        def collect_from_column(col: sql.ColumnRef):
            """Process a ColumnRef to extract entity type information."""
            if not col.column:
                return
            if col.column.is_id:
                # This is the table's identity column - get the entity type
                covered_entity_types.add(col.column.table.relation)
            elif isinstance(col.column.column, Field):
                field = col.column.column
                if field.type and is_entity_type(field.type):
                    # This field holds entity IDs (e.g., cars.cars holds Car IDs)
                    covered_entity_types.add(field.type)
                else:
                    # This is a property field - the entity that has this property is referenced
                    # The property belongs to field._relation.fields[0].type
                    if hasattr(field, '_relation') and field._relation:
                        rel = field._relation
                        if rel.fields and is_entity_type(rel.fields[0].type):
                            referenced_entity_types.add(rel.fields[0].type)
            elif isinstance(col.column.column, Relation):
                # This is a property column (stored as Relation, e.g., Person.name)
                rel = col.column.column
                if rel.fields and is_entity_type(rel.fields[0].type):
                    referenced_entity_types.add(rel.fields[0].type)

        def collect_references(node: sql.IRNode):
            """Recursively collect entity type references from an expression."""
            if isinstance(node, sql.ColumnRef):
                collect_from_column(node)
            elif isinstance(node, sql.RowId):
                # RowId represents a table's ID - we'd need the analysis to get entity type
                pass
            elif isinstance(node, sql.Expression):
                for arg in node.args:
                    collect_references(arg)

        # Process each projection item
        for col in agg.projection:
            if isinstance(col, sql.ColumnRef):
                collect_from_column(col)
            elif isinstance(col, sql.Expression):
                collect_references(col)

        # Also collect references from expressions to find which entity types contribute data
        for col in agg.projection:
            collect_references(col)

        # If any referenced entity type's ID is NOT covered, need DISTINCT
        for entity_type in referenced_entity_types:
            if entity_type not in covered_entity_types:
                return True

        return False

    def _rewrite_aggregates(self, frag: sql.Fragment) -> sql.Fragment:
        if frag.aggregates:
            # group aggs by their group vars
            agg_groups: dict[int, list[sql.AggregatePlaceholder]] = defaultdict(list)
            for agg in frag.aggregates:
                key = hash((
                    tuple(agg.projection),
                    tuple(agg.group),
                    tuple(agg.body.scans),
                    tuple(agg.body.joins),
                    agg.optional,
                ))
                agg_groups[key].append(agg)

            outer = frag.clone()
            outer.aggregates = []
            replacements: dict[int, sql.AggregatePlaceholder] = {}
            group_col_mapping: dict[sql.ColumnRef, sql.ColumnRef] = {}
            def _contains_agg(node: sql.IRNode) -> bool:
                found = False

                class Finder(sql.IRRewriter):
                    def rewrite_aggregateplaceholder(self, node: sql.AggregatePlaceholder) -> sql.IRNode:
                        nonlocal found
                        found = True
                        return node

                Finder().rewrite(node)
                return found

            outer_for_agg = outer.clone()
            outer_for_agg.filters = [f for f in outer.filters if not _contains_agg(f)]

            for group_ix, aggs in enumerate(agg_groups.values()):
                join_ref = sql.TableRef(name="agg_join", alias=f"agg_{group_ix}")
                agg_body = aggs[0].body.clone()
                if any(agg.body.filters != agg_body.filters for agg in aggs[1:]):
                    agg_body.filters = []
                sub_frag = merge_fragments([outer_for_agg, agg_body])
                if len(agg_groups) == 1 and sub_frag == outer:
                    # TODO: this is only true if the output cardinality is 1:1 with the group
                    # we can do this as a single query
                    print("COULD OPTIMIZE AGGREGATE AS SINGLE QUERY")
                    pass
                # Add GROUP BY columns from the aggregate
                if aggs[0].group:
                    sub_frag.group_by = list(aggs[0].group)
                    # Also add group columns to selects so they're available for joining
                    for i, group_col in enumerate(aggs[0].group):
                        col_name = getattr(group_col, 'name', None) or f"_group_{i}"
                        sub_frag.selects.append(sql.Alias(group_col, col_name))

                for agg_ix, agg in enumerate(aggs):
                    # Detect if DISTINCT is needed based on projection
                    needs_distinct = self._needs_distinct(agg)

                    args: list[sql.IRNode]
                    if agg.body.filters and agg.body.filters != sub_frag.filters:
                        first_arg = agg.args[0] if agg.args else sql.Literal(value=1, type="Number")
                        args = [sql.CaseWhen(
                            branches=[(sql.And(agg.body.filters), first_arg)],
                            else_result=sql.Literal(value=None, type="null")
                        )]
                    else:
                        args = list(agg.args)

                    # Wrap first arg in Distinct if needed
                    if needs_distinct:
                        if args:
                            args[0] = sql.Distinct(args[0])
                        elif agg.projection:
                            # For COUNT() with no args, use projection columns as the distinct key
                            args = [sql.Distinct(agg.projection[0])]

                    exp = sql.Expression(op=agg.op, args=args)
                    alias = sql.Alias(exp, f"{agg.op}_{agg_ix}")
                    new_ref = sql.ColumnRef(join_ref, name=alias.alias)
                    replacements[id(agg)] = sql.AggregatePlaceholder(
                        ref=new_ref,
                        op=agg.op,
                        args=agg.args,
                        body=agg.body,
                        projection=agg.projection,
                        group=agg.group,
                        optional=agg.optional,
                    )
                    sub_frag.selects.append(alias)

                agg_keys = {
                    key for key in [*aggs[0].projection, *aggs[0].group, *aggs[0].args]
                    if isinstance(key, (sql.ColumnRef, sql.RowId))
                }
                if agg_keys:
                    sub_frag = self._wrap_exists(sub_frag, agg_keys)

                # Build join filters to connect aggregate results back to outer query
                join_filters = []
                for i, group_col in enumerate(aggs[0].group):
                    col_name = getattr(group_col, 'name', None) or f"_group_{i}"
                    join_filters.append(sql.Filter(
                        op="=",
                        left=group_col,
                        right=sql.ColumnRef(join_ref, name=col_name)
                    ))
                # Build mapping of group columns to aggregate join columns
                # This is used later to rewrite selects
                group_col_mapping = {}
                for i, group_col in enumerate(aggs[0].group):
                    col_name = getattr(group_col, 'name', None) or f"_group_{i}"
                    group_col_mapping[group_col] = sql.ColumnRef(join_ref, name=col_name)

                if frag.scans:
                    join_type = "LEFT" if any(a.optional for a in aggs) else "INNER"
                    frag.joins.append(
                        sql.Join(
                            target=sql.Alias(sub_frag, join_ref.alias),
                            incoming=_incoming_table_refs(outer.scans, self._alias_name),
                            filters=join_filters,
                            type=join_type,
                        )
                    )
                else:
                    # No FROM clause, so we can just add the subquery as a scan
                    frag.scans.append(sql.Alias(sub_frag, join_ref.alias))
                    frag.filters.extend(join_filters)

            # Replace aggregates first (id-based), then group columns (object-based)
            if replacements:
                frag = self._replace_aggregates(frag, replacements)

            # Rewrite selects to use aggregate join columns instead of outer group columns
            # This allows outer scans to become non-live if not otherwise needed
            if group_col_mapping:
                frag = self._replace_group_cols(frag, group_col_mapping)
        return frag

    def _replace_aggregates(self, frag: sql.Fragment, replacements: dict[int, sql.AggregatePlaceholder]) -> sql.Fragment:
        class _AggregateReplacer(sql.IRRewriter):
            def __init__(self, mapping: dict[int, sql.AggregatePlaceholder]):
                self._mapping = mapping

            def rewrite_aggregateplaceholder(self, node: sql.AggregatePlaceholder) -> sql.IRNode:
                replacement = self._mapping.get(id(node))
                if replacement:
                    return replacement
                return node

        return _AggregateReplacer(replacements).rewrite(frag)

    def _replace_group_cols(self, frag: sql.Fragment, replacements: dict[sql.ColumnRef, sql.ColumnRef]) -> sql.Fragment:
        """Replace group column references in selects with aggregate join column references.

        This allows outer scans to become non-live when their columns are only used
        in aggregate group columns that are also available from the aggregate join.
        """
        class _GroupColReplacer(sql.IRRewriter):
            def __init__(self, mapping: dict[sql.ColumnRef, sql.ColumnRef]):
                self._mapping = mapping

            def rewrite_columnref(self, node: sql.ColumnRef) -> sql.IRNode:
                # Match by object equality (uses __eq__ and __hash__ from IRNode)
                replacement = self._mapping.get(node)
                if replacement:
                    return replacement
                return node

        # Only replace in selects, not in scans/joins/filters
        new_selects = [_GroupColReplacer(replacements).rewrite(s) for s in frag.selects]
        frag.selects = new_selects
        return frag

    #------------------------------------------------------
    # Replacers
    #------------------------------------------------------

    def rewrite_aggregateplaceholder(self, node: sql.AggregatePlaceholder) -> sql.IRNode:
        cached = self._agg_cache.get(id(node))
        if cached:
            return cached
        new = sql.AggregatePlaceholder(
            ref=self.rewrite(node.ref),
            op=node.op,
            args=[self.rewrite(a) for a in node.args],
            body=self.rewrite(node.body),
            projection=[self.rewrite(c) for c in node.projection],
            group=[self.rewrite(c) for c in node.group],
            optional=node.optional,
        )
        self._agg_cache[id(node)] = new
        return new

    def rewrite_fragment(self, node: sql.Fragment) -> sql.IRNode:
        was_root = self._at_root
        self._at_root = False
        frag = sql.Fragment(
            selects=[self.rewrite(s) for s in node.selects],
            scans=[self.rewrite(s) for s in node.scans],
            joins=[self.rewrite(j) for j in node.joins],
            filters=[self.rewrite(f) for f in node.filters],
            group_by=[self.rewrite(g) for g in node.group_by],
            aggregates=[self.rewrite(a) for a in node.aggregates],
        )
        frag = self._rewrite_aggregates(frag)
        if was_root:
            frag = self._wrap_exists(frag, self._root_keys)
        return frag

#------------------------------------------------------
# TableInfo
#------------------------------------------------------

class TableKind(Enum):
    CONCEPT = 1
    DATA = 2
    RELATIONSHIP = 3
    EXTERNAL = 4
    OUTPUT = 5

@dataclass
class TableInfo:
    relation: Relation = field()
    columns: list[ColumnInfo] = field(default_factory=list)
    kind: TableKind = field(default=TableKind.CONCEPT)

    @property
    def name(self) -> str:
        return self.relation.name

    def __hash__(self):
        return hash(self.relation)

@dataclass
class ColumnInfo:
    table: TableInfo = field()
    column: Relation|Field = field()

    @property
    def is_id(self) -> bool:
        return self.table.relation == self.column

    @property
    def name(self) -> str:
        if self.is_id:
            return "_id"
        return self.column.name

    def __hash__(self):
        return hash((self.table.relation.id, self.column.id))

#------------------------------------------------------
# Stratification
#------------------------------------------------------

class SimpleState(Enum):
    """State for memoized is_simple computation with cycle detection."""
    UNKNOWN = 0
    COMPUTING = 1  # Cycle detection
    SIMPLE = 2
    NOT_SIMPLE = 3

@dataclass
class Derivation:
    """Groups all writes to a column. Owns is_simple logic."""
    column: ColumnInfo
    writes: OrderedSet[ColumnWrite]
    _state: SimpleState = field(default=SimpleState.UNKNOWN, compare=False, repr=False)

    @property
    def reads(self) -> set[ColumnInfo]:
        """All columns read by any write to this column."""
        return {col for w in self.writes for col in w.reads}

    @property
    def is_local(self) -> bool:
        """True if all writes only use local row data."""
        return all(w.is_local for w in self.writes)

    def is_simple(self, derivations: dict[ColumnInfo, Derivation]) -> bool:
        """
        Simple if:
        1. All writes are local (no aggregations, etc.)
        2. All derived reads are from the same table
        3. All derived reads are also simple (recursive check)

        Note: External table reads (not in derivations) are always available
        and don't affect simplicity.
        """
        # Memoized result
        if self._state == SimpleState.SIMPLE:
            return True
        if self._state == SimpleState.NOT_SIMPLE:
            return False
        if self._state == SimpleState.COMPUTING:
            return False  # Cycle -> not simple

        self._state = SimpleState.COMPUTING

        # Check conditions
        if not self.is_local:
            self._state = SimpleState.NOT_SIMPLE
            return False

        target_table = self.column.table
        for col in self.reads:
            # External sources (not derived) are always available - skip
            if col not in derivations:
                continue
            # Derived from different table - need separate stratum
            if col.table != target_table:
                self._state = SimpleState.NOT_SIMPLE
                return False
            # Derived from same table - check if the dependency is simple
            if not derivations[col].is_simple(derivations):
                self._state = SimpleState.NOT_SIMPLE
                return False

        self._state = SimpleState.SIMPLE
        return True

@dataclass
class RecursiveGroup:
    """A group of derivations that form a recursive SCC and must use semi-naive evaluation."""
    derivations: list[Derivation]

    @property
    def columns(self) -> set[ColumnInfo]:
        """Columns in this recursive group."""
        return {d.column for d in self.derivations}

    @property
    def writes(self) -> set[ColumnWrite]:
        """All writes in this group."""
        return {w for d in self.derivations for w in d.writes}

    @property
    def base_writes(self) -> set[ColumnWrite]:
        """Writes that don't read from this group's columns (base case for semi-naive)."""
        my_columns = self.columns
        return {w for w in self.writes if not (w.reads & my_columns)}

    @property
    def recursive_writes(self) -> set[ColumnWrite]:
        """Writes that read from this group's columns (recursive case for semi-naive)."""
        my_columns = self.columns
        return {w for w in self.writes if w.reads & my_columns}

@dataclass
class Stratum:
    """A stratum represents a set of derivations at the same dependency level."""
    id: int
    derivations: list[Derivation]
    recursive_groups: list[RecursiveGroup] = field(default_factory=list)

    @property
    def is_recursive(self) -> bool:
        """True if this stratum contains any recursive groups."""
        return len(self.recursive_groups) > 0

    @property
    def writes(self) -> set[ColumnWrite]:
        """All writes in this stratum."""
        return {w for d in self.derivations for w in d.writes}

    @property
    def columns(self) -> set[ColumnInfo]:
        """All columns produced by this stratum."""
        return {d.column for d in self.derivations}

#------------------------------------------------------
# RuleContext
#------------------------------------------------------

@dataclass
class RuleContext:
    stack: list[RuleContext] = field(default_factory=list)
    optional:bool = field(default=False)
    isolated:bool = field(default=False)  # True for subquery contexts (NOT, UNION, MATCH)
    fragment: sql.Fragment = field(default_factory=sql.Fragment)
    var_map: dict[Value, sql.IRNode] = field(default_factory=dict)
    table_ref_map: dict[tuple[Var, str], sql.TableRef] = field(default_factory=dict)
    used_columns: set[ColumnInfo] = field(default_factory=set)
    construct_keys: dict[Var, list[sql.IRNode]] = field(default_factory=lambda: defaultdict(list))
    id:int = field(default_factory=lambda: RuleContext._next_id())

    _table_counts: dict[str, int] = field(default_factory=dict)  # shared across stack
    _id_counter: int = 0  # class-level

    @staticmethod
    def _next_id():
        RuleContext._id_counter += 1
        return RuleContext._id_counter

    @property
    def is_local(self) -> bool:
        """True if this context only uses local row data (no aggregations, etc.)."""
        if self.fragment.aggregates:
            return False
        # Check parent contexts too
        for parent in self.stack:
            if not parent.is_local:
                return False
        return True

    def is_table_in_scope(self, table: sql.TableRef) -> bool:
        """Check if a table is in scope. Walks up the stack but stops at isolated contexts."""
        def _matches(node: sql.IRNode, ref: sql.TableRef) -> bool:
            if node == ref:
                return True
            if isinstance(node, sql.TableRef):
                return node == ref
            alias = None
            if isinstance(node, sql.Alias):
                alias = node.alias
            elif isinstance(node, sql.Lateral) and isinstance(node.node, sql.Alias):
                alias = node.node.alias
            return alias is not None and alias == (ref.alias or ref.name)

        if any(_matches(s, table) for s in self.fragment.scans):
            return True
        if any(_matches(j.target, table) for j in self.fragment.joins):
            return True
        if self.isolated:
            return False
        # Walk up the stack, stopping at any isolated context
        for parent in reversed(self.stack):
            if any(_matches(s, table) for s in parent.fragment.scans):
                return True
            if any(_matches(j.target, table) for j in parent.fragment.joins):
                return True
            if parent.isolated:
                break
        return False

    #------------------------------------------------------
    # Stack methods
    #------------------------------------------------------

    def push(self, isolated: bool = False) -> RuleContext:
        return RuleContext(stack=self.stack + [self], isolated=isolated, _table_counts=self._table_counts, construct_keys=self.construct_keys)

    def pop(self) -> RuleContext:
        return self.stack[-1] if self.stack else self

    #------------------------------------------------------
    # Vars
    #------------------------------------------------------

    def bind_var(self, var: Value, sql_node: sql.IRNode):
        self.var_map[var] = sql_node

    def lookup_var(self, var: Value) -> sql.IRNode | None:
        parent = self
        while parent:
            v = parent.var_map.get(var, None)
            if v is not None:
                if isinstance(v, sql.ColumnRef):
                    if v.column is not None:
                        self.used_columns.add(v.column)
                return v
            parent = parent.stack[-1] if parent.stack else None

    #------------------------------------------------------
    # Table refs
    #------------------------------------------------------

    def lookup_table_ref(self, var: Var, table: TableInfo|sql.TableRef) -> sql.TableRef:
        parent = self
        while parent:
            t = parent.table_ref_map.get((var, table.name), None)
            if t is not None:
                return t
            parent = parent.stack[-1] if parent.stack else None
        # Not found, create and store it
        if isinstance(table, sql.TableRef):
            table_ref = table
        else:
            table_ref = self.TableRef(table)
        self.table_ref_map[(var, table.name)] = table_ref
        return table_ref

    #------------------------------------------------------
    # Items
    #------------------------------------------------------

    def TableRef(self, table: TableInfo) -> sql.TableRef:
        name = table.name
        base = name.rsplit('.', 1)[-1]
        count = self._table_counts.get(name, 0)
        self._table_counts[name] = count + 1
        alias_base = f"t_{base}"
        alias = alias_base if count == 0 else f"{alias_base}_{count}"
        table_ref = sql.TableRef(name=name, alias=alias)
        self.fragment.scans.append(table_ref)
        return table_ref

    def ColumnRef(self, table_ref: sql.TableRef, column: ColumnInfo) -> sql.ColumnRef:
        self.used_columns.add(column)
        return sql.ColumnRef(table=table_ref, name=column.name, column=column)

    def Filter(self, op: str, left: sql.IRNode, right: sql.IRNode) -> sql.Filter:
        f = sql.Filter(op, left, right)
        self.fragment.filters.append(f)
        return f

    def Join(self, left: sql.TableRef, right: sql.TableRef, on_: sql.Filter, type: str="INNER") -> sql.Join:
        # Collapse joins to same target table, accumulating incoming dependencies
        existing = next((j for j in self.fragment.joins if j.target == right), None)
        if existing:
            existing.filters.append(on_)
            existing.incoming.add(left)
            # Merge to the stricter join type when constraints are added later.
            if existing.type == "LEFT" and type == "INNER":
                existing.type = "INNER"
            return existing
        j = sql.Join(target=right, incoming={left}, filters=[on_], type=type)
        self.fragment.joins.append(j)
        return j

    def Aggregate(self, ref:sql.IRNode, op: str, args: list[sql.IRNode], body: sql.Fragment, projection: list[sql.ColumnRef], group: list[sql.ColumnRef]) -> sql.AggregatePlaceholder:
        agg = sql.AggregatePlaceholder(
            ref=ref,
            op=op,
            args=args,
            body=body,
            projection=projection,
            group=group,
            optional=self.optional,
        )
        self.fragment.aggregates.append(agg)
        return agg

#------------------------------------------------------
# RuleAnalyzer
#------------------------------------------------------

class RuleAnalyzer(Walker):
    def analyze(self, rule: Task, analysis: ModelAnalysis):
        self.ctx = RuleContext()
        self.analysis = analysis
        self(rule)
        pass

    def _node_alias(self, node: sql.IRNode) -> str | None:
        if isinstance(node, sql.TableRef):
            return node.alias or node.name
        if isinstance(node, sql.Alias):
            return node.alias
        if isinstance(node, sql.Lateral) and isinstance(node.node, sql.Alias):
            return node.node.alias
        return None

    def _table_nullability(self) -> dict[str, bool]:
        nullable: dict[str, bool] = {}
        for scan in self.ctx.fragment.scans:
            alias = self._node_alias(scan)
            if alias:
                nullable.setdefault(alias, False)
        for join in self.ctx.fragment.joins:
            incoming_nullable = any(
                nullable.get(alias, False)
                for alias in (self._node_alias(node) for node in join.incoming)
                if alias is not None
            )
            target_alias = self._node_alias(join.target)
            if target_alias is None:
                continue
            nullable[target_alias] = (join.type == "LEFT") or incoming_nullable
        return nullable

    def _optional_join_type(self, incoming: Iterable[sql.IRNode]) -> str:
        if not self.ctx.optional:
            return "INNER"
        # Optional logicals are null-extending at the boundary, not internally.
        # We therefore use LEFT only when attaching from a nullable side (or at the
        # first optional boundary attachment), and keep subsequent constraints INNER.
        if not self.ctx.fragment.joins:
            return "LEFT"
        nullable = self._table_nullability()
        if any(nullable.get(alias, False) for alias in (self._node_alias(node) for node in incoming) if alias is not None):
            return "LEFT"
        return "INNER"

    #------------------------------------------------------
    # Value mapping
    #------------------------------------------------------

    def try_resolve(self, node) -> sql.IRNode | None:
        if isinstance(node, Var):
            return self.ctx.lookup_var(node)
        elif isinstance(node, Literal):
            literal_type = node.type.name
            if isinstance(node.type, Type) and is_value_type(node.type):
                prim = get_primitive_supertype(node.type)
                if prim is not None:
                    literal_type = prim.name
            return sql.Literal(value=node.value, type=literal_type)
        elif isinstance(node, tuple):
            args: list[sql.IRNode] = []
            for item in node:
                resolved = self.try_resolve(item)
                if resolved is None:
                    return None
                args.append(resolved)
            return sql.Expression(op="TUPLE", args=args)
        else:
            raise NotImplementedError(f"Cannot convert node of type {type(node)} to SQL value.")

    def resolve(self, node) -> sql.IRNode:
        sql_v = self.try_resolve(node)
        if sql_v is None:
            raise ValueError(f"Cannot resolve node {node} to SQL value.")
        return sql_v

    #------------------------------------------------------
    # Require
    #------------------------------------------------------

    def enter_require(self, node: Task):
        return NO_WALK

    #------------------------------------------------------
    # Logical
    #------------------------------------------------------

    def enter_logical(self, node: Logical):
        self.ctx = self.ctx.push()
        self.ctx.optional = node.optional

    def logical(self, node: Logical):
        self.ctx = self.ctx.pop()

    def capture_child_context(self, node: Task, with_selects: Sequence[Value] = (), isolated: bool = True) -> RuleContext:
        new = self.ctx = self.ctx.push(isolated=isolated)
        if isinstance(node, Logical):
            for child in node.body:
                self(child)
        else:
            self(node)
        for s in with_selects:
            new.fragment.selects.append(self.resolve(s))
        self.ctx = self.ctx.pop()
        return new

    #------------------------------------------------------
    # Lookup
    #------------------------------------------------------

    def lookup(self, node: Lookup):
        r = node.relation
        if is_filter(r):
            self._lookup_filter(node)
        elif is_function(r):
            self._lookup_function(node)
        elif col := self.analysis.get_column_info(r):
            self._lookup_table_column(node, col)
        elif table := self.analysis.get_table_info(r):
            self._lookup_relationship(node, table)

    def _maybe_bind(self, arg: Value, value: sql.IRNode):
        existing = self.try_resolve(arg)
        if (isinstance(existing, (sql.ColumnRef, sql.RowId))
            and isinstance(value, (sql.ColumnRef, sql.RowId))
            and existing.table != value.table
            and self.ctx.is_table_in_scope(existing.table)
            ):
            self.ctx.Join(
                existing.table,
                value.table,
                sql.Filter("=", value, existing),
                type=self._optional_join_type([existing.table])
            )
        elif existing and existing != value:
            # we need to add a filter for this equality
            self.ctx.Filter("=", value, existing)
        elif isinstance(arg, Var) and existing is None:
            self.ctx.bind_var(arg, value)

    #------------------------------------------------------
    # Lookup - Filter
    #------------------------------------------------------

    def _lookup_filter(self, node: Lookup):
        r = node.relation
        left, right = [self.try_resolve(arg) for arg in node.args]
        # if we have an unbound equality, we should treat = as assignment
        if r.name == "=" and left is None and right is not None:
            self.ctx.bind_var(node.args[0], right)
            return
        if left is None or right is None:
            raise ValueError(f"Cannot create filter with unbound side: {node}")
        self.ctx.Filter(r.name, left, right)

    #------------------------------------------------------
    # Lookup - Function
    #------------------------------------------------------

    def _lookup_function(self, node: Lookup):
        r = node.relation
        if r.name in {"split", "range"}:
            self._lookup_table_function(node)
            return
        # special case cast
        if r.name == "cast":
            target_type = node.args[0]
            input = self.resolve(node.args[1])
            if isinstance(target_type, Type) and is_entity_type(target_type):
                self._maybe_bind(node.args[-1], input)
                return
            # For value types (concepts extending primitives), use the primitive supertype
            type_name = target_type.name if isinstance(target_type, Type) else str(target_type)
            if isinstance(target_type, Type) and is_value_type(target_type):
                prim = get_primitive_supertype(target_type)
                if prim:
                    type_name = prim.name
            exp = sql.Expression(op="CAST", args=[input, sql.Literal(value=type_name, type="string")])
        elif r.name == "parse_number":
            args = [self.resolve(node.args[0])]
            out = node.args[-1]
            if isinstance(out, Var) and isinstance(out.type, NumberType):
                args.extend([
                    sql.Literal(value=out.type.precision, type="number(38,0)"),
                    sql.Literal(value=out.type.scale, type="number(38,0)"),
                ])
            exp = sql.Expression(op=r.name, args=args)
        else:
            args = [self.resolve(arg) for arg in node.args[:-1]]
            exp = sql.Expression(op=r.name, args=args)
        self._maybe_bind(node.args[-1], exp)

    def _lookup_table_function(self, node: Lookup):
        r = node.relation
        input_indices = [ix for ix, f in enumerate(r.fields) if f.input]
        output_indices = [ix for ix, f in enumerate(r.fields) if not f.input]
        input_args = [self.resolve(node.args[ix]) for ix in input_indices]
        specs: dict[str, dict[str, object]] = {
            "range": {
                "table_op": "RANGE_TABLE",
                "input_arity": 3,
                "outputs": [("unnest", "identity")],
            },
            "split": {
                "table_op": "SPLIT_TABLE",
                "input_arity": 2,
                "outputs": [("ordinality", "minus_one"), ("unnest", "identity")],
            },
        }
        spec = specs.get(r.name)
        if spec is None:
            raise ValueError(f"Unsupported table-valued function: {r.name}")

        input_arity = spec["input_arity"]  # type: ignore[index]
        if len(input_args) != input_arity:
            raise ValueError(f"{r.name} expects {input_arity} inputs, got {len(input_args)}")

        output_specs: list[tuple[str, str]] = spec["outputs"]  # type: ignore[index]
        if len(output_indices) != len(output_specs):
            raise ValueError(f"{r.name} expects exactly {len(output_specs)} output fields")

        output_var_ids: list[str] = []
        for ix in output_indices:
            output_arg = node.args[ix]
            if isinstance(output_arg, Var):
                output_var_ids.append(str(output_arg.id))
        alias = f"{r.name}_{'_'.join(output_var_ids)}" if output_var_ids else f"{r.name}_{node.id}"

        scan_alias = f"{r.name}_src"
        scan_ref = sql.TableRef(name=scan_alias, alias=scan_alias)
        select_aliases: list[sql.Alias] = []
        for out_ix, (source_col, transform) in zip(output_indices, output_specs):
            out_name = r.fields[out_ix].name
            source_ref = sql.ColumnRef(table=scan_ref, name=source_col)
            if transform == "identity":
                out_expr: sql.IRNode = source_ref
            elif transform == "minus_one":
                out_expr = sql.Expression(
                    op="-",
                    args=[
                        source_ref,
                        sql.Literal(value=1, type="number(38,0)"),
                    ],
                )
            else:
                raise ValueError(f"Unsupported transform {transform!r} for {r.name}")
            select_aliases.append(sql.Alias(out_expr, out_name))

        fragment = sql.Fragment(
            scans=[sql.Alias(sql.TableFunction(op=spec["table_op"], args=input_args), scan_alias)],  # type: ignore[index]
            selects=cast(list[sql.IRNode], select_aliases),
        )
        self._attach_lateral(alias, fragment)

        output_ref = sql.TableRef(name=alias, alias=alias)
        for out_ix in output_indices:
            self._maybe_bind(node.args[out_ix], sql.ColumnRef(table=output_ref, name=r.fields[out_ix].name))

    def _attach_lateral(self, alias: str, fragment: sql.Fragment):
        lateral = sql.Lateral(sql.Alias(fragment, alias))
        if lateral in self.ctx.fragment.scans:
            return
        if any(join.target == lateral for join in self.ctx.fragment.joins):
            return
        if self.ctx.fragment.scans:
            self.ctx.fragment.joins.append(
                sql.Join(
                    target=lateral,
                    incoming=_incoming_table_refs(self.ctx.fragment.scans, self._node_alias),
                    filters=[],
                    type=self._optional_join_type(self.ctx.fragment.scans),
                )
            )
        else:
            self.ctx.fragment.scans.append(lateral)

    #------------------------------------------------------
    # Lookup - Column
    #------------------------------------------------------

    def _lookup_table_column(self, node: Lookup, col: ColumnInfo):
        if not isinstance(node.args[0], Var):
            raise ValueError(f"Expected table lookup first argument to be Var, got: {type(node.args[0])}")
        table_ref = self.ctx.lookup_table_ref(node.args[0], col.table)
        # Use RowId for ID columns of external tables
        external = col.table.kind == TableKind.EXTERNAL
        if col.is_id and external:
            exp = sql.RowId(table=table_ref)
        else:
            exp = self.ctx.ColumnRef(table_ref, col)
        self._maybe_bind(node.args[-1], exp)
        # We also need to check the binding on the identity var for the column
        if len(node.args) > 1 and not external:
            self._maybe_bind(node.args[0], self.ctx.ColumnRef(table_ref, col.table.columns[0]))

    #------------------------------------------------------
    # Lookup - Relationship
    #------------------------------------------------------

    def _lookup_relationship(self, node: Lookup, table: TableInfo):
        r = node.relation
        table_ref = self.ctx.TableRef(table)
        for ix, f in enumerate(r.fields):
            col = self.analysis.get_column_info(f)
            assert col is not None, f"Cannot find column info for field {f.name} in relationship {r.name}"
            exp = self.ctx.ColumnRef(table_ref, col)
            if ix == 0:
                self._maybe_bind(node.args[ix], exp)
                continue
            arg = node.args[ix]
            existing = self.try_resolve(arg)
            if (
                isinstance(existing, (sql.ColumnRef, sql.RowId))
                and isinstance(exp, (sql.ColumnRef, sql.RowId))
                and existing.table != exp.table
                and self.ctx.is_table_in_scope(existing.table)
            ):
                self.ctx.Join(
                    existing.table,
                    exp.table,
                    sql.Filter("=", exp, existing),
                    type="INNER",
                )
            elif existing and existing != exp:
                self.ctx.Filter("=", exp, existing)
            elif isinstance(arg, Var) and existing is None:
                self.ctx.bind_var(arg, exp)

    #------------------------------------------------------
    # Update
    #------------------------------------------------------

    def update(self, node: Update):
        r = node.relation
        col = self.analysis.get_column_info(r)
        if col:
            bindings = [self.resolve(arg) for arg in node.args]
            cardinality_keys = []
            for arg, bound in zip(node.args[:-1], bindings[:-1]):
                if isinstance(arg, Var):
                    cardinality_keys.extend(self.ctx.construct_keys.get(arg, [bound] if isinstance(bound, (sql.ColumnRef, sql.RowId)) else []))
            self.analysis.writes[col].add(ColumnWrite(column=col, ctx=self.ctx, bindings=bindings, cardinality_keys=cardinality_keys))
        else:
            # check if it's a relationship
            ref = sql.TableRef(name=r.name, alias=f"{r.name}{node.id}")
            for ix, f in enumerate(r.fields):
                col = self.analysis.get_column_info(f)
                if not col:
                    raise ValueError(f"Cannot find column info for relation {r.name} in update.")
                bindings = [ref, self.resolve(node.args[ix])]
                self.analysis.writes[col].add(ColumnWrite(column=col, ctx=self.ctx, bindings=bindings))


    #------------------------------------------------------
    # Construct
    #------------------------------------------------------

    def construct(self, node: Construct):
        args = [self.resolve(arg) for arg in node.values]
        exp = sql.Expression(op="md5", args=args)
        self.ctx.bind_var(node.id_var, exp)
        self.ctx.construct_keys[node.id_var].extend([arg for arg in args if isinstance(arg, (sql.ColumnRef, sql.RowId))])

    #-----------------------------------------------------
    # Aggregate
    #------------------------------------------------------

    def enter_aggregate(self, node: Aggregate):
        body = self.capture_child_context(node.body)
        prev_ctx = self.ctx
        self.ctx = body
        projection = [self.resolve(c) for c in node.projection]
        group = [self.resolve(c) for c in node.group]
        args = [self.resolve(arg) for arg in node.args[:-1]]
        projection_cols = [col for col in projection if isinstance(col, sql.ColumnRef)]
        group_cols = [col for col in group if isinstance(col, sql.ColumnRef)]
        self.ctx = prev_ctx
        prev_ctx.used_columns.update(body.used_columns)
        output_var = node.args[-1]
        assert isinstance(output_var, Var)
        exp = sql.Expression(op=node.aggregation.name.upper(), args=args)
        agg = self.ctx.Aggregate(
            ref = exp,
            op = node.aggregation.name.upper(),
            args = args,
            body=body.fragment,
            projection=projection_cols,
            group=group_cols,
        )
        self.ctx.bind_var(output_var, agg)
        return NO_WALK

    #------------------------------------------------------
    # Match
    #------------------------------------------------------

    def complex_match(self, node: Match, match_branches: list[sql.Fragment], match_contexts: list[RuleContext], cols: list[str]) -> None:
        match_ix = "_match_ix"
        match_min_ix = "_match_min_ix"
        union_name = f"match_union_{node.id}"
        ranked_name = f"match_ranked_{node.id}"
        match_name = f"match_{node.id}"

        shared_vars = set()
        if match_contexts:
            branch_var_sets = [
                {v for v in ctx.var_map.keys() if isinstance(v, Var)}
                for ctx in match_contexts
            ]
            shared_vars = set.intersection(*branch_var_sets) if branch_var_sets else set()

        output_vars = {v for v in node.outputs if isinstance(v, Var)}
        export_vars = [
            v for v in sorted(shared_vars - output_vars, key=lambda var: var.id)
            if self.ctx.lookup_var(v) is None
        ]
        key_cols = [f"key_{i}" for i in range(len(export_vars))]

        for branch_ctx, branch in zip(match_contexts, match_branches):
            key_selects = [
                sql.Alias(branch_ctx.lookup_var(var), alias=col) # type: ignore[arg-type]
                for var, col in zip(export_vars, key_cols)
            ]
            branch.selects.extend(key_selects)

        union_branches = []
        for branch_ix, branch in enumerate(match_branches):
            branch_frag = branch.clone()
            branch_frag.selects = [
                *branch.selects,
                sql.Alias(sql.Literal(value=branch_ix, type="Number"), match_ix),
            ]
            union_branches.append(branch_frag)

        union = sql.UnionAll(union_branches)
        union_ref = sql.TableRef(union_name, union_name)
        inner = sql.Fragment(
            selects=[
                *[sql.ColumnRef(union_ref, name=col) for col in cols],
                *[sql.ColumnRef(union_ref, name=col) for col in key_cols],
                sql.ColumnRef(union_ref, name=match_ix),
                sql.Alias(
                    sql.WindowExpression(
                        op="MIN",
                        args=[sql.ColumnRef(union_ref, name=match_ix)],
                    ),
                    match_min_ix,
                ),
            ],
            scans=[sql.Alias(union, union_name)],
        )

        ranked_ref = sql.TableRef(ranked_name, ranked_name)
        outer = sql.Fragment(
            selects=[
                *[sql.ColumnRef(ranked_ref, name=col) for col in cols],
                *[sql.ColumnRef(ranked_ref, name=col) for col in key_cols],
            ],
            scans=[sql.Alias(inner, ranked_name)],
            filters=[
                sql.Filter(
                    op="=",
                    left=sql.ColumnRef(ranked_ref, name=match_ix),
                    right=sql.ColumnRef(ranked_ref, name=match_min_ix),
                )
            ],
        )

        match_ref = sql.TableRef(match_name, match_name)
        lateral = sql.Lateral(sql.Alias(outer, match_name))
        if self.ctx.fragment.scans:
            self.ctx.fragment.joins.append(
                sql.Join(
                    target=lateral,
                    incoming=_incoming_table_refs(self.ctx.fragment.scans, self._node_alias),
                    filters=[],
                    type=self._optional_join_type(self.ctx.fragment.scans),
                )
            )
        else:
            self.ctx.fragment.scans.append(lateral)

        for output, col in zip(node.outputs, cols):
            self.ctx.bind_var(output, sql.ColumnRef(match_ref, name=col))
        for var, col in zip(export_vars, key_cols):
            self.ctx.bind_var(var, sql.ColumnRef(match_ref, name=col))

    def enter_match(self, node: Match):
        if not node.outputs:
            return self._or_branches(node)
        match_branches = []
        match_contexts = []
        cols = [f"col_{i}" for i in range(len(node.outputs))]
        for branch in node.tasks:
            branch_ctx = self.capture_child_context(branch, with_selects=node.outputs)
            branch_ctx.fragment.selects = [sql.Alias(sel, alias=col) for sel, col in zip(branch_ctx.fragment.selects, cols)]
            match_branches.append(branch_ctx.fragment)
            match_contexts.append(branch_ctx)
        classified = [branch.classify() for branch in match_branches]
        if all(c == "expression" for c in classified) and len(node.outputs) == 1:
            args = [sel.node for branch in match_branches for sel in branch.selects]
            exp = sql.Expression(op="COALESCE", args=args)
            self.ctx.bind_var(node.outputs[0], exp)
        elif any(c == "query" for c in classified) or len(node.outputs) > 1:
            self.complex_match(node, match_branches, match_contexts, cols)
        else:
            cases = []
            for branch in match_branches:
                val = branch.selects[0].node
                condition = sql.And(branch.filters)
                cases.append((condition, val))
            exp = sql.CaseWhen(cases)
            assert node.outputs
            self.ctx.bind_var(node.outputs[0], exp)

        return NO_WALK

    def _or_branches(self, node: Match|Union):
        ors = []
        for branch in node.tasks:
            assert isinstance(branch, Logical)
            branch_ctx = self.capture_child_context(branch)
            if branch_ctx.fragment.classify() == "query":
                branch_ctx.fragment.selects.append(sql.Literal(1, "Integer"))
                ors.append(sql.Exists(branch_ctx.fragment))
            else:
                ands = []
                for item in branch_ctx.var_map.values():
                    if isinstance(item, sql.ColumnRef):
                        ands.append(sql.Filter("IS NOT", item, sql.Literal(None, "null")))
                ands.extend(branch_ctx.fragment.filters)
                ors.append(sql.And(ands))
        self.ctx.fragment.filters.append(sql.Or(ors))
        return NO_WALK

    #------------------------------------------------------
    # Union
    #------------------------------------------------------

    def enter_union(self, node: Union):
        if not node.outputs:
            return self._or_branches(node)
        union_branches = []
        cols = [f"col_{i}" for i in range(len(node.outputs))]
        for branch in node.tasks:
            branch_ctx = self.capture_child_context(branch, with_selects=node.outputs)
            branch_ctx.fragment.selects = [sql.Alias(sel, alias=col) for sel, col in zip(branch_ctx.fragment.selects, cols)]
            union_branches.append(branch_ctx.fragment)
        union = sql.Alias(sql.Union(union_branches), alias=f"union_{node.id}")
        table_ref = sql.TableRef(name=union.alias, alias=union.alias)
        for output in node.outputs:
            self.ctx.bind_var(output, sql.ColumnRef(table=table_ref, name=f"col_{node.outputs.index(output)}"))
        lateral = sql.Lateral(union)
        if lateral in self.ctx.fragment.scans:
            return NO_WALK
        if any(join.target == lateral for join in self.ctx.fragment.joins):
            return NO_WALK
        if self.ctx.fragment.scans:
            self.ctx.fragment.joins.append(
                sql.Join(
                    target=lateral,
                    incoming=_incoming_table_refs(self.ctx.fragment.scans, self._node_alias),
                    filters=[],
                    type=self._optional_join_type(self.ctx.fragment.scans),
                )
            )
        else:
            self.ctx.fragment.scans.append(lateral)
        return NO_WALK

    #------------------------------------------------------
    # Not
    #------------------------------------------------------

    def enter_not(self, node: Not):
        body_ctx = self.capture_child_context(node.task)
        if body_ctx.fragment.classify() == "query":
            body_ctx.fragment.selects.append(sql.Literal(1, "Integer"))
            self.ctx.fragment.filters.append(
                sql.Not(sql.Exists(body_ctx.fragment))
            )
        elif not body_ctx.fragment.filters:
            for item in body_ctx.var_map.values():
                if isinstance(item, sql.ColumnRef):
                    self.ctx.Filter("IS", item, sql.Literal(None, "null"))
        else:
            not_items = []
            not_items.extend(sql.Filter("IS NOT", item, sql.Literal(None, "null")) for item in body_ctx.var_map.values() if isinstance(item, sql.ColumnRef))
            not_items.extend(body_ctx.fragment.filters)
            n = sql.Not(sql.And(not_items))
            self.ctx.fragment.filters.append(n)
        return NO_WALK

#------------------------------------------------------
# ModelAnalysis
#------------------------------------------------------

@dataclass
class ColumnWrite:
    column: ColumnInfo = field()
    ctx: RuleContext = field()
    bindings: list[sql.IRNode] = field(default_factory=list)
    cardinality_keys: list[sql.IRNode] = field(default_factory=list)

    @property
    def reads(self) -> set[ColumnInfo]:
        all_reads = set()
        all_reads.update(self.ctx.used_columns)
        for frame in self.ctx.stack:
            all_reads.update(frame.used_columns)
        return all_reads

    @property
    def is_local(self) -> bool:
        """True if this write only uses local row data."""
        return self.ctx.is_local

    def __hash__(self):
        return hash(id(self))

@dataclass
class ModelAnalysis:
    model: mmModel = field()

    writes: dict[ColumnInfo, OrderedSet[ColumnWrite]] = field(default_factory=lambda: defaultdict(OrderedSet[ColumnWrite]))
    derivations: dict[ColumnInfo, Derivation] = field(default_factory=dict)
    strata: list[Stratum] = field(default_factory=list)

    table_info: dict[Relation, TableInfo] = field(default_factory=dict)
    column_info: dict[Relation|Field, ColumnInfo] = field(default_factory=dict)

    def get_column_info(self, relation: Relation|Field) -> ColumnInfo | None:
        found = self.column_info.get(relation, None)
        if not found:
            self._capture_relation(relation if isinstance(relation, Relation) else relation._relation)
            found = self.column_info.get(relation, None)
        return found

    def get_table_info(self, relation: Relation) -> TableInfo | None:
        found = self.table_info.get(relation, None)
        if not found and relation.fields:
            self._capture_relation(relation)
            found = self.table_info.get(relation, None)
        return found

    def is_property(self, relation: Relation) -> bool:
        if self.model._analysis.is_property(relation):
            return True
        if relation.fields and isinstance(relation.fields[0].type, Table) and  relation in relation.fields[0].type.columns:
            return True
        return False

    def _capture_relation(self, relation: Relation):
        info = self.table_info
        col_info = self.column_info

        if is_function(relation) or relation in col_info or relation in info:
            return

        if isinstance(relation, Data):
            info[relation] = TableInfo(relation, kind=TableKind.DATA)
            info[relation].columns.append(ColumnInfo(info[relation], relation))
            col_info[relation] = info[relation].columns[-1]
        elif isinstance(relation, Table):
            info[relation] = TableInfo(relation, kind=TableKind.EXTERNAL if not relation.uri.startswith("dataframe://") else TableKind.OUTPUT)
            info[relation].columns.append(ColumnInfo(info[relation], relation))
            col_info[relation] = info[relation].columns[-1]
            for table_col in relation.columns:
                self._capture_relation(table_col)
        elif isinstance(relation, ScalarType):
            if not is_entity_type(relation):
                return
            info[relation] = TableInfo(relation)
            info[relation].columns.append(ColumnInfo(info[relation], relation))
            col_info[relation] = info[relation].columns[-1]
        elif (
            self.is_property(relation)
            and (len(relation.fields) == 2 or isinstance(relation.fields[0].type, Table))
            and is_entity_type(relation.fields[0].type)
        ):
            root = relation.fields[0].type
            info[root].columns.append(ColumnInfo(info[root], relation))
            col_info[relation] = info[root].columns[-1]
        else:
            info[relation] = TableInfo(relation, kind=TableKind.RELATIONSHIP)
            for f in relation.fields:
                col = ColumnInfo(info[relation], f)
                info[relation].columns.append(col)
                col_info[f] = col

    def for_query(self) -> ModelAnalysis:
        analysis = ModelAnalysis(self.model)
        analysis.table_info = self.table_info.copy()
        analysis.column_info = self.column_info.copy()
        return analysis

#------------------------------------------------------
# Analysis
#------------------------------------------------------

class SQLModelAnalyzer:
    def __init__(self, show_debug_logs: bool = False):
        self.typer = Typer(enforce=False)
        self.normalizer = Normalize()
        self.mm_analyzer = MMAnalyzer()
        self.rule_analyzer = RuleAnalyzer()
        self.show_debug_logs = show_debug_logs

    #------------------------------------------------------
    # Normalization
    #------------------------------------------------------

    def normalize_model(self, model: mmModel) -> mmModel:
        model = self.typer.infer_model(model)
        model = model.mut(root=self.normalizer.normalize(model.root))
        self.mm_analyzer.analyze(model)
        return model

    def normalize_fragment(self, fragment: Task) -> Task:
        fragment = self.typer.infer_query(fragment)
        fragment = self.normalizer.normalize(fragment)
        return fragment

    #------------------------------------------------------
    # Table discovery
    #------------------------------------------------------

    def determine_tables(self, model: mmModel, analysis: ModelAnalysis):
        for t in model.types:
            analysis._capture_relation(t)
        for r in model.relations:
            analysis._capture_relation(r)

    #------------------------------------------------------
    # Rule analysis
    #------------------------------------------------------

    def analyze_rule(self, rule: Task):
        assert self.model_analysis is not None
        analysis = self.model_analysis.for_query()
        rule = self.normalize_fragment(rule)
        self.rule_analyzer.analyze(rule, analysis)
        return analysis

    #------------------------------------------------------
    # Stratification
    #------------------------------------------------------

    def compute_strata(self, analysis: ModelAnalysis) -> list[Stratum]:
        """
        Compute strata from ColumnWrite objects using Tarjan's SCC algorithm.
        Returns strata grouped by level (dependencies first).

        Derivations at the same "level" (can be computed in parallel) are
        grouped into the same stratum.
        """
        derivations = analysis.derivations

        if not derivations:
            return []

        # Build adjacency list: derivation -> derivations it depends on
        adj: dict[ColumnInfo, list[ColumnInfo]] = {col: [] for col in derivations}
        for col, deriv in derivations.items():
            for read_col in deriv.reads:
                if read_col in derivations and read_col != col:
                    adj[col].append(read_col)

        # Tarjan's SCC algorithm
        index_counter = [0]
        stack: list[ColumnInfo] = []
        lowlinks: dict[ColumnInfo, int] = {}
        index: dict[ColumnInfo, int] = {}
        on_stack: set[ColumnInfo] = set()
        sccs: list[list[ColumnInfo]] = []

        def strongconnect(v: ColumnInfo):
            index[v] = index_counter[0]
            lowlinks[v] = index_counter[0]
            index_counter[0] += 1
            stack.append(v)
            on_stack.add(v)

            for w in adj[v]:
                if w not in index:
                    strongconnect(w)
                    lowlinks[v] = min(lowlinks[v], lowlinks[w])
                elif w in on_stack:
                    lowlinks[v] = min(lowlinks[v], index[w])

            # If v is a root node, pop the stack and generate an SCC
            if lowlinks[v] == index[v]:
                scc: list[ColumnInfo] = []
                while True:
                    w = stack.pop()
                    on_stack.remove(w)
                    scc.append(w)
                    if w == v:
                        break
                sccs.append(scc)

        for v in derivations:
            if v not in index:
                strongconnect(v)

        # Map each column to its SCC index
        col_to_scc: dict[ColumnInfo, int] = {}
        for scc_idx, scc in enumerate(sccs):
            for col in scc:
                col_to_scc[col] = scc_idx

        # Determine which SCCs are "simple" (all derivations in the SCC are simple)
        # Simple SCCs can be inlined and don't create level barriers
        scc_is_simple: list[bool] = []
        for scc in sccs:
            all_simple = all(derivations[col].is_simple(derivations) for col in scc)
            scc_is_simple.append(all_simple)

        # Build SCC dependency graph and compute levels
        # Level = max(level of non-simple dependencies) + 1, or 0 if no non-simple dependencies
        # Simple dependencies can be inlined, so they don't create level barriers
        scc_deps: list[set[int]] = [set() for _ in sccs]
        for col, deps in adj.items():
            my_scc = col_to_scc[col]
            for dep_col in deps:
                dep_scc = col_to_scc[dep_col]
                # Only count as dependency if it's a different SCC.
                # Simple dependencies can be inlined, but only within the same table.
                if dep_scc == my_scc:
                    continue
                same_table = dep_col.table == col.table
                if not scc_is_simple[dep_scc] or not same_table:
                    scc_deps[my_scc].add(dep_scc)

        # Compute level for each SCC
        scc_levels: list[int] = [-1] * len(sccs)

        def compute_level(scc_idx: int) -> int:
            if scc_levels[scc_idx] >= 0:
                return scc_levels[scc_idx]
            if not scc_deps[scc_idx]:
                scc_levels[scc_idx] = 0
            else:
                scc_levels[scc_idx] = max(compute_level(dep) for dep in scc_deps[scc_idx]) + 1
            return scc_levels[scc_idx]

        for i in range(len(sccs)):
            compute_level(i)

        # Group SCCs by level
        max_level = max(scc_levels) if scc_levels else 0
        levels: list[list[int]] = [[] for _ in range(max_level + 1)]
        for scc_idx, level in enumerate(scc_levels):
            levels[level].append(scc_idx)

        # Create Stratum objects per level
        strata: list[Stratum] = []
        for level, scc_indices in enumerate(levels):
            all_derivs: list[Derivation] = []
            recursive_groups: list[RecursiveGroup] = []

            for scc_idx in scc_indices:
                scc = sccs[scc_idx]
                derivs = [derivations[col] for col in scc]
                all_derivs.extend(derivs)

                # Check if this SCC is recursive
                scc_is_recursive = len(scc) > 1
                if not scc_is_recursive and len(scc) == 1:
                    col = scc[0]
                    scc_is_recursive = col in derivations[col].reads

                if scc_is_recursive:
                    recursive_groups.append(RecursiveGroup(derivations=derivs))

            stratum = Stratum(id=level, derivations=all_derivs, recursive_groups=recursive_groups)
            strata.append(stratum)

        return strata

    #------------------------------------------------------
    # Analyze
    #------------------------------------------------------

    def analyze(self, model: mmModel):
        model = self.normalize_model(model)
        self.model_analysis = ModelAnalysis(model)
        self.determine_tables(model, self.model_analysis)
        assert isinstance(model.root, Logical)
        for item in model.root.body:
            self.rule_analyzer.analyze(item, self.model_analysis)  # pass column info

        # Build derivations and compute strata
        self.model_analysis.derivations = {
            col: Derivation(col, writes)
            for col, writes in self.model_analysis.writes.items()
        }
        self.model_analysis.strata = self.compute_strata(self.model_analysis)

        return self.model_analysis

#------------------------------------------------------
# ColumnMapper
#------------------------------------------------------

class ColumnMapper(sql.IRRewriter):
    def __init__(self, ctx:PlanContext):
        self.ctx = ctx
        self.mappings: dict[ColumnInfo, sql.IRNode] = {}

    def rewrite_with_mappings(self, node: sql.IRNode, mappings: dict[ColumnInfo, sql.IRNode]) -> sql.IRNode:
        self.mappings = mappings
        out = self.rewrite(node)
        self.mappings = {}
        return out

    def rewrite_tableref(self, node: sql.TableRef) -> sql.IRNode:
        if node.name in self.ctx.active_views_str:
            active = self.ctx.active_views_str[node.name]
            return sql.TableRef(active.name, node.alias)
        return node

    def rewrite_columnref(self, node: sql.ColumnRef) -> sql.IRNode:
        if node.column is None:
            return node
        mapped = self.mappings.get(node.column, None)
        if mapped:
            return self.rewrite(mapped)
        return node

#------------------------------------------------------
# ViewRenamer
#------------------------------------------------------

class ViewRenamer(sql.IRRewriter):
    """Rewrites TableRef nodes to use final view names instead of intermediate names."""
    def __init__(self, renames: dict[str, str]):
        self.renames = renames

    def rewrite_tableref(self, node: sql.TableRef) -> sql.IRNode:
        if node.name in self.renames:
            return sql.TableRef(self.renames[node.name], node.alias)
        return node

#------------------------------------------------------
# Execution plan
#------------------------------------------------------

@dataclass
class PlanContext:
    step: ExecutionStep = field()
    active_views_str: dict[str, sql.TableRef] = field(default_factory=dict)
    last_table_view: dict[TableInfo, sql.CreateView] = field(default_factory=dict)
    active_table_columns: dict[TableInfo, set[ColumnInfo]] = field(default_factory=dict)

    def add_view(self, table: TableInfo, view: sql.CreateView, columns: set[ColumnInfo] | None = None):
        self.step.views.append(view)
        self.active_views_str[table.name] = sql.TableRef(name=view.name, alias=table.name)
        self.last_table_view[table] = view
        self.active_table_columns[table] = columns or set(table.columns)

class ExecutionStep:
    def __init__(self, stratum: Stratum):
        self.stratum = stratum
        self.views: list[sql.CreateView] = []
        self.semi_naive_loops: list[sql.SemiNaiveLoop] = []

class ExecutionPlan:
    def __init__(self, analysis: ModelAnalysis, model_schema: str | None = None):
        self.analysis = analysis
        self.model_schema = model_schema
        self.strata = analysis.strata
        self.steps: list[ExecutionStep] = []
        self.ctx = PlanContext(step=None) # type: ignore
        self.remapper = ColumnMapper(self.ctx)
        self._build()

    #------------------------------------------------------
    # Helpers
    #------------------------------------------------------

    def _group_writes(self, stratum: Stratum):
        table_derivations = defaultdict(list)
        for deriv in stratum.derivations:
            table_derivations[deriv.column.table].append(deriv)
        groups: defaultdict[TableInfo, defaultdict[sql.IRNode, list]] = defaultdict(lambda: defaultdict(list))
        for col in stratum.columns:
            for write in self.analysis.writes.get(col, []):
                groups[write.column.table][write.bindings[0]].append(write)
        return groups, table_derivations

    def _qualified_name(self, table: TableInfo, name: str | None = None) -> str:
        if table.kind in {TableKind.EXTERNAL, TableKind.OUTPUT}:
            return name or table.name

        target = name or table.name
        if not self.model_schema or "." in target:
            return target
        return f"{self.model_schema}.{target}"

    def _group_writes_by_table(self, writes: Iterable[ColumnWrite]) -> dict[TableInfo, dict[sql.IRNode, list[ColumnWrite]]]:
        groups: dict[TableInfo, dict[sql.IRNode, list[ColumnWrite]]] = defaultdict(lambda: defaultdict(list))
        for write in writes:
            groups[write.column.table][write.bindings[0]].append(write)
        return groups

    def _filter_write_groups(self, write_groups: dict[sql.IRNode, list[ColumnWrite]], excluded: set[ColumnInfo]) -> dict[sql.IRNode, list[ColumnWrite]]:
        if not excluded:
            return write_groups
        filtered: dict[sql.IRNode, list[ColumnWrite]] = {}
        for key, writes in write_groups.items():
            kept = [w for w in writes if w.column not in excluded]
            if kept:
                filtered[key] = kept
        return filtered

    def _selects_for_table(self, table: TableInfo, write_group: Sequence[ColumnWrite]) -> list[sql.Alias]:
        by_col = {w.column: w for w in write_group}
        null_lit = sql.Literal(value=None, type="null")
        return [
            sql.Alias(by_col[col].bindings[-1], col.name) if col in by_col
            else sql.Alias(null_lit, col.name)
            for col in table.columns
        ]

    def _fragment_from_existing(self, table: TableInfo, existing: sql.TableRef, existing_cols: set[ColumnInfo]) -> sql.Fragment:
        null_lit = sql.Literal(value=None, type="null")
        selects = [
            sql.Alias(sql.ColumnRef(existing, name=col.name, column=col), col.name) if col in existing_cols
            else sql.Alias(null_lit, col.name)
            for col in table.columns
        ]
        return sql.Fragment(selects=cast(list[sql.IRNode], selects), scans=[existing])

    def _empty_fragment_for_table(self, table: TableInfo) -> sql.Fragment:
        null_lit = sql.Literal(value=None, type="null")
        selects = [sql.Alias(null_lit, col.name) for col in table.columns]
        # WHERE 1 = 0 to produce an empty result
        false_filter = sql.Filter("=", sql.Literal(value=1, type="number"), sql.Literal(value=0, type="number"))
        return sql.Fragment(selects=cast(list[sql.IRNode], selects), filters=[false_filter])

    #------------------------------------------------------
    # Semi-naive recursion
    #------------------------------------------------------

    def _union_or_empty(self, table: TableInfo, fragments: list[sql.Fragment]) -> sql.IRNode:
        if not fragments:
            return self._empty_fragment_for_table(table)
        if len(fragments) == 1:
            return fragments[0]
        return sql.UnionAll(fragments)

    def _normalize_recursive(self, table: TableInfo, source: sql.IRNode, alias: str) -> sql.IRNode:
        union_ref = sql.TableRef(alias, alias)
        if table.kind == TableKind.RELATIONSHIP:
            group_cols = [sql.ColumnRef(union_ref, name=col.name, column=col) for col in table.columns]
            max_cols: list[sql.ColumnRef] = []
        else:
            id_col = table.columns[0]
            group_cols = [sql.ColumnRef(union_ref, name=id_col.name, column=id_col)]
            max_cols = [sql.ColumnRef(union_ref, name=col.name, column=col) for col in table.columns[1:]]
        return sql.GroupByMax(
            sql.Alias(source, alias),
            group_cols=group_cols,
            max_cols=max_cols,
        )

    def _remap_with_recursive_passthrough(self, frag: sql.Fragment, recursive_tables: set[TableInfo]) -> sql.Fragment:
        saved: dict[str, sql.TableRef] = {}
        for table in recursive_tables:
            if table.name in self.ctx.active_views_str:
                saved[table.name] = self.ctx.active_views_str[table.name]
            # Force recursive tables to map to their logical name
            self.ctx.active_views_str[table.name] = sql.TableRef(name=table.name, alias=table.name)
        out = self.remapper.rewrite(frag)
        for table in recursive_tables:
            if table.name in saved:
                self.ctx.active_views_str[table.name] = saved[table.name]
            else:
                self.ctx.active_views_str.pop(table.name, None)
        return out

    def _collect_recursive_refs(self, frag: sql.Fragment, recursive_names: set[str]) -> list[sql.TableRef]:
        class RefCollector(sql.IRRewriter):
            def __init__(self):
                self.refs: list[sql.TableRef] = []

            def rewrite_tableref(self, node: sql.TableRef) -> sql.IRNode:
                self.refs.append(node)
                return node

            def rewrite_columnref(self, node: sql.ColumnRef) -> sql.IRNode:
                self.refs.append(node.table)
                return node

            def rewrite_rowid(self, node: sql.RowId) -> sql.IRNode:
                self.refs.append(node.table)
                return node

        collector = RefCollector()
        collector.rewrite(frag)
        seen: set[int] = set()
        refs: list[sql.TableRef] = []
        for ref in collector.refs:
            ref_id = id(ref)
            if ref.name in recursive_names and ref_id not in seen:
                refs.append(ref)
                seen.add(ref_id)
        return refs

    def _build_delta_variants(
        self,
        frag: sql.Fragment,
        recursive_names: set[str],
        delta_names: dict[str, str],
        accum_names: dict[str, str],
    ) -> list[sql.Fragment]:
        recursive_refs = self._collect_recursive_refs(frag, recursive_names)
        if not recursive_refs:
            return []

        class RefRewriter(sql.IRRewriter):
            def __init__(self, mapping: dict[int, sql.TableRef]):
                self.mapping = mapping

            def rewrite_tableref(self, node: sql.TableRef) -> sql.IRNode:
                return self.mapping.get(id(node), node)

            def rewrite_columnref(self, node: sql.ColumnRef) -> sql.IRNode:
                mapped = self.mapping.get(id(node.table))
                if mapped:
                    return sql.ColumnRef(mapped, name=node.name, column=node.column)
                return node

        # Group refs by alias. Multiple Python objects can represent the same
        # logical table reference (scan, join target, join incoming, ColumnRef.table).
        # All refs sharing an alias must map consistently to delta or accum to
        # avoid duplicate-alias errors in the generated SQL.
        alias_groups: dict[str, list[sql.TableRef]] = {}
        for ref in recursive_refs:
            alias_groups.setdefault(ref.alias, []).append(ref)

        # Create one variant per unique alias: the alias's refs map to delta,
        # all other aliases map to accum.
        variants: list[sql.Fragment] = []
        for delta_alias in alias_groups:
            mapping: dict[int, sql.TableRef] = {}
            for alias, refs in alias_groups.items():
                is_delta = alias == delta_alias
                for ref in refs:
                    target_name = delta_names[ref.name] if is_delta else accum_names[ref.name]
                    mapping[id(ref)] = sql.TableRef(name=target_name, alias=ref.alias)
            variants.append(RefRewriter(mapping).rewrite(frag))
        return variants

    def _build_recursive_group(self, ctx: PlanContext, group: RecursiveGroup, stratum_id: int, group_index: int) -> tuple[sql.SemiNaiveLoop, list[TableInfo]]:
        base_groups = self._group_writes_by_table(group.base_writes)
        recursive_groups = self._group_writes_by_table(group.recursive_writes)

        tables = list(dict.fromkeys(deriv.column.table for deriv in group.derivations))
        recursive_tables = set(tables)
        suffix = f"{stratum_id}_{group_index}"
        accum_names = {t.name: f"{t.name}__accum_{suffix}" for t in tables}
        delta_names = {t.name: f"{t.name}__delta_{suffix}" for t in tables}
        next_delta_names = {t.name: f"{t.name}__delta_next_{suffix}" for t in tables}
        recursive_names = set(accum_names)

        def _star_fragment(ref: sql.TableRef) -> sql.Fragment:
            return sql.Fragment(selects=[sql.Star(ref)], scans=[ref])

        def _write_fragments(
            table: TableInfo,
            groups: dict[TableInfo, dict[sql.IRNode, list[ColumnWrite]]],
            *,
            recursive: bool,
        ) -> list[sql.Fragment]:
            fragments: list[sql.Fragment] = []
            for write_group in groups.get(table, {}).values():
                selects = self._selects_for_table(table, write_group)
                frag = self._ctx_to_fragment(write_group, selects)
                if recursive:
                    frag = self._remap_with_recursive_passthrough(frag, recursive_tables)
                else:
                    frag = self.remapper.rewrite(frag)
                fragments.append(frag)
            return fragments

        loop_tables: list[sql.SemiNaiveTable] = []

        for table in tables:
            base_fragments: list[sql.Fragment] = []
            existing = ctx.active_views_str.get(table.name)
            if existing:
                existing_cols = ctx.active_table_columns.get(table, set())
                base_fragments.append(self._fragment_from_existing(table, existing, existing_cols))
            base_fragments.extend(_write_fragments(table, base_groups, recursive=False))

            base_query = self._normalize_recursive(
                table,
                self._union_or_empty(table, base_fragments),
                alias=f"{table.name}_base_union_{suffix}",
            )

            recursive_fragments = _write_fragments(table, recursive_groups, recursive=True)
            delta_variants = [
                variant
                for frag in recursive_fragments
                for variant in self._build_delta_variants(frag, recursive_names, delta_names, accum_names)
            ]
            delta_query = self._normalize_recursive(
                table,
                self._union_or_empty(table, delta_variants),
                alias=f"{table.name}_delta_union_{suffix}",
            )

            accum_name = accum_names[table.name]
            next_name = next_delta_names[table.name]
            accum_ref = sql.TableRef(accum_name, accum_name)
            next_ref = sql.TableRef(next_name, next_name)
            merge_query = self._normalize_recursive(
                table,
                sql.UnionAll([_star_fragment(accum_ref), _star_fragment(next_ref)]),
                alias=f"{table.name}_merge_union_{suffix}",
            )

            final_query = self._normalize_recursive(
                table,
                _star_fragment(sql.TableRef(accum_name, accum_name)),
                alias=f"{table.name}_final_union_{suffix}",
            )
            loop_view_name = self._qualified_name(table, f"{table.name}_rec_{suffix}")

            loop_tables.append(sql.SemiNaiveTable(
                table_name=loop_view_name,
                columns=[col.name for col in table.columns],
                base_query=base_query,
                delta_query=delta_query,
                merge_query=merge_query,
                final_query=final_query,
                accum_view_name=accum_names[table.name],
                delta_view_name=delta_names[table.name],
                next_delta_view_name=next_delta_names[table.name],
            ))

        return sql.SemiNaiveLoop(tables=loop_tables), tables

    #------------------------------------------------------
    # Build
    #------------------------------------------------------

    def _build(self):

        self.ctx.step = ExecutionStep(Stratum(id=-1, derivations=[]))
        self.steps.append(self.ctx.step)
        for table in self.analysis.table_info.values():
            if table.kind == TableKind.DATA:
                self._build_data(self.ctx, table)

        for stratum in self.strata:
            self.ctx.step = ExecutionStep(stratum)
            self.steps.append(self.ctx.step)
            grouped, table_derivations = self._group_writes(stratum)
            recursive_columns = {col for group in stratum.recursive_groups for col in group.columns}
            recursive_tables = {col.table for col in recursive_columns}

            for table, write_groups in grouped.items():
                filtered_groups = self._filter_write_groups(write_groups, recursive_columns)
                filtered_derivs = [d for d in table_derivations.get(table, []) if d.column not in recursive_columns]
                if table.kind == TableKind.RELATIONSHIP:
                    if table in recursive_tables:
                        continue
                    if filtered_groups:
                        self._build_relationship(self.ctx, table, filtered_groups)
                elif table.kind == TableKind.CONCEPT:
                    if filtered_groups or filtered_derivs:
                        self._build_concept(self.ctx, table, filtered_groups, filtered_derivs)
                elif table.kind == TableKind.DATA:
                    pass

            for ix, group in enumerate(stratum.recursive_groups):
                loop, tables = self._build_recursive_group(self.ctx, group, stratum.id, ix)
                self.ctx.step.semi_naive_loops.append(loop)
                for table, loop_table in zip(tables, loop.tables):
                    active_name = loop_table.table_name
                    self.ctx.active_views_str[table.name] = sql.TableRef(name=active_name, alias=table.name)
                    self.ctx.active_table_columns[table] = set(table.columns)
                    # Track latest recursive view; it may be renamed to the table's final name later.
                    self.ctx.last_table_view[table] = sql.CreateView(
                        name=active_name,
                        query=loop_table.final_query,
                    )

        # Build a mapping from intermediate view names to final table names
        view_renames: dict[str, str] = {}
        for table, view in self.ctx.last_table_view.items():
            final_name = self._qualified_name(table)
            if view.name != final_name:
                view_renames[view.name] = final_name
                view.name = final_name

        # Rewrite all views to use the final table names instead of intermediate view names
        if view_renames:
            renamer = ViewRenamer(view_renames)
            for step in self.steps:
                for view in step.views:
                    view.query = renamer.rewrite(view.query)
                for i, loop in enumerate(step.semi_naive_loops):
                    step.semi_naive_loops[i] = renamer.rewrite(loop)
            for table_name, active in list(self.ctx.active_views_str.items()):
                renamed = view_renames.get(active.name)
                if renamed:
                    self.ctx.active_views_str[table_name] = sql.TableRef(name=renamed, alias=active.alias)

    def _ctx_to_fragment(self, writes: Sequence[ColumnWrite], selects: Sequence[sql.IRNode]) -> sql.Fragment:
        frag = merge_contexts([w.ctx for w in writes])
        frag.selects.extend(selects)

        cardinality_keys = {
            key for w in writes for key in w.cardinality_keys
            if isinstance(key, (sql.ColumnRef, sql.RowId))
        }
        normalizer = FragmentNormalizer(cardinality_keys)
        frag = normalizer.rewrite(frag)

        return frag

    #------------------------------------------------------
    # Concept
    #------------------------------------------------------

    @staticmethod
    def _root_ctx_id(ctx: RuleContext) -> int:
        """Get the identity of the root context (from the same analyze() call)."""
        return id(ctx.stack[0]) if ctx.stack else id(ctx)

    def _build_concept(self, ctx: PlanContext, table: TableInfo, write_groups: dict[sql.IRNode, list[ColumnWrite]], table_derivations: list[Derivation]):
        base_cols = set()
        base_writes = []
        for write_group in write_groups.values():
            id_writes = [w for w in write_group if w.column.is_id]
            if id_writes:
                # Only include writes from the same rule that asserts set membership (_id).
                id_roots = {self._root_ctx_id(w.ctx) for w in id_writes}
                same_rule = [w for w in write_group if self._root_ctx_id(w.ctx) in id_roots]
                base_writes.append(same_rule)
                base_cols.update(w.column for w in same_rule)
        if base_writes:
            self._build_concept_base(ctx, table, base_writes)
        remaining = [d for d in table_derivations if d.column not in base_cols]
        if remaining:
            self._build_concept_derived(ctx, table, remaining)

    def _build_concept_base(self, ctx: PlanContext, table: TableInfo, write_groups: list[list[ColumnWrite]]):
        # Collect all columns that appear in any write group, preserving table column order.
        base_cols_set = {w.column for group in write_groups for w in group}
        base_cols = [c for c in table.columns if c in base_cols_set]

        null_lit = sql.Literal(value=None, type="null")

        fragments: list[sql.Fragment] = []
        for group in write_groups:
            by_col = {w.column: w for w in group}
            selects = [
                sql.Alias(by_col[col].bindings[-1], col.name) if col in by_col
                else sql.Alias(null_lit, col.name)
                for col in base_cols
            ]
            frag = self._ctx_to_fragment(group, selects)
            fragments.append(frag)

        final_query = fragments[0]
        if len(fragments) > 1:
            union_name = f"{table.name}_union"
            union_ref = sql.TableRef(union_name, union_name)

            final_query = sql.GroupByMax(
                sql.Alias(sql.UnionAll(fragments), union_name),
                group_cols=[sql.ColumnRef(union_ref, name=table.columns[0].name, column=table.columns[0])],  # ID column
                max_cols=[sql.ColumnRef(union_ref, name=col.name, column=col) for col in base_cols[1:]],
            )

        ctx.add_view(
            table,
            sql.CreateView(name=self._qualified_name(table, f"{table.name}_base"), query=final_query),
            columns=set(base_cols),
        )

    def _build_concept_derived(self, ctx: PlanContext, table: TableInfo, table_derivations: list[Derivation]):
        branches = []
        cols = [deriv.column for deriv in table_derivations]
        local_mappings = {}
        for deriv in table_derivations:
            for write in deriv.writes:
                selects = [
                    sql.Alias(write.bindings[0], deriv.column.table.columns[0].name),
                    *[sql.Alias(write.bindings[-1], deriv.column.name) if deriv.column == col else sql.Alias(sql.Literal(value=None, type="null"), col.name) for col in cols]
                ]
                frag = self._ctx_to_fragment([write], selects)
                frag = self.remapper.rewrite_with_mappings(frag, local_mappings)
                local_mappings[deriv.column] = write.bindings[-1]
                branches.append(frag)
        active_ref = self.ctx.active_views_str.get(table.name)
        existing = None
        existing_cols = set()
        if active_ref is not None:
            active_name = active_ref.name
            existing = next((scan for scan in branches[0].scans if isinstance(scan, sql.TableRef) and scan.name == active_name), None)
            if existing is None:
                existing = sql.TableRef(name=active_name, alias=f"t_{table.name}")
            existing_cols = self.ctx.active_table_columns.get(table, set())
        if len(branches) == 1:
            if existing is None:
                final = branches[0]
            else:
                derived_ref = sql.TableRef("derived", "derived")
                alias_derived = sql.Alias(branches[0], "derived")
                derived_cols = set(cols)
                null_lit = sql.Literal(value=None, type="null")
                final = sql.Fragment(
                    selects=[
                        sql.ColumnRef(derived_ref, name=col.name, column=col)
                        if col in derived_cols
                        else sql.ColumnRef(existing, name=col.name, column=col)
                        if col in existing_cols
                        else sql.Alias(null_lit, col.name)
                        for col in table.columns
                    ],
                    scans=[existing],
                    joins=[
                        sql.Join(
                            target=alias_derived,
                            incoming={existing},
                            filters=[
                                sql.Filter(
                                    "=",
                                    sql.ColumnRef(existing, name=table.columns[0].name, column=table.columns[0]),
                                    sql.ColumnRef(derived_ref, name=table.columns[0].name, column=table.columns[0]),
                                )
                            ],
                            type="LEFT",
                        )
                    ],
                )
        else:
            union_name = f"{table.name}_derived__union"
            union_ref = sql.TableRef(union_name, union_name)
            union = sql.UnionAll(branches)
            max = sql.GroupByMax(
                sql.Alias(union, union_name),
                group_cols=[sql.ColumnRef(union_ref, name=table.columns[0].name, column=table.columns[0])],  # ID column
                max_cols=[sql.ColumnRef(union_ref, name=c.name, column=c) for c in cols],
            )
            collapsed_ref = sql.TableRef("collapsed", "collapsed")
            alias_max = sql.Alias(max, "collapsed")
            if existing is not None:
                derived_cols = set(cols)
                null_lit = sql.Literal(value=None, type="null")
                final = sql.Fragment(
                    selects=[
                        sql.ColumnRef(collapsed_ref, name=col.name, column=col)
                        if col in derived_cols
                        else sql.ColumnRef(existing, name=col.name, column=col)
                        if col in existing_cols
                        else sql.Alias(null_lit, col.name)
                        for col in table.columns
                    ],
                    scans=[existing],
                    joins=[
                        sql.Join(
                            target=alias_max,
                            incoming={existing},
                            filters=[
                                sql.Filter("=", sql.ColumnRef(existing, name=table.columns[0].name, column=table.columns[0]), sql.ColumnRef(collapsed_ref, name=table.columns[0].name, column=table.columns[0]))
                            ],
                            type="LEFT"
                        )
                    ]
                )
            else:
                final = sql.Fragment(
                    selects=[sql.Star(collapsed_ref)],
                    scans=[alias_max],
                )
        view = sql.CreateView(name=self._qualified_name(table, f"{table.name}_{ctx.step.stratum.id}"), query=final)
        ctx.add_view(table, view, columns=set(table.columns))

    #------------------------------------------------------
    # Relationship
    #------------------------------------------------------

    def _build_relationship(self, ctx: PlanContext, table: TableInfo, write_groups: dict[sql.IRNode, list[ColumnWrite]]):
        fragments = []
        for cols in write_groups.values():
            sorted_cols = sorted(cols, key=lambda w: table.columns.index(w.column))
            selects = [sql.Alias(w.bindings[-1], w.column.name) for w in sorted_cols]
            frag = self._ctx_to_fragment(sorted_cols, selects)
            # Relationship tuples should not materialize NULL-valued fields.
            frag.filters.extend(
                sql.Filter("IS NOT", w.bindings[-1], sql.Literal(None, "null"))
                for w in sorted_cols
            )
            frag = self.remapper.rewrite(frag)
            fragments.append(frag)
        union = sql.UnionAll(fragments)
        view = sql.CreateView(name=self._qualified_name(table), query=union)
        ctx.add_view(table, view, columns=set(table.columns))

    #------------------------------------------------------
    # Data
    #------------------------------------------------------

    def _build_data(self, ctx: PlanContext, table: TableInfo):
        assert isinstance(table.relation, Data)
        frag = sql.Fragment(
            selects=[sql.Star(sql.TableRef(name=table.name+"_data"))],
            scans=[sql.Values(
                data=table.relation.data,
                alias=table.name+"_data",
                columns=[col for col in table.columns],
            )]
        )
        view = sql.CreateView(name=self._qualified_name(table), query=frag)
        ctx.add_view(table, view, columns=set(table.columns))
        return



#------------------------------------------------------
# Compiler
#------------------------------------------------------

class SQLCompiler:
    def __init__(self, dialect: SQLDialect | None = None, config: Config | None = None):
        self.last_model_version = -1
        self.config = config
        show_debug_logs = config.debug.show_debug_logs if config else False
        self._analyzer = SQLModelAnalyzer(show_debug_logs=show_debug_logs)
        self.emitter = SQLEmitter(dialect or DuckDBDialect())

    def _resolve_model_schema(self, model: Model) -> str | None:
        config_source = self.config if self.config is not None else model.config
        if config_source is None:
            return None
        model_cfg = getattr(config_source, "model", None)
        schema = getattr(model_cfg, "schema_", None)
        if not isinstance(schema, str):
            return None
        schema = schema.strip()
        return schema or None

    def compile_model(self, model: Model):
        if model._version == self.last_model_version:
            # nothing to do, we're up to date
            return []

        # otherwise we need to recompile
        self.last_model_version = model._version

        analysis = self._analyzer.analyze(model.to_metamodel())

        # Debug: show strata
        # for stratum in analysis.strata:
        #     cols = [d.column.name for d in stratum.derivations]
        #     print(f"Stratum {stratum.id}: {cols} recursive={stratum.is_recursive}")
        #     if stratum.is_recursive:
        #         print(f"  base_writes: {[w.column.name for w in stratum.base_writes]}")
        #         print(f"  recursive_writes: {[w.column.name for w in stratum.recursive_writes]}")

        # Build execution plan
        model_schema = self._resolve_model_schema(model)
        execution_plan = ExecutionPlan(analysis, model_schema=model_schema)
        self.execution_plan = execution_plan

        view_sql = []
        for step in execution_plan.steps:
            # print(f"Execution Step {step.stratum.id}:")
            for view in step.views:
                sql_str = self.emitter.emit(view)
                # print(sql_str)
                view_sql.append(sql_str)
            for loop in step.semi_naive_loops:
                pass

        return view_sql

    def compile_fragment(self, fragment: Fragment):
        metamodel_rule = fragment.to_metamodel()
        assert isinstance(metamodel_rule, Task), "Expected fragment.to_metamodel() to return a single Task."
        analysis = self._analyzer.analyze_rule(metamodel_rule)
        for table in analysis.table_info.values():
            if table.kind == TableKind.OUTPUT:
                ws = self.group_writes(analysis, table)
                assert len(ws) == 1, "Output tables should only have one write group."
                for write_group in ws.values():
                    return self.compile_select(
                        [w for w in write_group if not w.column.is_id],
                        quote_select_aliases=True,
                    )

    #------------------------------------------------------
    # Select handling
    #------------------------------------------------------

    def group_writes(self, analysis: ModelAnalysis, table:TableInfo) -> dict[sql.IRNode, list[ColumnWrite]]:
        groups = defaultdict(list)
        for col in table.columns:
            for write in analysis.writes.get(col, []):
                groups[write.bindings[0]].append(write)
        return groups

    def compile_select(self, writes: list[ColumnWrite], quote_select_aliases: bool = False):
        selects = [sql.Alias(w.bindings[-1], w.column.name) for w in writes]
        frag = self.execution_plan._ctx_to_fragment(writes, selects)
        frag = self.execution_plan.remapper.rewrite(frag)
        return self.emitter.emit(frag, quote_select_aliases=quote_select_aliases)


#------------------------------------------------------
# TODO
#------------------------------------------------------

"""
- Not
- Match
- Union
- Semi-naive


"""
