"""Semantics DSL building blocks.

This module defines the core *frontend* building blocks used to construct
RelationalAI semantics queries in Python. Objects like :class:`Variable`,
:class:`Concept`, :class:`Relationship`, and :class:`Expression` represent
**symbolic expressions**: you combine them with Python syntax (attributes, calls,
operators) to build a query, and the query is executed later by the RelationalAI runtime.

Notes
-----
Most users start from :class:`Model` and create concepts/relationships from there;
you typically do not instantiate the classes in this module directly.

Examples
--------
Create a model, a concept, and a simple query:

>>> from relationalai.semantics import Model
>>> m = Model()
>>> Person = m.Concept("Person")
>>> q = m.select(Person).where(Person.age >= 18)
"""
from __future__ import annotations

import decimal
import math
import re
from typing import TYPE_CHECKING, Any, Iterator, NoReturn, Optional, Sequence, Tuple, Type, TypeGuard, cast
import itertools
from more_itertools import peekable
from enum import Enum, EnumMeta

from relationalai.config.config import Config, create_config, NoOpConfig

if TYPE_CHECKING:
    from pandas import DataFrame
    from ..experimental.loopy.builder import ProcedureBuilder
    from ..backends.executor import Executor

from ...util import schema as schema_util
from ...util.structures import KeyedSet
from ...util.naming import Namer, sanitize
from ...util.python import pytype_to_concept_name, column_to_concept_name
from ...util.source import SourcePos
from ...util.tracing import get_tracer
from ...util.error import exc, warn, info, source
from ...util.docutils import include_in_docs
from ..metamodel.builtins import builtins
from ..metamodel.metamodel import Model as mModel

__include_in_docs__ = True

tracer = get_tracer()
tracer.start_program()

#------------------------------------------------------
# Global ID Generator
#------------------------------------------------------

_global_id = peekable(itertools.count(0))

#------------------------------------------------------
# Helpers
#------------------------------------------------------

def _find_field(fields: list[Field], field: str|int|Concept) -> tuple[Field, int]|None:
    resolved = None
    if isinstance(field, int):
        if not (0 <= field < len(fields)):
            raise IndexError(f"Field index {field} out of range for relationship with {len(fields)} fields")
        resolved = fields[field]
    elif isinstance(field, str):
        resolved = next((f for f in fields if f.name.lower() == field.lower()), None)
    elif isinstance(field, Concept):
        resolved = next((f for f in fields if f.type is field), None)
    if not resolved:
        return None
    return resolved, fields.index(resolved)

def _find_concept(model: Model, passed_type: Any, extra_types: Sequence[Concept] = [], default:Concept|None = None) -> Concept | None:
    if isinstance(passed_type, Concept):
        return passed_type
    if isinstance(passed_type, str):
        for ext_type in extra_types:
            if ext_type._name == passed_type:
                return ext_type
        name = pytype_to_concept_name.get(passed_type, passed_type)
        if name.startswith("Number(") or name.startswith("Decimal("):
            return cast(Any, CoreConcepts["Number"]).parse(name)
        return model._find_concept(name)
    #TODO: try to find a variable in the frame that is a concept with this name
    return default

#------------------------------------------------------
# Primitive
#------------------------------------------------------

Primitive = str|int|float|bool
"""Union of Python primitive literal types.

``Primitive`` is used for values that can be embedded directly as DSL literals
(for example ``"Alice"`` or ``42``) when building expressions.
"""

def is_primitive(value: Any) -> TypeGuard[str | int | float | bool]:
    return isinstance(value, (str, int, float, bool))

#------------------------------------------------------
# DSLBase
#------------------------------------------------------

@include_in_docs
class DSLBase:
    """Base class for semantics DSL objects.

    ``DSLBase`` provides a unique, stable identity for objects in the semantics
    DSL and captures source information used to produce helpful error
    messages.

    Notes
    -----
    Most users should not instantiate this class directly. You typically work
    with subclasses such as :class:`Variable`, :class:`Concept`, and :class:`Fragment`.

    Parameters
    ----------
    model : Model
        The model this object belongs to.
    """

    def __init__(self, model:Model):
        self._model = model
        self._id = next(_global_id)
        self._source = SourcePos.new()

    #------------------------------------------------------
    # Hashing
    #------------------------------------------------------

    def to_dict_key(self) -> int|tuple[int, int]:
        return self._id

    def __hash__(self) -> NoReturn:
        exc("Unhashable type", f"{self.__class__.__name__} objects are unhashable. Use `to_dict_key()` instead.", [source(self)])

#------------------------------------------------------
# DSLDict
#------------------------------------------------------

def dsl_key(obj:Any) -> Any:
    if isinstance(obj, DSLBase):
        return obj.to_dict_key()
    elif isinstance(obj, tuple):
        return tuple(dsl_key(o) for o in obj)
    return obj

#------------------------------------------------------
# Variable
#------------------------------------------------------

@include_in_docs
class Variable(DSLBase):
    """Represents a composable value in the semantics DSL.

    A ``Variable`` is the base type for most objects you use when building
    queries and definitions. Variables are *placeholders* that can be
    combined into larger expressions via Python operators and then passed to
    fragment/model methods like :meth:`Model.where` and :meth:`Model.select`.

    Common subclasses include :class:`Concept` (entity types), :class:`Relationship`,
    :class:`Ref` (named entity references created by :meth:`Concept.ref`),
    :class:`Chain` (property/relationship access like ``Person.age``), and
    :class:`Expression` (relationship calls and composed arithmetic/comparison expressions).

    Notes
    -----
    Most users should not instantiate this class directly.
    You typically obtain variables by:

    - Creating concepts via :meth:`Model.Concept`.
    - Accessing relationships as attributes (e.g. ``Person.age``).
    - Creating reference variables via :meth:`Concept.ref`.

    Variables overload several Python operators to build DSL expressions:

    - Arithmetic operators like ``+`` and ``*`` return an
      :class:`Expression`.
    - Comparison operators like ``==`` and ``>=`` return an expression suitable
      for use in ``where``/``require`` clauses.
    - Use ``&`` and ``|`` to combine conditions; do not use Python's ``and``/``or``.
    - For relationships/properties/chains (e.g. ``Person.age``), operators apply
      to the **output field** (the last field). To work with a **non-output** field,
      index the relationship/property/chain to get a :class:`FieldRef`, e.g.
      ``Person.jobs["start_date"]``.

    ``Variable`` objects cannot be used in Python boolean contexts (``if`` /
    ``while``), and they cannot be iterated or passed to ``len()``; these usages
    raise :class:`relationalai.util.error.RAIException` with a helpful message.

    Parameters
    ----------
    model : Model
        The semantic model this variable belongs to.

    Examples
    --------
    Build a filter with variable comparisons and select an aliased column:

    >>> from relationalai.semantics import Integer, Model, String
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> m.define(
    ...     Person.new(id=1, name="Alice", age=30),
    ...     Person.new(id=2, name="Bob", age=17),
    ... )
    >>> m.where(Person.age >= 18).select(Person.name.alias("NAME")).to_df()
    """
    #--------------------------------------------------
    # Infix operator overloads
    #--------------------------------------------------

    def _bin_op(self, op, left, right) -> Expression:
        res = CoreConcepts["Numeric"].ref()
        return Expression(CoreRelationships[op], [left, right, res])

    @include_in_docs
    def __add__(self, other):
        """Return an addition expression (``self + other``).

        This enables Python's ``+`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable addition expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("+", self, other)

    @include_in_docs
    def __radd__(self, other):
        """Return an addition expression (``other + self``).

        This is the reflected form of :meth:`.Variable.__add__`. It enables
        expressions like ``1 + Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable addition expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("+", other, self)

    @include_in_docs
    def __mul__(self, other):
        """Return a multiplication expression (``self * other``).

        This enables Python's ``*`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable multiplication expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("*", self, other)

    @include_in_docs
    def __rmul__(self, other):
        """Return a multiplication expression (``other * self``).

        This is the reflected form of :meth:`.Variable.__mul__`. It enables
        expressions like ``365 * Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable multiplication expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("*", other, self)

    @include_in_docs
    def __sub__(self, other):
        """Return a subtraction expression (``self - other``).

        This enables Python's ``-`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable subtraction expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("-", self, other)

    @include_in_docs
    def __rsub__(self, other):
        """Return a subtraction expression (``other - self``).

        This is the reflected form of :meth:`.Variable.__sub__`. It enables
        expressions like ``5 - Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable subtraction expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("-", other, self)

    @include_in_docs
    def __truediv__(self, other):
        """Return a division expression (``self / other``).

        This enables Python's ``/`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable division expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("/", self, other)

    @include_in_docs
    def __rtruediv__(self, other):
        """Return a division expression (``other / self``).

        This is the reflected form of :meth:`.Variable.__truediv__`. It enables
        expressions like ``1 / Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable division expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("/", other, self)

    @include_in_docs
    def __floordiv__(self, other):
        """Return a floor division expression (``self // other``).

        This enables Python's ``//`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable floor division expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("//", self, other)

    @include_in_docs
    def __rfloordiv__(self, other):
        """Return a floor division expression (``other // self``).

        This is the reflected form of :meth:`.Variable.__floordiv__`. It enables
        expressions like ``5 // Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable floor division expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("//", other, self)

    @include_in_docs
    def __pow__(self, other):
        """Return an exponentiation expression (``self ** other``).

        This enables Python's ``**`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable exponentiation expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("^", self, other)

    @include_in_docs
    def __rpow__(self, other):
        """Return an exponentiation expression (``other ** self``).

        This is the reflected form of :meth:`.Variable.__pow__`. It enables
        expressions like ``2 ** Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable exponentiation expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("^", other, self)

    @include_in_docs
    def __mod__(self, other):
        """Return a modulo expression (``self % other``).

        This enables Python's ``%`` operator for semantics DSL variables.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable modulo (remainder) expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("%", self, other)

    @include_in_docs
    def __rmod__(self, other):
        """Return a modulo expression (``other % self``).

        This is the reflected form of :meth:`.Variable.__mod__`. It enables
        expressions like ``10 % Person.age``.

        Parameters
        ----------
        other : Value
            The left-hand operand.

        Returns
        -------
        Expression
            A composable modulo (remainder) expression.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._bin_op("%", other, self)

    @include_in_docs
    def __neg__(self):
        """Return a negation expression (``-self``).

        This enables Python's unary ``-`` operator for semantics DSL variables.
        The negation is represented as multiplication by ``-1`` in the DSL.

        Returns
        -------
        Expression
            A composable negation expression.
        """
        return self._bin_op("*", self, -1)

    #--------------------------------------------------
    # Filter overloads
    #--------------------------------------------------

    def _filter(self, op, left, right) -> Expression:
        return Expression(CoreRelationships[op], [left, right])

    @include_in_docs
    def __gt__(self, other):
        """Return a greater-than filter expression (``self > other``).

        This enables Python's ``>`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter(">", self, other)

    @include_in_docs
    def __ge__(self, other):
        """Return a greater-than-or-equal filter expression (``self >= other``).

        This enables Python's ``>=`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter(">=", self, other)

    @include_in_docs
    def __lt__(self, other):
        """Return a less-than filter expression (``self < other``).

        This enables Python's ``<`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter("<", self, other)

    @include_in_docs
    def __le__(self, other):
        """Return a less-than-or-equal filter expression (``self <= other``).

        This enables Python's ``<=`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter("<=", self, other)

    @include_in_docs
    def __eq__(self, other) -> Any:
        """Return an equality filter expression (``self == other``).

        This enables Python's ``==`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter("=", self, other)

    @include_in_docs
    def __ne__(self, other) -> Any:
        """Return an inequality filter expression (``self != other``).

        This enables Python's ``!=`` operator for semantics DSL variables.
        The returned expression is typically passed to :meth:`Model.where` or
        :meth:`Fragment.where` to filter a query.

        Parameters
        ----------
        other : Value
            The right-hand operand.

        Returns
        -------
        Expression
            A composable boolean expression suitable for ``where``/``require``
            clauses.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is an invalid value in an expression (for example, a
            table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.
        """
        return self._filter("!=", self, other)

    #--------------------------------------------------
    # And/Or
    #--------------------------------------------------

    @include_in_docs
    def __or__(self, other) -> Match:
        """Return a match expression (``self | other``).

        This enables Python's ``|`` operator for semantics DSL values and
        statements.

        ``a | b`` builds a :class:`Match` with two branches. At execution time,
        the match picks the first branch (left-to-right) that can succeed for
        the current bindings.

        This makes ``|`` the right tool for *fallback/default* behavior ("use
        this value if available, otherwise use that one"). If you instead want
        to combine the results from multiple branches (set-style union), use
        :func:`~relationalai.semantics.union` or :meth:`Model.union`.

        Notes
        -----
        Do not use Python's boolean ``or`` with DSL variables. Use ``|``,
        :func:`~relationalai.semantics.union`, or :meth:`Model.union` instead.

        Parameters
        ----------
        other : Statement
            The right-hand branch.

        Returns
        -------
        Match
            A composable match expression. You can extend it with additional
            branches via ``|``.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - Branches return different numbers of values (for example, mixing a
              value-producing branch with a multi-column select).

        Examples
        --------
        Set a default value for a property that may be missing:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> User = m.Concept("User")
        >>> m.define(
        ...     User.new(username="alice", age=30, status="active"),
        ...     User.new(username="bob", age=15, status="inactive"),
        ...     User.new(username="carol", age=40),
        ... )
        >>> # Get status with a default of "missing"
        >>> status = User.status | "missing"
        >>> m.select(User.username, status).to_df()
        """
        return Match(self._model, self, other)

    @include_in_docs
    def __and__(self, other) -> Fragment:
        """Return a fragment that combines conditions with AND (``self & other``).

        Use ``&`` to explicitly conjoin conditions when building filters. This is
        equivalent to passing multiple conditions to :meth:`Model.where`.
        If ``other`` is a :class:`Fragment`, this returns a new fragment with
        ``self`` added to ``other``'s filters (roughly ``other.where(self)``).

        Notes
        -----
        Do not use Python's boolean ``and`` with DSL variables. Use ``&``
        (or pass multiple conditions to ``where``) instead.

        Parameters
        ----------
        other : Statement or Fragment
            The right-hand operand. This is typically a condition expression,
            but it may also be an existing fragment to extend.

        Returns
        -------
        Fragment
            A composable fragment representing the conjunction.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is not a valid statement in a filter.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Extend an existing fragment by adding another condition, then continue
        chaining:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> User = m.Concept("User")
        >>> m.define(
        ...     User.new(username="alice", age=30, status="active"),
        ...     User.new(username="bob", age=15, status="inactive"),
        ...     User.new(username="carol", age=40),
        ... )
        >>> active_users = m.where(User.status == "active")
        >>> active_adults = active_users & (User.age >= 18)
        >>> active_adults.select(User.username).to_df()
        """
        if isinstance(other, Fragment):
            new = other.where()
            new._where.insert(0, self)
            return new
        return self._model.where(self, other)

    #------------------------------------------------------
    # AsBool
    #------------------------------------------------------

    @include_in_docs
    def as_bool(self) -> AsBool:
        """Return a boolean-typed wrapper around this variable.

        ``as_bool()`` converts a DSL value/statement into a boolean expression
        that can be selected or composed like any other value. At execution
        time, the result is ``True`` when the wrapped expression can succeed
        for the current bindings, and ``False`` otherwise.

        This is useful for turning *presence* into a column (for example,
        whether an optional property/relationship exists) and for explicitly
        selecting conditions as data.

        Notes
        -----
        - You cannot use the result in Python boolean contexts (``if`` /
          ``while``); it is only meaningful inside semantics DSL expressions.
        - Comparisons and other filter-style expressions selected via
          :meth:`Model.select` are automatically treated as boolean outputs,
          so calling ``as_bool()`` is usually optional.

        Returns
        -------
        AsBool
            A composable boolean expression.

        Examples
        --------
        Select whether an optional property is present:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(
        ...     Person.new(name="Alice", nickname="Al", age=30),
        ...     Person.new(name="Bob", age=17),
        ... )
        >>> has_nickname = Person.nickname.as_bool()
        >>> m.select(Person.name, has_nickname.alias("has_nickname")).to_df()

        Select a condition as a column explicitly without using ``as_bool()``:

        >>> is_adult = Person.age >= 18
        >>> m.select(Person.name, is_adult.alias("is_adult")).to_df()
        """
        return AsBool(self)

    #------------------------------------------------------
    # Alias
    #------------------------------------------------------

    @include_in_docs
    def alias(self, alias: str) -> Alias:
        """Return this value labeled with an output alias.

        ``alias()`` wraps this value in an :class:`Alias`. The alias is primarily
        used to name the output column when you pass the value to :meth:`Model.select`
        (and therefore affects column labels in :meth:`Fragment.to_df`).

        When a selected value has an alias, the resulting fragment can also be
        indexed by that name (for example ``query[\"AGE\"]``).

        Parameters
        ----------
        alias : str
            The output name to attach to this value.

        Returns
        -------
        Alias
            A composable aliased value.

        Notes
        -----
        ``alias()`` does not rename the underlying concept/relationship; it only
        influences how the value is labeled in query outputs.

        Examples
        --------
        Alias selected columns for readable output and later indexing:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(
        ...     Person.new(name="Alice", age=30),
        ...     Person.new(name="Bob", age=17),
        ... )
        >>> m.select(Person.name.alias("person_name"), Person.age.alias("person_age")).to_df()
        """
        return Alias(self, alias)

    #--------------------------------------------------
    # in_
    #--------------------------------------------------

    @include_in_docs
    def in_(self, values:Sequence[Value]|Variable):
        """Return an SQL IN-style filter expression.

        Use ``in_`` to test whether this value equals *any* of a set of
        candidate values.

        This is a DSL-friendly alternative to Python's ``in`` operator (which
        cannot be overloaded for DSL variables). The method lowers to an
        equality against a one-column value source:

        - If ``values`` is a :class:`Variable`, this is equivalent to ``self == values``.
        - If ``values`` is a sequence of Python primitives (``str``, ``int``,
          ``float``, ``bool``), the method builds a one-column table via
          :meth:`Model.data` and compares against that column.
        - Otherwise, the method unions the provided items via :meth:`Model.union`
          and compares against the resulting value.

        Parameters
        ----------
        values : Sequence[Value] or Variable
            Candidate values to match.

        Returns
        -------
        Expression
            A composable boolean expression suitable for :meth:`Model.where`
            and :meth:`Fragment.where`.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - Any provided value is not valid in an expression (for example, a
              table schema).
            - The constructed union branches return different numbers of
              values.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Filter rows by membership in a Python list:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(
        ...     Person.new(name="Alice", age=10),
        ...     Person.new(name="Bob", age=30),
        ...     Person.new(name="David", age=50),
        ... )
        >>> m.where(Person.name.in_(["Alice", "Bob"])).select(Person.name, Person.age).to_df()

        Pass a variable (for example the result of :meth:`Model.union`):

        >>> allowed = m.union("Alice", "Bob")
        >>> m.where(Person.name.in_(allowed)).select(Person.name).to_df()
        """
        if isinstance(values, Variable):
            return self == values
        if all(isinstance(v, (str, int, float, bool)) for v in values):
            data_table = self._model.data([(v,) for v in values])
            return self == data_table[0]
        else:
            return self == self._model.union(*values)

    #--------------------------------------------------
    # Check value
    #--------------------------------------------------

    def _check_value(self) -> bool:
        return True

    def _to_concept(self) -> Concept|None:
        return None

    #------------------------------------------------------
    # Error handling
    #------------------------------------------------------

    def __bool__(self) -> NoReturn:
        cur_source = self._source.block.source or ""
        invalid = next((bool_check for bool_check in ["if ", "while ", " and ", " or "] if bool_check in cur_source), "bool check").strip()
        mapped = {"and": "`&` or `,`", "or": "`|`", "if": "`where`"}
        if m := mapped.get(invalid):
            exc("Invalid operator", f"Cannot use python's `{invalid}` in model expressions. Use {m} instead.", [source(self)])
        else:
            exc("Invalid operator", f"Cannot use python's `{invalid}` in model expressions.", [source(self)])

    if not TYPE_CHECKING:
        def __iter__(self):
            common_incorrect = ["sum", "min", "max"]
            for agg in common_incorrect:
                if agg in self._source.block.source:
                    exc("Invalid built-in", f"The Python built-in `{agg}()` was used instead of the RAI equivalent.", [
                        source(self),
                        f"Use [cyan]`relationalai.semantics.std.aggregates.{agg}`[/cyan] instead.",
                    ])
            exc("Invalid iteration", f"Cannot iterate over {self.__class__.__name__} objects.", [
                source(self)
            ])

        def __len__(self) -> NoReturn:
            exc("Invalid operator", "Cannot use len() on model variables.", [source(self)])

#------------------------------------------------------
# Concept
#------------------------------------------------------

@include_in_docs
class Concept(Variable):
    """Represents an entity type (concept) in a semantic model.

    Concepts are typically created via :meth:`Model.Concept` and act as typed
    placeholders for entities in definitions and queries.

    A concept also serves as the entry point for its relationships. Properties
    and relationships are commonly attached by attribute assignment
    (for example ``Person.name = m.Property(...)``). Accessing ``Person.name``
    later yields a chained value suitable for comparisons and selection.

    Notes
    -----
    Users typically create concepts via :meth:`Model.Concept` rather than
    instantiating this class directly.

    Parameters
    ----------
    name : str
        Name of the concept.
    extends : list[Concept]
        Base concepts this concept extends.
    identify_by : dict[str, Property | Concept]
        Mapping from identity field name to its type (a :class:`Property`
        or :class:`Concept`). When provided, the mapping is used to **create**
        identity properties on the concept using generic/default readings.
        For identity defined after creation, or to use custom readings, see
        :meth:`Concept.identify_by`.
    model : Model
        Model this concept belongs to.
    identity_includes_type : bool, optional
        Whether the concept's type participates in the entity key.

    Examples
    --------
    Create a concept in a model:

    >>> from relationalai.semantics import Model
    >>> m = Model()
    >>> Person = m.Concept("Person")

    Assign properties to the concept via attribute assignment:

    >>> from relationalai.semantics import String, Integer
    >>> Person.name = m.Property(f"{Person} is named {String:name}")
    >>> Person.age = m.Property(f"{Person} is {Integer:age} years old")

    Use :meth:`Concept.new` to create instances of the concept in a :meth:`Model.define` block:

    >>> m.define(
    ...     Person.new(name="Alice", age=10),
    ...     Person.new(name="Bob", age=20),
    ... )

    Use the concept object and its properties as placeholders in queries:

    >>> m.select(Person.name).where(Person.id == 1).to_df()

    Use the ``extends`` parameter to create a concept hierarchy:

    >>> # Create an Adult concept that extends Person
    >>> Adult = m.Concept("Adult", extends=[Person])

    Call concept (see :meth:`Concept.__call__`) to express membership in that concept:

    >>> # If a Person's age is at least 18, they are an Adult
    >>> m.where(Person.age >= 18).define(Adult(Person))
    """
    lookup_by_id: dict[int, Concept] = {}
    RESERVED_NAMES: frozenset[str] = frozenset({
        "alias", "annotate", "as_bool", "filter_by", "identify_by",
        "in_", "new", "ref", "require", "to_identity",
    })

    def __init__(self, name: str, extends: list[Concept], identify_by: dict[str, Property|Concept], model:Model, identity_includes_type:bool = False):
        super().__init__(model)
        Concept.lookup_by_id[self._id] = self
        self._name = name
        self._identity_includes_type = identity_includes_type
        clean_extends = []
        for extend in extends:
            if not isinstance(extend, Concept):
                exc("Invalid extend", f"Cannot extend from non-Concept type: {extend}", [source(self)])
            if "Number" in CoreConcepts and extend is CoreConcepts["Number"]:
                clean_extends.append(extend.size(38, 14))
            else:
                clean_extends.append(extend)
        self._extends = clean_extends
        self._annotations: list[Expression|Relationship] = []
        self._relationships = {}
        self._identify_by:list[Relationship] = []
        for id_name, id_val in identify_by.items():
            if not isinstance(id_val, (Concept, Property)):
                exc("Invalid identify_by", f"identify_by values must be Concepts, got: {type(id_val)}", [source(self)])
            id_val = self._dot(id_name, with_type=id_val, allow_implicit=True)
            self._identify_by.append(id_val)

    @include_in_docs
    def __setattr__(self, name: str, value: Any) -> None:
        """Register relationships declared via attribute assignment.

        Assigning a :class:`Relationship` (including :class:`Property`) to a
        concept stores the relationship in the concept's relationship registry.
        This is what enables the common declaration pattern ``Person.age =
        m.Property(...)``.

        Relationship names are tracked case-insensitively (the attribute name is
        lowercased). If the relationship does not already have a ``short_name``,
        it is set from the attribute name.

        Non-relationship assignments fall back to normal attribute setting.

        Parameters
        ----------
        name : str
            Attribute name being assigned.
        value : Any
            Value being assigned. If this is a :class:`Relationship` (including
            :class:`Property`), it is treated as a declared relationship/property.

        Raises
        ------
        relationalai.util.error.RAIException
            If a relationship with the same name (case-insensitive) is already
            defined on this concept.
        """
        if isinstance(value, Relationship):
            key = name.lower()
            if key in Concept.RESERVED_NAMES and not isinstance(self, Table):
                exc("Reserved relationship name",
                    f"'{name}' is a reserved name and cannot be used as a relationship name on concept '{self._name}'.",
                    [source(value)]
                )
            if self._relationships.get(key) is not None:
                exc("Duplicate relationship", f"Relationship '{name}' is already defined on concept '{self._name}'", [
                    source(self._relationships[key])
                ])
            if not value._short_name:
                value._short_name = name
            self._relationships[key] = value
            # Bump model version when relationship is added to a concept
            if self._model and self._model is not CoreLibrary:
                self._model._bump_version()
        else:
            super().__setattr__(name, value)

    def _dot_recur(self, name: str, with_type: Any = None, allow_implicit=False) -> Relationship|None:
        if name.lower() in self._relationships:
            return self._relationships[name.lower()]
        for ext in self._extends:
            rel = ext._dot_recur(name, with_type)
            if rel is not None:
                return rel
        if not name.startswith("__row_id_") and not allow_implicit and not self._model.config.model.implicit_properties:
            cur_loc = SourcePos.new()
            return exc("Implicit property", f"`{self._name}` does not have a `{name}` property declared.", [
                source(cur_loc),
                "Implicit properties are disabled in the model config. You'll need to explicitly declare this property.",
            ])
        # We should create the relationship on the root type if it's not
        # found anywhere
        if self._model is not CoreLibrary:
            if not isinstance(self, Table) and name.lower() in Concept.RESERVED_NAMES:
                exc("Reserved relationship name",
                    f"'{name}' is a reserved name and cannot be used as a relationship name on concept '{self._name}'.",
                    [source(SourcePos.new())]
                )
            field_type = _find_concept(self._model, with_type, default=CoreConcepts["Any"])
            assert field_type is not None
            rel = self._relationships[name.lower()] = self._model.Property(fields=[
                Field(self._name.lower(), self),
                Field(name, field_type)
            ], short_name=name)
            return rel
        return None

    def _dot(self, name: str, with_type: Any = None, allow_implicit=False) -> Relationship:
        if self._model is CoreLibrary:
            exc("Invalid relationship", f"Cannot access relationships on core concept '{self._name}'.", [source(self)])
        if with_type is None:
            with_type = CoreConcepts["Any"]
        rel = self._dot_recur(name, with_type, allow_implicit)
        assert rel is not None
        return rel

    @include_in_docs
    def __getattr__(self, item: str) -> Chain:
        """Return a chained relationship reference for attribute access.

        Concepts expose declared relationships as Python attributes. Accessing
        ``Person.name`` returns a :class:`Chain` that can be used in filters and
        selections (for example ``Person.name == \"Alice\"`` or ``Person.pets.name == \"Fluffy\"``).
        If the concept does not have a relationship with the given name, this method looks for it
        recursively in base concepts (via ``extends``).

        If the attribute name starts with an underscore, this method defers to
        normal Python attribute access so internal attributes (like ``_name``)
        remain accessible.

        Parameters
        ----------
        item : str
            Relationship/property name to access.

        Returns
        -------
        Chain
            A chained value representing the relationship reading.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - A relationship with this name is not declared and the
              ``model.implicit_properties`` config flag is disabled.
              (Table concepts allow for implicit properties, even when
              the config flag is disabled.)
            - Relationships are accessed on a core concept.

        Examples
        --------
        Declare a property and use attribute access in a query:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(Person.new(name="Bob"), age=20))
        >>> # Access the `name` properties via attribute access
        >>> m.select(Person.name).to_df()

        Access properties from a base concept:

        >>> Adult = m.Concept("Adult", extends=[Person])
        >>> m.where(Person.age >= 18).define(Adult(Person))
        >>> m.select(Adult.name).to_df()
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        return Chain(self, self._dot(item, allow_implicit=isinstance(self, Table)))

    @include_in_docs
    def __call__(self, *args: Any, **kwargs: Any) -> Expression:
        """Return an expression asserting membership in this concept.

        Calling a concept creates a composable :class:`Expression` that you can
        use in :meth:`Model.where` and :meth:`Model.define`. This is used to
        express membership in the concept, for example ``Adult(Person)`` asserts
        that ``Person`` is an ``Adult``.

        Notes
        -----
        This method does not create entities. To create new instances of a
        concept, use :meth:`Concept.new`.

        Parameters
        ----------
        *args : Any
            Positional arguments to include in the membership expression.
            Although multiple arguments can be provided, the most common usage is
            to pass a single concept object (for example, ``Adult(Person)``).
        **kwargs : Any
            Extra named values attached to the resulting expression. Currently,
            these keyword arguments are ignored.

        Returns
        -------
        Expression
            A composable expression asserting membership in this concept.

        Raises
        ------
        relationalai.util.error.RAIException
            If an argument is not valid in an expression (for example, passing
            a table schema).
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Define membership in a derived concept:

        >>> from relationalai.semantics import Model, Integer, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} is {Integer} years old")
        >>> Adult = m.Concept("Adult", extends=[Person])
        >>> m.define(Adult(Person)).where(Person.age >= 18)

        Assert membership in a :meth:`Model.where` clause:

        >>> m.select(Person.age).where(Adult(Person)).to_df()
        """
        return Expression(self, list(args), kwargs)

    @include_in_docs
    def new(self, *args: StatementAndSchema, **kwargs: Any) -> New:
        """Create a statement that constructs a new entity of this concept.

        ``Concept.new(...)`` returns a :class:`New` expression. It becomes part
        of the model when you pass it to :meth:`Model.define` (or the
        convenience function :func:`~relationalai.semantics.define`).

        Use keyword arguments to set properties on the new entity. Keyword names
        resolve like attribute access (case-insensitively), so ``Person.new(name="Alice")``
        corresponds to ``Person.name``. If a keyword refers to a property that has
        not been declared and the ``model.implicit_properties`` config flag is disabled,
        an exception is raised; otherwise the property is created implicitly.

        You can also pass a :class:`TableSchema` (typically from :meth:`Table.to_schema`)
        to create one entity per source row and fill in missing keyword fields
        from table columns.

        Parameters
        ----------
        *args : StatementAndSchema
            Optional statements and/or table schemas to attach to the entity
            constructor. If a :class:`TableSchema` is provided, its columns are
            used to fill in keyword fields that are not explicitly present in ``**kwargs``.
        **kwargs : Any
            Property/relationship values to set for the new entity. Values may
            be Python literals (for example ``"Alice"`` or ``1``) or DSL values
            (for example a variable or expression).

        Returns
        -------
        New
            A composable "new entity" statement that can be passed to
            :meth:`Model.define` and then used in other statements to refer to
            the created entity.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - The concept defines identity properties (via ``identify_by``) but
              one or more required identity values are missing when the
              resulting ``New`` is defined/compiled.
            - A keyword refers to a relationship/property that is not declared
              and the ``model.implicit_properties`` config flag is disabled.
            - The concept belongs to the core library (relationships cannot be
              accessed on core concepts).

        Notes
        -----
        Think of ``Concept.new`` as **find-or-create**. When you define the returned
        :class:`New`, the model uses the concept type plus its identity values to
        decide which entity you mean. If an entity with the same identity already
        exists, the statement refers to it; otherwise a new entity is created.

        If the concept declares identity properties (via the ``identify_by`` keyword
        argument at concept creation time, or later via :meth:`Concept.identify_by`),
        you must provide values for those identity properties in ``Concept.new``.
        If the concept has no declared identity, the model derives identity from
        the values you pass (and, when creating from a :class:`TableSchema`, from each
        source row).

        As a result:

        - Calling ``Person.new`` twice with the same identity values refers
          to the same entity.
        - Defining the same ``New`` expression more than once is redundant; it
          does not create a second, distinct entity.
        - To create multiple distinct entities that share the same non-identity
          properties (for example, two people both aged 30), define an explicit
          identity (e.g. an ``id`` property via :meth:`Concept.identify_by`)
          and pass different identity values.

        Examples
        --------
        Create entities with properties and query one value:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"id": Integer})
        >>> Person.name = m.Property(f"{Person} has name {String}")
        >>> m.define(Person.new(id=1, name="Alice"), Person.new(id=2, name="Bob"))
        >>> m.where(Person.id == 1).select(Person.name).to_df()

        Create one entity per row of an external table schema:

        >>> customers = m.Table("DB.SCHEMA.CUSTOMERS", schema={"id": Integer, "name": String})
        >>> Customer = m.Concept("Customer", extends=[Person])
        >>> m.define(Customer.new(customers.to_schema()))
        """
        return New(self, args, kwargs)

    @include_in_docs
    def to_identity(self, *args: Any, **kwargs: Any) -> New:
        """Return an identity-only entity reference for this concept.

        The returned :class:`New` is *identity-only*: it refers to an existing
        entity by its identity key. It is compiled as a lookup (it cannot create
        or update entities).

        Use ``to_identity(...)`` when you know the full identity of the entity
        you want. In contrast, :meth:`Concept.filter_by` builds a general property
        filter and may match multiple entities.

        Parameters
        ----------
        *args : Any
            Optional statements and/or table schemas, expanded like :meth:`Concept.new`.
        **kwargs : Any
            Identity values (and optional extra property filters).

        Returns
        -------
        New
            A composable entity reference.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - Required identity values are missing.
            - A keyword refers to a relationship/property that is not declared
              and the ``model.implicit_properties`` config flag is disabled.
            - The identity-only expression is passed directly to :meth:`Model.define`.

        Examples
        --------
        >>> from relationalai.semantics import Model, String, define, select
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"id": String})
        >>> Person.name = m.Property(f"{Person} is named {String}")
        >>> define(Person.new(id="P1", name="Alice"))
        >>> select(Person.name).where(Person.to_identity(id="P1")).to_df()
        """
        return New(self, args, kwargs, identity_only=True)

    @include_in_docs
    def identify_by(self, *properties: Property|Chain) -> Concept:
        """Mark one or more properties as identity for this concept.

        ``Concept.identify_by(...)`` declares which properties form the unique
        identity key for entities of this concept. Identity is used by
        :meth:`Concept.new` to decide which entity you mean (find-or-create by key).

        This is an in-place configuration method: it appends the provided
        properties to this concept's identity list and returns ``self`` for
        chaining.

        This method differs from the ``identify_by=...`` argument used when
        creating a concept (for example via
        :meth:`Model.Concept`): the
        constructor form **creates** identity properties automatically, while
        ``identify_by(...)`` marks **existing** properties (including custom
        readings created with :meth:`Relationship.alt`)
        as identity.

        Parameters
        ----------
        *properties : Property | Chain
            One or more properties that form the identity key.
            In typical usage you pass relationship attributes such as
            ``Person.ssn`` (a :class:`Chain`).
            You may also pass a :class:`Reading`
            produced by :meth:`Relationship.alt`,
            as long as the underlying relationship is a
            :class:`Property`.

            Each provided property must have this concept as its first field
            (i.e., it must be a property *of* this concept).

        Returns
        -------
        Concept
            This concept (returned to enable fluent chaining).

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - An argument is not a property (or a property reading).
            - A property's first field is not this concept.

        Examples
        --------
        Declare a property, mark it as identity, define a couple entities, and
        query by the identity value:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.ssn = m.Property(f"{Person} has {String:ssn}")
        >>> Person.identify_by(Person.ssn)
        >>> m.define(
        ...     Person.new(ssn="123-45-6789", name="Alice"),
        ...     Person.new(ssn="987-65-4321", name="Bob")
        ... )
        >>> m.select(Person.name).where(Person.ssn == "123-45-6789").to_df()
        """
        for prop in properties:
            if isinstance(prop, Chain):
                prop = prop._next
            if not isinstance(prop, Property) and not (isinstance(prop, Reading) and isinstance(prop._relationship, Property)):
                exc("Invalid identify_by", f"identify_by expects Properties, got: {type(prop).__name__}", [source(self)])
            if prop._fields[0].type is not self:
                exc("Invalid identify_by", f"Property {prop} have the first field be {self._name}", [source(self)])
            self._identify_by.append(prop)
        return self

    @include_in_docs
    def filter_by(self, **kwargs: Any) -> FilterBy:
        """Return an expression that refers to entities filtered by property values.

        ``Concept.filter_by(...)`` builds a composable
        :class:`FilterBy` expression that matches
        entities of this concept whose properties/relationships match the
        provided keyword arguments. Keyword names are resolved like attribute
        access (case-insensitively), so ``Person.filter_by(name="Alice")`` uses
        the ``Person.name`` property. Values may be Python literals (e.g., ``"Alice"``, ``1``)
        or DSL values (e.g., variables/expressions).

        This is a general property filter and may match multiple entities. If
        you want to refer to a single entity by identity, use :meth:`Concept.to_identity`.

        Parameters
        ----------
        **kwargs : Any
            Property/relationship values to match. Keys are treated
            case-insensitively.

        Returns
        -------
        FilterBy
            A composable expression that can be used in
            :meth:`Model.where` or chained
            from a fragment via
            :meth:`Fragment.where`.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - A keyword refers to a relationship/property that is not declared
              and the ``model.implicit_properties`` configuration flag is disabled.
            - Relationships are accessed on a core concept.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Filter entities by a property value:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(Person.new(id=1, name="Alice"), Person.new(id=2, name="Bob"))
        >>> m.where(Person.filter_by(name="Alice")).select(Person.id).to_df()
        """
        return FilterBy(self, kwargs)

    @include_in_docs
    def require(self, *items: Variable|Fragment) -> Fragment:
        """Return a fragment that adds scoped requirements for this concept.

        This is a convenience shorthand for starting a :meth:`Model.where`
        domain on this concept and then adding one or more requirements, such
        as ``model.where(Person).require(...)``. Compared to calling
        :meth:`Model.require` (or :func:`~relationalai.semantics.require`)
        directly, this form scopes the requirement to entities matched by the
        concept domain, making it a common way to express per-entity constraints
        (for example, “every Person must have an age”).

        Parameters
        ----------
        *items : Variable | Fragment
            One or more items to require within the concept domain.
            Common examples include relationship/property chains such as
            ``Person.age`` (require that an age fact exists) and comparisons
            such as ``Person.age > 0``.

        Returns
        -------
        Fragment
            A composable fragment representing the requirement(s), which can be
            further chained with methods like :meth:`Fragment.where`,
            :meth:`Fragment.select`, and :meth:`Fragment.require`.

        Examples
        --------
        Require that every person has a positive age:

        >>> from relationalai.semantics import Model, define
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=20))
        >>> Person.require(Person.age > 0)
        """
        return self._model.where(self).require(*items)

    @include_in_docs
    def ref(self, name="") -> Ref:
        """Return a named reference variable for entities of this concept.

        ``Concept.ref(...)`` creates a fresh :class:`Ref`
        that you can use as a placeholder for an (unknown) entity of this
        concept when writing queries and definitions. This is most useful when you
        need multiple distinct variables of the same concept (for example
        ``u`` and ``v`` nodes in a graph).

        The optional ``name`` is used as the reference's display/name in
        generated readings and debugging output. When omitted, a default name
        is chosen based on the concept.

        Parameters
        ----------
        name : str, optional
            Human-friendly name for the reference variable.

        Returns
        -------
        Ref
            A reference variable that behaves like a :class:`Variable`:
            you can access relationships as attributes (e.g. ``u.id``) and pass
            the reference as an argument to relationships and derived concepts.

        Examples
        --------
        Create two independent reference variables and query pairs:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(
        ...     Person.new(id=1, name="Alice", age=30),
        ...     Person.new(id=2, name="Bob", age=30),
        ...     Person.new(id=3, name="Carol", age=25),
        ... )
        >>> p, q = Person.ref("person1"), Person.ref("person2")
        >>> m.select(p.name, q.name).where(p.age == q.age, p.id < q.id).to_df()
        """
        return Ref(self, name)

    @include_in_docs
    def annotate(self, *annos:Expression|Relationship) -> Concept:
        """Attach one or more annotations to this concept.

        An annotation is recorded on the concept and emitted into the compiled
        model. Annotations are primarily used by backends and internal tooling
        for debugging and support workflows (for example, to attach metadata
        such as tracking labels). In general you should not need to use
        annotations unless RelationalAI support instructs you to add them for
        diagnostic purposes.

        Parameters
        ----------
        *annos : Expression | Relationship
            One or more annotation nodes to attach.

        Returns
        -------
        Concept
            This concept (returned to enable fluent chaining).

        Raises
        ------
        relationalai.util.error.RAIException
            Raised when an annotation argument has an unsupported type.
        """
        self._annotations.extend(annos)
        return self

    def _to_concept(self) -> Concept:
        return self

    def __dir__(self):
        default = set(super().__dir__())
        return sorted(default.union(self._relationships.keys()))

    @include_in_docs
    def __format__(self, format_spec: str) -> str:
        """Return the placeholder used when formatting this concept in readings.

        Concepts are commonly interpolated into f-strings when you declare a
        relationship or property reading (for example
        ``m.Relationship(f"{Person} has pet {String:pet_name}")``).
        In that context, :class:`Reading` parses the formatted output and resolves
        it back to the correct concept.

        When ``format_spec`` is provided, it is treated as the *field name* to
        use for this concept occurrence in the reading (whitespace is stripped).
        When omitted, a default field name is derived from the concept name.

        Parameters
        ----------
        format_spec : str
            Optional field name to use in the reading.

        Returns
        -------
        str
            A placeholder string of the form ``"{Name#id}"`` or
            ``"{Name#id:field_name}"``.

        Notes
        -----
        This method is primarily for building relationship/property readings.
        It is not intended as a general-purpose display formatting hook.

        Examples
        --------
        Declare a relationship reading and name a field via a format spec:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.pets = m.Relationship(f"{Person} has pet {String:pet_name}")
        """
        if not format_spec:
            return f"{{{self._name}#{self._id}}}"
        return f"{{{self._name}#{self._id}:{format_spec.strip()}}}"

#------------------------------------------------------
# NumberConcept
#------------------------------------------------------

@include_in_docs
class NumberConcept(Concept):
    """Represents a concrete decimal ``Number(precision, scale)`` concept.

    A :class:`NumberConcept` is the concrete type produced by
    :meth:`~relationalai.semantics.frontend.core.AnyNumber.size` or
    :meth:`~relationalai.semantics.frontend.core.AnyNumber.parse` via the
    :data:`~relationalai.semantics.frontend.core.Number` type.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`~relationalai.semantics.frontend.core.AnyNumber.size` or
    :meth:`~relationalai.semantics.frontend.core.AnyNumber.parse` via
    :data:`~relationalai.semantics.frontend.core.Number`.

    Parameters
    ----------
    name : str
        Type name, typically ``"Number(p,s)"``.
    precision : int
        Total number of digits.
    scale : int
        Number of digits after the decimal point.
    model : Model
        Model or library this concept belongs to.
    """
    def __init__(self, name: str, precision: int, scale: int, model: Model):
        super().__init__(name, extends=[CoreConcepts["Numeric"]], identify_by={}, model=model)
        self._precision = precision
        self._scale = scale

    def size(self, precision: int, scale: int) -> "NumberConcept":
        """Return a new NumberConcept with the given precision and scale."""
        new_number = CoreConcepts["Number"].size(precision, scale)
        assert isinstance(new_number, NumberConcept)
        return new_number

#------------------------------------------------------
# Ref
#------------------------------------------------------

@include_in_docs
class Ref(Variable):
    """Represents a named reference variable for entities of a concept.

    A :class:`Ref` is a DSL variable that stands in for an (unknown) entity of
    some :class:`Concept`. You typically create refs via :meth:`Concept.ref`
    when you need multiple independent variables of the same concept (for
    example two endpoints of an edge).

    A ref behaves like other variables in the semantics DSL:

    - Attribute access (e.g. ``u.id`` or ``u.name``) resolves relationships on
      the underlying concept and returns a :class:`Chain`.
    - :meth:`Ref.filter_by` builds a composable filter expression rooted at this reference.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Concept.ref`.

    Parameters
    ----------
    concept : Concept
            The concept this reference ranges over.
    name : str, optional
            Display name for the reference, used in generated readings and
            debugging output. When omitted, a default name is derived from the
            concept (numeric concepts default to ``"number"``).

    Examples
    --------
    Create two refs and use attribute access in a join:

    >>> from relationalai.semantics import Integer, Model, String
    >>> m = Model()
    >>> Person = m.Concept("Person", identify_by={"id": Integer})
    >>> Person.name = m.Property(f"{Person} has name {String}")
    >>> m.define(Person.new(id=1, name="Alice"), Person.new(id=2, name="Bob"))
    >>> p, q = Person.ref("p"), Person.ref("q")
    >>> m.select(p.name, q.name).where(p.id < q.id).to_df()
    """
    def __init__(self, concept: Concept, name: str|None = None):
        super().__init__(concept._model)
        self._concept = concept
        self._name = name or ("number" if (isinstance(concept, NumberConcept) or concept is CoreConcepts["Numeric"])
                              else concept._name)

    @include_in_docs
    def __getattr__(self, item: str) -> Chain:
        """Return a chained relationship reference for attribute access.

        A :class:`Ref` behaves like a variable of its underlying :class:`Concept`.
        Accessing ``p.name`` (where ``p = Person.ref()``) returns a :class:`Chain`
        that can be used in filters and selections.

        If the attribute name starts with an underscore, this method defers to
        normal Python attribute access so internal attributes remain accessible.

        Parameters
        ----------
        item : str
            Relationship/property name to access.

        Returns
        -------
        Chain
            A chained value representing the relationship reading.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - A relationship with this name is not declared and the
              ``.model.implicit_properties`` config flag is disabled.
            - Relationships are accessed on a core concept.

        Examples
        --------
        Access a property through a reference variable:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(Person.new(name="Alice"))
        >>> p = Person.ref("p")
        >>> m.select(p.name).to_df()
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        return Chain(self, self._concept._dot(item))

    def _to_concept(self) -> Concept:
        return self._concept

    @include_in_docs
    def filter_by(self, **kwargs: Any) -> FilterBy:
        """Return an expression that constrains this reference by property values.

        ``Ref.filter_by(...)`` is like :meth:`Concept.filter_by`, but it
        is rooted at this specific reference variable. This is useful when you
        want to keep using the same ref elsewhere in a query (for example in a
        join) while also filtering it.

        Keyword names are resolved like attribute access (case-insensitively),
        and values may be Python literals (e.g., ``"Alice"``) or DSL values.

        Parameters
        ----------
        **kwargs : Any
            Property/relationship values to match. Keys are treated
            case-insensitively.

        Returns
        -------
        FilterBy
            A composable expression that can be used in
            :meth:`Model.where`.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - A keyword refers to a relationship/property that is not declared
              and the ``model.implicit_properties`` config flag is disabled.
            - Relationships are accessed on a core concept.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Filter a named reference and keep using it in the query:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(Person.new(id=1, name="Alice"), Person.new(id=2, name="Bob"))
        >>> p = Person.ref("p")
        >>> m.where(p.filter_by(name="Alice")).select(p.id).to_df()
        """
        return FilterBy(self, kwargs)

    @include_in_docs
    def __format__(self, format_spec: str) -> str:
        """Return the placeholder used when formatting this reference in readings.

        This forwards to :meth:`Concept.__format__` on the underlying concept so
        refs can be interpolated into relationship and property reading strings.

        Parameters
        ----------
        format_spec : str
            Optional field name to use for this occurrence (whitespace is
            stripped).

        Returns
        -------
        str
            A placeholder string for this concept occurrence.
        """
        return self._concept.__format__(format_spec)

#------------------------------------------------------
# Table
#------------------------------------------------------

@include_in_docs
class Table(Concept):
    """Represents an external table reference in a semantic model.

    A :class:`Table` is a lightweight handle for an external table name (for
    example, a SQL table or view) that can be used in two common ways:

    - As a **source** of rows when defining entities via :meth:`Table.to_schema`.
    - As a **target** for query exports via :meth:`Fragment.into`.

    Tables behave like concepts with column relationships: you can access a
    column by index (``table[0]``) or by name (``table["col"]``), and iterate
    over columns.

    Notes
    -----
    Users typically create table references via :meth:`Model.Table` rather than
    instantiating this class directly.

    Parameters
    ----------
    name : str
        Table name or path used by the backend.
    schema : dict[str, Concept]
        Optional mapping of column names to their types. When provided, column
        relationships are created immediately. When omitted, column metadata is
        resolved lazily the first time columns are accessed.
    model : Model
        The model this table reference belongs to.

    Raises
    ------
    IndexError
        If a column is accessed by an out-of-range integer index.
    KeyError
        If a column is accessed by a name that is not present.
    relationalai.util.error.RAIException
        If :meth:`Table.new` or
        :meth:`Table.to_identity` is
        called (tables cannot be used to create new entities directly).

    Examples
    --------
    Use a table as an input when defining entities:

    >>> from relationalai.semantics import Model, Integer, String
    >>> m = Model()
    >>> source = m.Table("DB.SCHEMA.CUSTOMERS", schema={"id": Integer, "name": String})
    >>> Customer = m.Concept("Customer", identify_by={"id": Integer})
    >>> m.define(Customer.new(source.to_schema()))

    Create a table reference as an export target:

    >>> from relationalai.semantics import Integer, Model
    >>> m = Model()
    >>> Person = m.Concept("Person", identify_by={"id": Integer})
    >>> m.define(Person.new(id=1, name="Alice"), Person.new(id=2, name="Bob"))
    >>> out = m.Table("DB.SCHEMA.PERSONS_EXPORT")
    >>> m.select(Person.id, Person.name).into(out).exec()
    """
    def __getattribute__(self, item: str):
        if not item.startswith("_") and item.lower() in Concept.RESERVED_NAMES:
            name = object.__getattribute__(self, "_name")
            exc("Reserved column name",
                f"'{item}' is a reserved name. Access the column via indexing instead, e.g. {name}[\"{item}\"].",
                [source(SourcePos.new())]
            )
        return super().__getattribute__(item)

    def __init__(self, name:str, schema: dict[str, Concept], model: Model):
        super().__init__(name, [], {}, model)
        self._skip_cdc = False
        self._known_columns = []
        for col_name, col_type in schema.items():
            if isinstance(col_type, Property):
                col_rel = col_type
                setattr(self, col_name, col_rel)
            else:
                col_rel = self._dot(col_name, with_type=col_type, allow_implicit=True)
            self._known_columns.append(col_rel)

    @property
    def _columns(self) -> list[Relationship]:
        if self._known_columns:
            if self._model.config.debug.show_debug_logs:
                info("Table Schema Cache - Bypass", "Using provided schema information for table.", [source(self)])
            return self._known_columns
        try:
            cached, schema = schema_util.fetch(self._name, model=self._model)
            if self._model.config.debug.show_debug_logs:
                if cached:
                    info("Table Schema Cache - Hit", "Using cached schema information for table.", [source(self)])
                else:
                    info("Table Schema Cache - Miss", "Caching schema information for table.", [source(self)])
        except Exception:
            exc("Unknown Table",
                f"Failed to fetch schema for table '{self._name}'.",
                [source(self)]
            )
        for col_name, col_type_name in schema.items():
            col_type = _find_concept(self._model, col_type_name, default=CoreConcepts["Any"])
            col_rel = self._dot(col_name, with_type=col_type, allow_implicit=True)
            col_rel._short_name = col_name
            if col_type is not None:
                col_rel._fields[1].type = col_type
            self._known_columns.append(col_rel)
        return self._known_columns

    @include_in_docs
    def __getitem__(self, index: int|str) -> Chain:
        """Return a chained reference to a table column.

        Indexing a :class:`Table` returns a :class:`Chain` for the selected
        column relationship. You can select a column either by 0-based integer
        position (``table[0]``) or by column name (``table["col"]``).

        If the table was created without an explicit ``schema=``, the first
        column access triggers lazy schema discovery.

        Parameters
        ----------
        index : int | str
            Column selector: a 0-based index or a column name.

        Returns
        -------
        Chain
            A chained value representing the column.

        Raises
        ------
        IndexError
            If ``index`` is an integer index that is out of range.
        KeyError
            If ``index`` is a name that is not present.

        Examples
        --------
        Select columns by name and by index:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> t = m.Table("DB.SCHEMA.CUSTOMERS", schema={"id": Integer, "name": String})
        >>> m.select(t["name"], t[0]).to_df()
        """
        if isinstance(index, int):
            if not (0 <= index < len(self._columns)):
                raise IndexError(f"Column index {index} out of range, there are {len(self._columns)} columns")
            return Chain(self, self._columns[index])
        col = next((c for c in self._columns if c._short_name.lower() == index.lower()), None)
        if col is None:
            raise KeyError(f"Column name '{index}' not found, columns have names {[c._short_name for c in self._columns]}")
        return Chain(self, col)

    @include_in_docs
    def __iter__(self) -> Iterator[Relationship]:
        """Return an iterator over this table's column relationships.

        Iterating over a :class:`Table` yields each column as a :class:`Relationship`.
        This is useful when you want to select all columns (for example ``m.select(*t)``)
        or inspect available columns.

        If the table was created without an explicit ``schema=``, the first
        iteration triggers lazy schema discovery.

        Returns
        -------
        Iterator[Relationship]
            Iterator over the table's column relationships.

        Examples
        --------
        Select all columns from a table:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> t = m.Table("DB.SCHEMA.CUSTOMERS")
        >>> m.select(*t).to_df()
        """
        return iter(self._columns)

    @include_in_docs
    def new(self, *args:StatementAndSchema, **kwargs: Any) -> NoReturn:
        """Raise an error because tables cannot construct new entities.

        A :class:`Table` is an external table reference used as a row source (via
        :meth:`Table.to_schema`) or as a query export target (via :meth:`Fragment.into`).
        It is not an entity type, so ``table.new`` is always invalid.

        Parameters
        ----------
        *args : StatementAndSchema
            Unused. Present for API compatibility with
            :meth:`Concept.new`.
        **kwargs : Any
            Unused. Present for API compatibility.

        Raises
        ------
        relationalai.util.error.RAIException
            Always raised.
        """
        exc("Invalid new call", "Cannot create new instances of Tables.", [source(self)])

    @include_in_docs
    def to_identity(self, *args: Any, **kwargs: Any) -> NoReturn:
        """Raise an error because tables cannot be referenced by identity.

        A :class:`Table` is an external table handle, not an entity concept. If
        you need an identity-only entity reference, use :meth:`Concept.to_identity` on a
        :class:`Concept`.

        Parameters
        ----------
        *args : Any
            Unused. Present for API compatibility.
        **kwargs : Any
            Unused. Present for API compatibility.

        Raises
        ------
        relationalai.util.error.RAIException
            Always raised.
        """
        exc("Invalid identity call", "Cannot create identity instances of Tables.", [source(self)])

    @include_in_docs
    def to_schema(self, *, exclude: list[str] = []) -> TableSchema:
        """Return a schema object for using this table as an entity row source.

        The returned :class:`TableSchema` is typically passed to :meth:`Concept.new`,
        e.g. ``Customer.new(source.to_schema())``. Column names in ``exclude``
        are matched case-insensitively.

        Parameters
        ----------
        exclude : list[str], optional
            Column names to omit from the schema.

        Returns
        -------
        TableSchema
            A schema projection of this table's columns.

        Examples
        --------
        Define one entity per source row:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> source = m.Table("DB.SCHEMA.CUSTOMERS")
        >>> Customer = m.Concept("Customer")
        >>> m.define(Customer.new(source.to_schema()))

        Exclude a foreign key column and map it to a concept reference:

        >>> Order = m.Concept("Order")
        >>> Order.customer = m.Property(f"{Order} placed by {Customer}")
        >>> orders_table = m.Table("DB.SCHEMA.ORDERS")
        >>> m.define(
        ...     Order.new(
        ...         orders_table.to_schema(exclude=["customer_id"]),
        ...         customer=Customer.new(id=orders_table.customer_id),
        ...     )
        ... )
        """
        return TableSchema(self, exclude=exclude)

#------------------------------------------------------
# TableSchema
#------------------------------------------------------

@include_in_docs
class TableSchema(DSLBase):
    """Represents a schema projection derived from a :class:`Table`.

    A :class:`TableSchema` is typically created via :meth:`Table.to_schema` and
    then passed to :meth:`Concept.new` (to create one entity per source row) or to
    :meth:`Model.select` (to select all columns). Column names in ``exclude`` are
    matched case-insensitively.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Table.to_schema`.

    Parameters
    ----------
    table : Table
        Source table whose columns make up the schema.
    exclude : list[str], optional
        Column names to omit from the schema.

    Attributes
    ----------
    exclude
        Set of excluded column names (stored in lowercase).
    """
    def __init__(self, table: Table, exclude: list[str] = []):
        super().__init__(table._model)
        self._table = table
        self.exclude: set[str] = set([e.lower() for e in exclude])

    @include_in_docs
    def get_columns(self) -> list[Relationship]:
        """Return the table columns included in this schema.

        This filters the underlying :class:`Table` columns using the ``exclude``
        list passed to :meth:`Table.to_schema` (matched case-insensitively).

        Returns
        -------
        list[Relationship]
            Column relationships in the same order as the source table.
        """
        return [col for col in self._table._columns if col._short_name.lower() not in self.exclude]

#--------------------------------------------------
# DerivedTable
#--------------------------------------------------

@include_in_docs
class DerivedTable(Variable):
    """Base class for table-like query results with derived output columns.

    A :class:`DerivedTable` is the common base for query-like results such as
    :class:`Fragment` and branching constructs like :class:`Match` and :class:`Union`.

    You can iterate over a derived table or index into it to access
    :class:`DerivedColumn` objects. Integer indexing selects by 0-based position.
    Name-based indexing works when the corresponding selected value was aliased
    (for example via :class:`Alias`).

    Notes
    -----
    Most users should not instantiate this class directly.
    """
    def __init__(self, model: Model):
        super().__init__(model)
        self.__cols = []

    @property
    def _columns(self):
        if not self.__cols:
            self.__cols = self._get_cols()
        if not self.__cols:
            raise ValueError(f"Cannot use {self.__class__.__name__} as it has no columns")
        return self.__cols

    def _check_value(self):
        return bool(self._columns)

    @include_in_docs
    def __iter__(self) -> Iterator[DerivedColumn]:
        """Return an iterator over this table's derived output columns.

        Iterating over a :class:`DerivedTable` yields each column as a :class:`DerivedColumn`.
        This is useful when you want to select all columns from a derived result.

        Returns
        -------
        Iterator[DerivedColumn]
            Iterator over the derived output columns.
        """
        return iter(self._columns)

    @include_in_docs
    def __getitem__(self, index: int|str) -> DerivedColumn:
        """Return a derived output column by position or by alias.

        Integer indexing selects the 0-based output column position
        (for example ``fragment[0]``). Name-based indexing (for example
        ``fragment["total"]``) works when the corresponding selected value was
        aliased.

        Parameters
        ----------
        index : int | str
            Column selector: a 0-based index or an output column name.

        Returns
        -------
        DerivedColumn
            The selected derived output column.

        Raises
        ------
        IndexError
            If ``index`` is an integer index that is out of range.
        KeyError
            If ``index`` is a name that is not present.
        """
        if isinstance(index, int):
            if not (0 <= index < len(self._columns)):
                raise IndexError(f"Column index {index} out of range, there are {len(self._columns)} columns")
            return self._columns[index]
        col = next((c for c in self._columns if c._name == index), None)
        if not col:
            raise KeyError(f"Column name '{index}' not found, columns have names {[c._name for c in self._columns]}")
        return col

    def _get_cols(self):
        raise NotImplementedError()

#--------------------------------------------------
# DerivedColumn
#--------------------------------------------------

@include_in_docs
class DerivedColumn(Variable):
    """Represents one output column of a :class:`DerivedTable`.

    A derived column is produced by query-like results such as
    :class:`Fragment` and is typically accessed via iteration (``for c in fragment``)
    or indexing (``fragment[0]`` or ``fragment["alias"]``).

    Notes
    -----
    Most users should not instantiate this class directly.
    """
    def __init__(self, table: DerivedTable, index: int, name: str|None = None, type_: Concept|None = None):
        super().__init__(table._model)
        self._table = table
        self._index = index
        self._name = name
        self._type = type_
        self._relationships = {}

    def _dot(self, name: str) -> Relationship:
        if self._type is not None:
            return self._type._dot(name)
        if name.lower() not in self._relationships:
            self._relationships[name.lower()] = Property(fields=[
                Field("entity", CoreConcepts["Any"]),
                Field(name, CoreConcepts["Any"])
            ], short_name=name, model=self._model, is_unresolved=True)
        return self._relationships[name.lower()]

    @include_in_docs
    def __getattr__(self, item):
        """Return a chained relationship reference for attribute access.

        Accessing an attribute on a derived column (for example ``fragment[0].name``)
        returns a :class:`Chain` rooted at this column. If the column has a known
        concept type, relationship lookup is resolved on that type; otherwise an
        unresolved property on ``Any`` is used.

        If the attribute name starts with an underscore, this method defers to
        normal Python attribute access so internal attributes remain accessible.

        Parameters
        ----------
        item : str
            Relationship/property name to access.

        Returns
        -------
        Chain
            A chained value representing the relationship reading.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised if relationship lookup is resolved on a typed concept and
            that concept does not declare the relationship while implicit
            properties are disabled.
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        return Chain(self, self._dot(item))

    @staticmethod
    def from_value(table: DerivedTable, index: int, var: Value) -> DerivedColumn:
        name = None
        if isinstance(var, Alias):
            name = var._alias
        if not isinstance(var, Variable):
            var = Literal(var, table._model)
        return DerivedColumn(table=table, index=index, name=name, type_=var._to_concept())

#------------------------------------------------------
# Field
#------------------------------------------------------

@include_in_docs
class Field:
    """Describe one named field in a relationship or property.

    A :class:`Field` pairs a field name with a :class:`Concept` type and marks
    whether the field is an input and/or list-valued.

    Fields are usually created implicitly by parsing a reading string. You
    typically construct ``Field`` objects only when creating a relationship or
    property with the explicit ``fields=`` keyword argument via
    :meth:`Model.Relationship` or :meth:`Model.Property`.

    Parameters
    ----------
    name : str
        Field name used for indexing and display.
    type_ : Concept
        Concept type of the field.
    is_input : bool, optional
        Whether this field is an input field. In most relationships/properties,
        all but the last field are inputs and the final field is the output.
    is_list : bool, optional
        Whether the field represents a list of values.
    source : SourcePos | None, optional
        Optional source position used for error reporting.

    Attributes
    ----------
    name
        Field name.
    type
        Field concept type.
    is_input
        True for input fields.
    is_list
        True for list-valued fields.

    Examples
    --------
    Most fields are created implicitly by parsing a reading string:

    >>> from relationalai.semantics import Integer, Model, String
    >>> m = Model()
    >>> Person = m.Concept("Person", identify_by={"id": Integer})
    >>> Person.name = m.Property(f"{Person} has {String:name}")
    >>> # The Property constructor parses the reading string and produces an
    >>> # input field of type Person and an output field named "name" of type String.

    You can also declare fields explicitly via ``fields=``. For example,
    the following is equivalent to the previous example:

    >>> from relationalai.semantics.frontend.base import Field
    >>> Person.name = m.Property(
    ...     fields=[Field("person", Person), Field("name", String)],
    ...     short_name="name"
    ... )
    """
    def __init__(self, name: str, type_: Concept, is_input: bool = False, is_list: bool = False, source: SourcePos | None = None):
        self._id = next(_global_id)
        self.name: str = name
        self.type: Concept = type_
        self.is_input: bool = is_input
        self.is_list: bool = is_list
        self._source: SourcePos | None = source

    @classmethod
    def input(cls, name: str, concept: Concept, is_list: bool = False) -> Field:
        return Field(name, concept, is_input=True, is_list=is_list)

    def _match(self, other: Field) -> bool:
        return self.name == other.name and self.type is other.type

#--------------------------------------------------
# Relationship
#--------------------------------------------------

@include_in_docs
class Relationship(Variable):
    """Represents a relationship between entities in a semantic model.

    A :class:`Relationship` describes how one or more :class:`Concept`s are related.
    You usually create relationships via :meth:`Model.Relationship` or
    :meth:`Model.Property` constructors.

    Relationships are creates from a **reading string** that describes the relationship
    in a human-readable format. These are typically f-strings that interpolate concept
    placeholders into one or more **fields**, which may be given optional names for indexing
    via the fields format specification. Field types can be any concept object or one of the
    core types in :mod:`relationalai.semantics.frontend.core`, such as
    :data:`~relationalai.semantics.frontend.core.String` and :data:`~relationalai.semantics.frontend.core.Integer`.

    A relationship is a kind of mapping that associates a sequence of input values to an output value.
    The last field is the **output field**, and all preceding fields are **input fields**.
    You can select a relationship's fields in a :meth:`Model.select` fragment or use them to build
    filter expressions in :meth:`Model.where`.

    Relationships are **multi-valued**: for the same input values, there
    may be zero, one, or many matching output values (for example, a person can
    have many pets). For this reason, relationships are often given plural names,
    like "employers" or "pets". If you want an attribute that is **single-valued**
    (at most one output value for each input), use :class:`Property`.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer using the
    :meth:`Model.Relationship` factory method instead.

    Parameters
    ----------
    model : Model
        The model this relationship belongs to.
    reading_str : str, optional
        A reading string describing the relationship fields.
        When omitted and ``fields`` is provided, a default reading is generated.
    fields : list[Field], optional
        Explicit field definitions for this relationship.
    short_name : str, optional
        Optional identifier for indexing (for example in ``model.relationship_index``).
    allow_no_fields : bool, optional
        If True, allow constructing a relationship with zero fields.
    overloads : list[list[Concept]], optional
        Optional overload signatures for the relationship.
    is_unresolved : bool, optional
        If True, marks this relationship as unresolved (used for dynamic
        property access on types like ``Any``).

    Raises
    ------
    ValueError
        If neither ``reading_str`` nor ``fields`` is provided and
        ``allow_no_fields`` is False.

    Examples
    --------
    Declare a relationship with an explicit reading string:

    >>> from relationalai.semantics import Model, String
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> Company = m.Concept("Company")
    >>> Person.employers = m.Relationship(f"{Person} works at {Company:employer}")

    Call a relationship to create an expression asserting a fact, which can be
    used in :meth:`Model.define` to define facts in the model:

    >>> m.define(
    ...     alice := Person.new(name="Alice"),
    ...     acme := Company.new(name="Acme Inc."),
    ...     # Add the fact that Alice works at Acme to the model
    ...     alice.employers(acme)
    ... )

    Call a relationship to create a condition in :meth:`Model.where` that filters query results
    based on existing facts:

    >>> # Get names of people where acme is one of their employers.
    >>> m.select(Person.name).where(Person.employers(acme))

    Declare a relationship with explicit fields instead of a reading string:

    >>> from relationalai.semantics.frontend.base import Field
    >>> Person.employers = m.Relationship(
    ...     fields=[Field("employee", Person), Field("employer", Company)],
    ... )
    """
    def __init__(self, model: Model, reading_str:str = "", fields: list[Field] = [], short_name: str = "", allow_no_fields: bool = False, overloads:list[list[Concept]]|None = None, is_unresolved: bool = False):
        super().__init__(model)
        if not reading_str and not fields and not allow_no_fields:
            raise ValueError("Either reading_str or fields must be provided")
        if not reading_str and fields:
            reading_str = " and ".join([f"{{{f.type._name}#{f.type._id}:{f.name}}}" for f in fields[:-1]])
            reading_str = f"{reading_str} has {{{fields[-1].type._name}#{fields[-1].type._id}:{fields[-1].name}}}"
        parts = []
        if not fields:
            (fields, parts) = Reading.parse(model, reading_str, fields, _source=self)
        self._fields = fields
        if not fields and not allow_no_fields:
            exc("Invalid Relationship", "A Relationship must have at least one field.", [source(self)])
        self._readings = [Reading(model, self, reading_str, fields, parts)]
        self._short_name = short_name
        self._relationships = {}
        self._annotations: list[Expression|Relationship] = []
        self._overloads = overloads
        self._is_unresolved = is_unresolved
        for f in self._fields:
            if f._source is None:
                f._source = self._source

    def _dot(self, name: str) -> Relationship:
        field_type = self._fields[-1].type
        if name.lower() in self._relationships:
            return self._relationships[name.lower()]
        if field_type is CoreConcepts["Any"] or field_type is CoreConcepts["AnyEntity"]:
            rel = Property(fields=[
                Field(self._fields[-1].name, field_type),
                Field(name, CoreConcepts["Any"])
            ], short_name=name, model=self._model, is_unresolved=True)
            self._relationships[name.lower()] = rel
            return rel
        return field_type._dot(name)

    @include_in_docs
    def __getattr__(self, item):
        """Return a chain that references a relationship on this relationship's output
        field.

        Relationships can be chained through their output field using normal
        Python attribute access. Accessing ``item`` returns a :class:`Chain`
        that represents following this relationship and then reading the
        relationship/property ``item`` on the output field's concept.

        If the ``item`` starts with an underscore, this method defers to
        normal Python attribute access so internal attributes remain accessible.

        Parameters
        ----------
        item : str
            Relationship/property name to access on this relationship's output
            concept.

        Returns
        -------
        Chain
            A chained value that represents following this relationship and
            then reading ``item`` on the relationship's output concept.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - The output concept does not have a relationship with this name and
              the ``model.implicit_properties`` config flag is False.
            - Relationships are accessed on a core concept (for example
                ``String`` or  ``Integer``).

        Notes
        -----
        Attribute access for relationships is only supported on user concepts.
        It is not supported on core (built-in) concepts such as ``String`` and
        ``Integer``.

        Examples
        --------
        Chain from a relationship to a property on its output concept:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Company = m.Concept("Company")
        >>> Person.employers = m.Relationship(f"{Person} works for {Company:employer}")
        >>> m.define(
        ...     alice := Person.new(name="Alice"),
        ...     acme := Company.new(name="Acme Inc."),
        ...     alice.employers(acme)
        ... )
        >>> # Select the names of Alice's employers by chaining from the
        >>> # relationship to the Company.name property:
        >>> m.select(alice.employers.name).to_df()
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        op = self._fields[-1].type._dot(item)
        return Chain(self, op)

    @include_in_docs
    def __call__(self, *args: Any, **kwargs: Any) -> Expression:
        """Return an expression asserting a fact in this relationship.

        Calling a :class:`Relationship` creates a composable :class:`Expression`
        that you can use in :meth:`Model.where` and :meth:`Model.define`. A common
        pattern is to use a relationship call as a readable condition, for
        example ``owns(alice, boots)`` to mean “Alice owns Boots”.

        This method is typically used in one of two ways:

        - **As a statement/condition**: provide all fields (including the output
            field) to assert that a specific fact holds.
        - **To return output values**: provide only the input fields and
            omit the output field; it is auto-filled with a fresh reference (a
            :class:`Ref` placeholder), and
            the expression evaluates to that output value. This is useful for
            selecting “all outputs that match these inputs”.

        Relationships are often attached as attributes on a concept (for example
        ``Person.pets = model.Relationship(...)``). Then ``Person.pets(alice, boots)``
        is the same as ``pets(alice, boots)``. Calling through a value is shorthand:
        ``alice.pets(boots)`` means ``pets(alice, boots)``. ``Person.pets(boots)``
        is another useful form: it means “Boots is some person's pet” and can be used
        to filter which ``Person`` values match.

        Notes
        -----
        This method does not write data by itself. To add facts to a model, use
        :meth:`Model.define` with the returned expression.

        Parameters
        ----------
        *args : Any
            Values for this relationship's fields. Provide either:

            - ``N`` values to specify all ``N`` fields explicitly, or
            - ``N-1`` values to omit the output field, in which case it is
                auto-filled with a fresh :class:`Ref`
                that represents output values matching the provided inputs.

        **kwargs : Any
            Extra named values attached to the resulting expression. Currently,
            these keyword arguments are ignored.

        Returns
        -------
        Expression
            A composable expression representing this relationship applied to
            the provided values.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - Too few or too many positional arguments are provided.
            - A :class:`TableSchema` is
              passed as an argument.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Attach a relationship and assert one fact:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Pet = m.Concept("Pet")
        >>> Person.pets = m.Relationship(f"{Person} cares for {Pet}")
        >>> m.define(
        ...     alice := Person.new(name="Alice"),
        ...     boots := Pet.new(),
        ...     alice.pets(boots)
        ... )

        Use the concept form as a readable filter ("Boots is some person's pet"):

        >>> m.select(Person.name).where(Person.pets(boots)).to_df()

        Return output values for a relationship with multiple input fields:

        >>> from relationalai.semantics import String
        >>> Company = m.Concept("Company")
        >>> Person.works_as = m.Relationship(f"{Person} works at {Company} as {String:title}")
        >>> m.define(acme := Company.new(name="Acme Inc."), alice.works_as(acme, "Engineer"))
        >>> # Select Alice's job title at Acme Inc.
        >>> m.select(alice.works_as(acme)).to_df()
        """
        return Expression(self, list(args), kwargs, root=self)

    @include_in_docs
    def __getitem__(self, field: str|int|Concept) -> FieldRef:
        """Return a reference to one of this relationship's fields.

        Indexing a :class:`Relationship` returns a :class:`FieldRef` that
        identifies a specific field in the relationship's ``fields`` list.
        This is commonly used when selecting or joining on non-output fields.

        The field can be selected by:

        - Integer position (0-based): ``rel[0]``
        - Exact field name: ``rel["owner"]``
        - Field type (concept): ``rel[Person]``

        Parameters
        ----------
        field : str | int | Concept
            Field selector.

        Returns
        -------
        FieldRef
            A reference to the selected field.

        Raises
        ------
        IndexError
            If ``field`` is an integer index that is out of range.
        KeyError
            If no field matches the provided selector.

        Examples
        --------
        Access fields by name:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.pets = m.Relationship(f"{Person:pet_parent} has pet {String:pet_name}")
        >>> m.define(alice := Person.new(name="Alice"), alice.pets("boots"))
        >>> m.select(Person.pets["pet_parent"].name, Person.pets["pet_name"]).to_df()

        Access fields by position:

        >>> m.select(Person.pets[0].name, Person.pets[1]).to_df()
        """
        resolved = _find_field(self._fields, field)
        if not resolved:
            raise KeyError(f"Field {field} not found in relationship with fields {[f.name for f in self._fields]}")
        return FieldRef(self, field, *resolved)

    @include_in_docs
    def alt(self, reading_str: str) -> Reading:
        """Add an alternative reading for this relationship.

        An alternative reading is a :class:`Reading` that refers to the same underlying
        relationship fields but uses a different human-readable reading string.
        This is commonly used to introduce inverse names (for example, defining
        ``Project.team`` as an alternative reading of ``Team.works_on``) or to
        provide a more natural phrase while keeping a single underlying relationship.

        The returned :class:`Reading` can be used anywhere a relationship can:
        you can call it to build expressions, select it in queries, and attach
        it as an attribute on a concept.

        Parameters
        ----------
        reading_str : str
            A reading string describing the relationship using the same field
            concepts and field names as this relationship. The string is parsed
            using the same placeholder format as relationship creation, typically
            written as an f-string (for example ``f"{Team:team} works on {Project}"``).

        Returns
        -------
        Reading
            A relationship-like handle that renders using ``reading_str``.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - The reading references a concept name that cannot be found in the model.
            - The reading contains the serialized form of a Python type
              (for example ``<class 'int'>``) instead of a concept.
        ValueError
            If the reading's fields do not match this relationship's fields.

        Examples
        --------
        Create a relationship and add an inverse reading:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Team = m.Concept("Team")
        >>> Project = m.Concept("Project")
        >>> Team.works_on = m.Relationship(f"{Team} works on {Project}")
        >>> m.define(
        ...     alpha := Team.new(name="Alpha"),
        ...     phoenix := Project.new(name="Phoenix"),
        ...     alpha.works_on(phoenix)
        ... )
        >>> # Declare an alternative reading so we can also say "Project involves Team"
        >>> Project.team = Team.works_on.alt(f"{Project} involves {Team}")
        >>> # Use the alternative reading to select the team for the Phoenix project
        >>> m.where(Project.name == "Phoenix").select(Project.team.name).to_df()
        """
        reading = Reading(self._model, self, reading_str)
        self._readings.append(reading)
        return reading

    @include_in_docs
    def annotate(self, *annos:Expression|Relationship) -> Relationship:
        """Attach one or more annotations to this relationship.

        An annotation is recorded on the relationship and emitted into the compiled
        model. Annotations are primarily used by backends and internal tooling
        for debugging and support workflows (for example, to attach metadata
        such as tracking labels). In general you should not need to use
        annotations unless RelationalAI support instructs you to add them for
        diagnostic purposes.

        Parameters
        ----------
        *annos : Expression | Relationship
            One or more annotation nodes to attach.

        Returns
        -------
        Relationship
            This relationship (returned to enable fluent chaining).

        Raises
        ------
        relationalai.util.error.RAIException
            Raised when an annotation argument has an unsupported type.
        """
        self._annotations.extend(annos)
        return self

    def _to_concept(self) -> Concept|None:
        return self._fields[-1].type

    def __dir__(self):
        default = set(super().__dir__())
        return sorted(default.union(self._fields[-1].type.__dir__()))

    @include_in_docs
    def to_df(self) -> DataFrame:
        """Materialize this relationship as a pandas DataFrame.

        This is a convenience method for inspecting the set of facts currently
        asserted for the relationship. It selects **all fields** of the
        relationship (including non-output fields) and filters to rows where
        the relationship holds.

        If you want to select only certain fields or apply additional filters,
        build a query with :meth:`Model.select`
        and :meth:`Model.where` instead.

        Returns
        -------
        pandas.DataFrame
            A DataFrame with one column per relationship field and one row per
            matching tuple.

        Notes
        -----
        ``to_df`` can be expensive for large relationships. Calling this method
        executes a query and materializes the full result in memory as a pandas
        DataFrame. In particular, it selects *all* fields of the relationship,
        which can produce many rows and wide columns.

        Examples
        --------
        Materialize all tuples of a relationship:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Pet = m.Concept("Pet")
        >>> Person.pets = m.Relationship(f"{Person} has pet {String:pet_name}")
        >>> m.define(
        ...     alice := Person.new(name="Alice"),
        ...     bob := Person.new(name="Bob"),
        ...     boots := Pet.new(name="Boots"),
        ...     miso := Pet.new(name="Miso"),
        ...     alice.pets(boots),
        ...     bob.pets(miso),
        ... )
        >>> Person.pets.to_df()
        """
        refs = [self[ix] for ix in range(len(self._fields))]
        return self._model.select(*refs).where(self(*refs)).to_df()

    @include_in_docs
    def inspect(self, end:str='\n'):
        """Print this relationship as a pandas DataFrame.

        This is a convenience method for quick interactive inspection. It is
        equivalent to ``print(self.to_df()); print(end=end)``.

        Parameters
        ----------
        end : str, optional
            String appended after the DataFrame is printed. Default is a newline.

        Returns
        -------
        None
            This method returns ``None`` and prints the DataFrame to stdout.

        Notes
        -----
        Like :meth:`Relationship.to_df`,
        this method executes a query and materializes the full result in memory.
        For large relationships, prefer building a query that selects only the
        fields you need (for example via
        :meth:`Model.select`).

        Examples
        --------
        Inspect all instances of a relationship:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.pets = m.Relationship(f"{Person} has pet {String:pet_name}")
        >>> m.define(alice := Person.new(name="Alice"), alice.pets("boots"))
        >>> Person.pets.inspect()
        """
        print(self.to_df())
        print(end=end)

#------------------------------------------------------
# Reading
#------------------------------------------------------

@include_in_docs
class Reading(Relationship):
    """Represents an alternative reading for a relationship.

    A :class:`Reading` refers to the same underlying fields as a :class:`Relationship`
    but uses a different human-readable reading string and possibly different
    field ordering.

    You typically create readings via :meth:`Relationship.alt`
    (or :meth:`Chain.alt`) and then use the
    result anywhere a relationship can be used.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Relationship.alt`.
    """
    def __init__(self, model: Model, relationship: Relationship, reading_str: str, reading_fields: list[Field] = [], reading_parts: list[str|int] = []):
        Variable.__init__(self, model)
        self._relationship = relationship
        if not reading_fields or not reading_parts:
            parsed_fields, parsed_parts = Reading.parse(model, reading_str, reading_fields, _source=self)
            reading_fields = reading_fields or parsed_fields
            reading_parts = reading_parts or parsed_parts
        rel_fields_len = len(relationship._fields)
        alt_fields_len = len(reading_fields)
        if rel_fields_len != alt_fields_len:
            exc(
                "Invalid alt reading",
                f"Relationship {relationship._short_name} has {rel_fields_len} fields but {alt_fields_len} were provided for the alternative reading.",
                [source(self)]
            )
        # make sure we can find all these fields in the base relationship
        index_map = {}
        matched = []
        for field_i, field in enumerate(reading_fields):
            found = next(((i, f) for i, f in enumerate(relationship._fields) if f._match(field)), None)
            if not found:
                raise ValueError(f"Field {field.name}:{field.type._name} not found in relationship with fields {[f'{f.name}:{f.type._name}' for f in relationship._fields]}")
            matched.append(found[1])
            index_map[field_i] = found[0]
        self._fields = matched
        self._reading = reading_str
        self._short_name = ""
        self._parts:list[str|int] = [index_map.get(p, p) for p in reading_parts]
        self._relationships = {}
        self._annotations: list[Expression|Relationship] = []

    #------------------------------------------------------
    # Parse
    #------------------------------------------------------

    @classmethod
    def parse(
        cls, model: Model, reading: str, known_fields: list[Field], _source
    ) -> Tuple[list[Field], list[str | int]]:
        # match <class 'foo'> which is the serialized form of a python type mistakenly passed instead of a concept
        class_pattern = re.compile(r'<class \'(.*?)\'>')
        match = class_pattern.search(reading)
        if match:
            extra: list = [source(_source)]
            if match.group(1) in pytype_to_concept_name:
                concept = pytype_to_concept_name[match.group(1)]
                extra.append(f" Did you mean to use [cyan]relationalai.semantics.{concept}[/cyan]?")
            exc("Invalid field type", f"The type '{match.group(1)}' is not a valid Concept.", extra)

        # {Type} or {name:Type}, where Type can include Number(38,14)
        pattern = re.compile(r'\{([a-zA-Z0-9_.#]+(?:(?:\([0-9]+,[0-9]+\))(?:[#0-9]+)?)?)(?::\s*([a-zA-Z0-9_.]+(?:\([0-9]+,[0-9]+\))?))?\}')

        namer = Namer()
        fields: list[Field] = []
        parts: list[str|int] = []

        last_end = 0
        is_old_style = True
        for m in pattern.finditer(reading):
            # literal chunk before this match
            parts.append(reading[last_end:m.start()])
            field_name, field_type_name = m.group(1), m.group(2)
            field_type_id = None
            if "#" in field_name:
                temp_name = field_type_name
                field_type_name, field_type_id = field_name.split("#")
                field_name = temp_name or sanitize(field_type_name.lower())
                is_old_style = False

            # if we don't have a type_name, then only a type was provided
            if not field_type_name:
                field_type_name = field_name
                field_name = sanitize(field_name.lower())

            field_name = namer.get_name(field_name)
            field_type = Concept.lookup_by_id.get(int(field_type_id)) \
                            if field_type_id \
                            else _find_concept(model, field_type_name, extra_types=[f.type for f in known_fields])
            if field_type is None:
                exc("Unknown Concept", f"The Concept '{field_type_name}' couldn't be found in the model", [
                    source(_source),
                ])

            fields.append(Field(field_name, field_type))
            parts.append(len(fields) - 1)
            last_end = m.end()

        # trailing literal after the final match
        if(last_end < len(reading)):
            parts.append(reading[last_end:])

        if is_old_style and fields:
            correct = []
            for part in parts:
                if isinstance(part, int):
                    type_name = fields[part].type._name
                    correct_type_name = type_name
                    if isinstance(fields[part].type, NumberConcept):
                        precision = fields[part].type._precision
                        scale = fields[part].type._scale
                        if scale == 0:
                            correct_type_name = "Integer"
                        else:
                            correct_type_name = f"Number.size({precision},{scale})"
                    if type_name.lower() != fields[part].name and correct_type_name.lower() != fields[part].name:
                        correct.append(f"{{{correct_type_name}:{fields[part].name}}}")
                    else:
                        correct.append(f"{{{correct_type_name}}}")
                else:
                    correct.append(part)
            warn("Deprecated format", "Plain strings for Relationships/Properties is deprecated. Use an f-string instead.", [
                source(_source),
                f'For example: [cyan]f"{"".join(correct)}"',
            ])

        return fields, parts

#------------------------------------------------------
# Property
#------------------------------------------------------

@include_in_docs
class Property(Relationship):
    """Represents a single-valued attribute on entities.

    A :class:`Property` is a specialized :class:`Relationship` that is intended
    to behave like an attribute (for example, a person's age or a pet's name).
    Unlike a general relationship, a property is treated as **single-valued**:
    the non-output fields (all fields except the last) uniquely determine the
    output field (the last field). For a typical binary property like
    ``age(person, value)``, this means a given person can have **at most one**
    age value.

    Properties are usually created via :meth:`Model.Property` and assigned as
    attributes on a :class:`Concept`. Calling a property produces an
    :class:`Expression` that you can use in :meth:`Model.define`,
    :meth:`Model.where`, and :meth:`Model.require`.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Model.Property`.

    The single-valued behavior is enforced by a uniqueness constraint that is
    added when the model is compiled.
    “Single-valued” means *at most one* output value for a given input; a
    property can still be missing for some entities.

    Examples
    --------
    Declare a property, define a couple of facts, and query:

    >>> from relationalai.semantics import Model, Integer, String
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> Person.name = m.Property(f"{Person} is named {String:name}")
    >>> Person.age = m.Property(f"{Person} is {Integer:age} years old
    >>> m.define(Person.new(name="Alice", age=30), Person.new(name="Bob", age=17))
    >>> m.where(Person.age >= 18).select(Person.name).to_df()
    """
    pass


#------------------------------------------------------
# Literal
#------------------------------------------------------

@include_in_docs
class Literal(Variable):
    """Wrap a Python value so it can be used in DSL expressions.

    ``Literal`` objects are typically created implicitly when you pass Python
    values (for example ``42`` or ``\"Alice\"``) to a relationship/property call
    or when building fragments via :meth:`Model.define`, :meth:`Model.where`, and
    :meth:`Model.select`.

    Parameters
    ----------
    value : Any
        The Python value to embed in the DSL.
    model : Model
        The model this literal belongs to.
    type : Concept, optional
        Explicit concept type for the literal. If omitted, the type is inferred.

    Raises
    ------
    NotImplementedError
        If the value type is not supported as a literal.

    Notes
    -----
    Most users should not instantiate this class directly; prefer passing Python
    values directly (for example ``Person.age(30)``).
    """

    def __init__(self, value: Any, model:Model, type: Concept|None = None):
        super().__init__(model)
        self._value = value
        self._type = type if type is not None else self._get_type(value)

    def _to_concept(self) -> Concept:
        return self._type

    @staticmethod
    def _get_type(value: Any) -> Concept:
        if type(value) is float:
            if math.isnan(value) or math.isinf(value):
                return CoreConcepts["Float"]
            str_value = str(value)
            if "e" in str_value:
                str_value = format(decimal.Decimal(value), "f")
            fractional_digits = min(len(str_value.split(".")[1]) if "." in str_value else 1, 14)
            return CoreConcepts["Number"].size(38, fractional_digits)  # type: ignore
        if type(value) is decimal.Decimal:
            str_value = format(value, 'f')
            if '.' in str_value:
                _, fractional_part = str_value.split('.')
                scale = min(len(fractional_part), 14)
            else:
                scale = 14
            return CoreConcepts["Number"].size(38, scale) # type: ignore
        if type(value) in pytype_to_concept_name:
            return CoreConcepts[pytype_to_concept_name[type(value)]]
        else:
            raise NotImplementedError(f"Literal type not implemented for value type: {type(value)}")

#------------------------------------------------------
# Chain
#------------------------------------------------------

@include_in_docs
class Chain(Variable):
    """Represents a chained relationship path created by attribute access.

    ``Chain`` objects are produced when you access a relationship/property as a
    Python attribute on a DSL value (for example ``Person.name`` or
    ``Person.works_at.name``). Calling a chain builds an :class:`Expression` that
    you can use in :meth:`Model.define` and :meth:`Model.where`.

    Parameters
    ----------
    start
        Root value for the chain (for example a :class:`Concept`,
        :class:`Ref`, or another chain).
    next
        Relationship/property to apply next.
    is_ref : bool, optional
        If True, mark the chain as a reference.

    Notes
    -----
    Most users should not instantiate this class directly.
    """
    def __init__(self, start: Chain|Concept|Relationship|DerivedColumn|Table|Ref|FieldRef|Expression, next: Relationship, is_ref = False):
        super().__init__(start._model)
        self._start = start
        self._next = next
        self._is_ref = is_ref

    @include_in_docs
    def __getattr__(self, item: str) -> Chain:
        """Return a new chain by extending this path via attribute access.

        Accessing an attribute on a :class:`Chain`
        (for example ``Person.pets.name``) returns another chain that represents
        following the current relationship path and then reading the
        relationship/property named by ``item``.

        Names that start with an underscore are treated as normal Python
        attribute access.

        Parameters
        ----------
        item : str
            Relationship/property name to access next.

        Returns
        -------
        Chain
            A chained value representing the extended relationship path.

        Raises
        ------
        relationalai.util.error.RAIException
            If the next relationship/property cannot be resolved (for example
            when the ``model.implicit_properties`` config flag is disabled).
        """
        if item.startswith("_"):
            return object.__getattribute__(self._next, item)
        next_rel = self._next._dot(item)
        return Chain(self, next_rel)

    @include_in_docs
    def __call__(self, *args: Any, **kwargs: Any) -> Expression:
        """Return an expression by calling the next relationship in the chain.

        Calling a :class:`Chain` is shorthand for calling the underlying
        :class:`Relationship` while implicitly supplying the chain's start value
        as the first argument when you omit it (for example, ``alice.pets(boots)``
        instead of ``pets(alice, boots)``).

        Parameters
        ----------
        *args : Any
            Positional arguments for the underlying relationship/property.
        **kwargs : Any
            Keyword field values passed through to the underlying call.

        Returns
        -------
        Expression
            A composable expression that can be used in
            :meth:`Model.define` and
            :meth:`Model.where`.
        """
        last = self._next
        assert not isinstance(last, Chain)
        if len(last._fields) > len(args):
            return Expression(last, [self._start, *args], kwargs, root=self)
        return Expression(last, [*args], kwargs, root=self)

    def to_dict_key(self) -> int|tuple:
        if self._is_ref:
            return self._id
        return (self._start.to_dict_key(), self._next.to_dict_key())

    @include_in_docs
    def __getitem__(self, field: str|int|Concept) -> FieldRef:
        """Return a reference to one of the next relationship's fields.

        Indexing a :class:`Chain` returns a :class:`FieldRef` that identifies
        a specific field of the relationship at the end of the chain.

        Parameters
        ----------
        field : str | int | Concept
            Field selector: a 0-based index, an exact field name, or a field
            type (concept).

        Returns
        -------
        FieldRef
            A reference to the selected field.

        Raises
        ------
        IndexError
            If ``field`` is an integer index that is out of range.
        KeyError
            If no field matches the provided selector.
        """
        resolved = _find_field(self._next._fields, field)
        if not resolved:
            raise KeyError(f"Field {field} not found in relationship with fields {[f.name for f in self._next._fields]}")
        return FieldRef(self, field, *resolved)

    @include_in_docs
    def ref(self) -> Chain:
        """Return an independent occurrence of this chain.

        Use this when you need two separate matches of the same chain path
        (for example two different results from a multi-valued relationship).
        This does not introduce a new entity variable; it only makes this chain
        distinct from other identical paths.

        Returns
        -------
        Chain
            A chain treated as distinct from other identical chains.

        Examples
        --------
        Select pairs of pets for the same person:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.pets = m.Relationship(f"{Person} has pet {String:pet_name}")
        >>> m.define(alice := Person.new(name="Alice"), alice.pets("boots"), alice.pets("miso"))
        >>> pet1, pet2 = alice.pets.ref(), alice.pets.ref()
        >>> m.where(pet1 != pet2).select(pet1, pet2).to_df()
        """
        return Chain(self._start, self._next, is_ref=True)

    @include_in_docs
    def alt(self, reading_str: str) -> Reading:
        """Add an alternative reading for the relationship at the end of this chain.

        This is a convenience wrapper around :meth:`Relationship.alt` that lets
        you call ``.alt(...)`` on a chained value (for example ``alice.pets``).
        The reading is added to the underlying relationship, not to the chain's
        start value.

        Parameters
        ----------
        reading_str : str
            Reading string passed through to
            :meth:`Relationship.alt`.

        Returns
        -------
        Reading
            A relationship-like handle that renders using ``reading_str``.
        """
        return self._next.alt(reading_str)

    @include_in_docs
    def annotate(self, *annos:Expression|Relationship) -> Relationship:
        """Attach one or more annotations to the relationship at the end of this chain.

        This is a convenience wrapper around :meth:`Relationship.annotate`.
        Annotations are recorded on the underlying relationship and emitted into
        the compiled model. Annotations are primarily used by backends and
        internal tooling for debugging and support workflows (for example, to
        attach metadata such as tracking labels). In general you should not need
        to use annotations unless RelationalAI support instructs you to add them
        for diagnostic purposes.

        Parameters
        ----------
        *annos : Expression | Relationship
            One or more annotation nodes to attach.

        Returns
        -------
        Relationship
            The underlying relationship (returned to enable fluent chaining).

        Raises
        ------
        relationalai.util.error.RAIException
            Raised when an annotation argument has an unsupported type.
        """
        return self._next.annotate(*annos)

    def _to_concept(self) -> Concept | None:
        return self._next._to_concept()

#------------------------------------------------------
# Expression
#------------------------------------------------------

@include_in_docs
class Expression(Variable):
    """Represents an expression in the DSL.

    ``Expression`` objects are produced when you call a :class:`Relationship`,
    :class:`Property`, :class:`Chain`, or :class:`Concept` (and by operators
    like ``==`` and ``+``). You can pass them to :meth:`Model.define`,
    :meth:`Model.where`, and :meth:`Model.select`.

    Because expressions are composable, you can build complex nested expressions
    that combine multiple relationships, properties, and operators.

    Notes
    -----
    Most users should not instantiate this class directly.

    Parameters
    ----------
    op : Relationship | Concept
        The operation being applied.
    args : Sequence[Value]
        Positional arguments. Python primitives are wrapped as
        :class:`Literal`.
    kwargs : dict, optional
        Keyword field values for the call.
    root : Chain | Relationship | Concept, optional
        Root used for chained expressions.

    Examples
    --------
    Build a query by comparing a relationship's output to a literal value:

    >>> from relationalai.semantics import Model, String
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> Person.pets = m.Relationship(f"{Person} has pet {String:pet_name}")
    >>> m.define(alice := Person.new(name="Alice"), alice.pets("Boots"))
    >>> # The expression alice.pets == "Boots" produces an Expression
    >>> # that we can use in a where clause to filter to people whose pet is Boots
    >>> m.where(alice.pets == "Boots").select(alice).to_df()
    """
    def __init__(self, op: Relationship|Concept, args: Sequence[Value], kwargs: dict|None = None, root: Chain|Relationship|Concept|None = None):
        super().__init__(op._model)
        self._op = op
        self._has_output = isinstance(op, Concept) or (any(not f.is_input for f in op._fields))
        self._auto_filled = False
        self._root = root

        # clean args
        clean_args = []
        for arg in args:
            if isinstance(arg, (ModelEnum, Field, TupleVariable)):
                pass
            elif isinstance(arg, TableSchema):
                exc("Invalid argument", "Cannot use a schema as an argument to an Expression.", [source(self)])
            elif not isinstance(arg, Variable):
                arg = Literal(arg, self._model)
            elif isinstance(arg, (Match, Fragment)):
                arg._columns
            clean_args.append(arg)
        self._args = clean_args

        self._kwargs = kwargs or {}
        # clean kwargs
        for k, v in self._kwargs.items():
            if not isinstance(v, Variable):
                self._kwargs[k] = Literal(v, self._model)

        if isinstance(op, Relationship):
            op_len = len(op._fields)
            arg_len = len(self._args)
            if op_len - arg_len == 1:
                if arg_len == 0 or (arg_len == 1 and isinstance(root, Chain) and self._args[0] is root._start):
                    from relationalai.semantics.frontend.pprint import format
                    if isinstance(root, Relationship):
                        exp = root._short_name if root._short_name else f"`{format(root)}`"
                    else:
                        exp = format(root)
                    exc(
                        "Missing arg",
                        f"Expression {exp}() cannot be used without an argument.",
                        [source(self),
                        f"Either use the expression without empty parentheses ([cyan]{exp}[/cyan]) or provide an argument."]
                    )
                self._args.append(op._fields[-1].type.ref(name=op._fields[-1].name))
                self._auto_filled = True
            elif op_len != arg_len:
                dir = "Too few" if arg_len < op_len - 1 else "Too many"
                exc(f"{dir} args", f"{op._short_name or 'Relationship'} requires {op_len - 1}-{op_len} arguments but got {arg_len}", [
                    source(self),
                ])
        elif not isinstance(self, New) and isinstance(op, Concept) and len(self._args) == 0 and len(self._kwargs) == 0:
            exc(
                "Missing arg",
                f"Concept expression {op._name}() cannot be used without an argument.",
                [source(self),
                f"Usually, you just need [cyan]`{op._name}`[/cyan] unless you want to cast an existing expression."]
            )

    @include_in_docs
    def __getattr__(self, item: str) -> Chain:
        """Follow a relationship/property from this expression.

        Attribute access returns a :class:`Chain`
        rooted at this expression. This is mainly useful when a relationship call
        leaves its output "open" (by providing fewer than the full number of
        arguments), and you then want to access another relationship/property from
        that output.

        Names that start with an underscore are treated as normal Python attribute
        access.

        Parameters
        ----------
        item : str
            Relationship/property name to access next.

        Returns
        -------
        Chain
            A chained value representing the extended relationship path.

        Raises
        ------
        relationalai.util.error.RAIException
            If the relationship/property cannot be resolved (for example when
            the ``model.implicit_properties`` config flag is disabled).

        Examples
        --------
        Access a property on a relationship's output field via chaining from an expression:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Company = m.Concept("Company")
        >>> works_at = m.Relationship(f"{Person} works at {Company}")
        >>> Company.name = m.Property(f"{Company} is named {String:name}")
        >>> alice = Person.new(name="Alice")
        >>> # Follow the relationship output (Company) and then read its name
        >>> m.select(works_at(alice).name).to_df()
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        rel = self._op._dot(item)
        return Chain(self, rel)

    @include_in_docs
    def __getitem__(self, field: str|int|Concept) -> FieldRef:
        """Return a reference to one of this expression's relationship fields.

        Indexing an :class:`Expression` returns a :class:`FieldRef` for
        the underlying relationship.

        Parameters
        ----------
        field : str | int | Concept
            Field selector: a 0-based index, an exact field name, or a field
            type (concept).

        Returns
        -------
        FieldRef
            A reference to the selected field.

        Raises
        ------
        TypeError
            If this expression is not a relationship call.
        IndexError
            If ``field`` is an integer index that is out of range.
        KeyError
            If no field matches the provided selector.
        """
        if not isinstance(self._op, Relationship):
            raise TypeError(f"Cannot index into Expression with non-Relationship: {self._op}")
        resolved = _find_field(self._op._fields, field)
        if not resolved:
            raise KeyError(f"Field {field} not found in relationship with fields {[f.name for f in self._op._fields]}")
        return FieldRef(self, field, *resolved)

    def _to_concept(self) -> Concept | None:
        return self._args[-1]._to_concept() if self._args else self._op._to_concept()

#------------------------------------------------------
# New
#------------------------------------------------------

@include_in_docs
class New(Expression):
    """Represents a "new entity" statement for a concept.

    ``New`` objects are produced by :meth:`Concept.new` (and :meth:`Concept.to_identity`) rather
    than constructed directly. They can be passed to :meth:`Model.define` to create/update an
    entity, and they can also be used as a value in other statements (for example
    as the argument to a relationship).

    Parameters
    ----------
    concept : Concept
        The entity concept being constructed/referred to.
    args : Sequence[Any]
        Optional attached statements and/or schemas (expanded like
        :meth:`Concept.new`).
    kwargs : dict[str, Any]
        Property/relationship values to set (or to filter by, when used as a
        lookup).
    identity_only : bool, optional
        If True, compile as an identity-only lookup (see
        :meth:`Concept.to_identity`).

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Concept.new`.

    Examples
    --------
    >>> from relationalai.semantics import Model
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> m.define(Person.new(id=1, name="Alice"))
    >>> m.where(Person.id == 1).select(Person.name).to_df()
    """
    def __init__(self, concept: Concept, args: Sequence[Any], kwargs: dict[str, Any], identity_only: bool = False):
        clean_args = []
        row_ids = []
        for arg in args:
            if isinstance(arg, TableSchema):
                row_ids.append(arg._table)
                lower_case_kwargs = {k.lower() for k, v in kwargs.items()}
                # add any keyword args that aren't already in kwargs
                for col in arg.get_columns():
                    if col._short_name and col._short_name.lower() not in lower_case_kwargs:
                        kwargs[col._short_name] = col(arg._table)
            else:
                clean_args.append(arg)
        for k, v in kwargs.items():
            if not isinstance(concept, Table) and k.lower() in Concept.RESERVED_NAMES:
                exc("Reserved relationship name",
                    f"'{k}' is a reserved name and cannot be used as a relationship name on concept '{concept._name}'.",
                    [source(v)]
                )
            concept._dot(k)
        super().__init__(concept, clean_args, kwargs)
        self._identity_only = identity_only
        self._row_ids = row_ids

#------------------------------------------------------
# FilterBy
#------------------------------------------------------

@include_in_docs
class FilterBy(Expression):
    """Represents a keyword-based filter expression for a concept or reference.

    This is the expression returned by :meth:`Concept.filter_by` and
    :meth:`Ref.filter_by`. It can be used anywhere an :class:`Expression` is
    accepted (most commonly in :meth:`Model.where`).

    Parameters
    ----------
    item : Concept | Ref
        The concept (or a specific reference to it) being filtered.
    kwargs : dict[str, Any]
        Property/relationship values to match (keys are treated case-insensitively).

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Concept.filter_by` and
    :meth:`Ref.filter_by`.
    """
    def __init__(self, item: Concept|Ref, kwargs: dict[str, Any]):
        concept = item._concept if isinstance(item, Ref) else item
        super().__init__(concept, [item], kwargs)

#------------------------------------------------------
# FieldRef
#------------------------------------------------------

@include_in_docs
class FieldRef(Variable):
    """Refer to a specific field of a relationship.

    ``FieldRef`` objects are created by indexing a :class:`Relationship`,
    :class:`Chain`, or :class:`Expression` (for example, ``Person.pets["pet_name"]``).
    They are commonly used to select, join, or filter on non-output fields of a relationship.

    Parameters
    ----------
    root : Chain | Relationship | Expression
        The relationship-like object being indexed.
    field : str | int | Concept
        The selector used to identify the field (name, 0-based index, or field
        concept type).

    Notes
    -----
    Most users should not construct ``FieldRef`` objects directly. Prefer
    indexing helpers such as
    :meth:`Relationship.__getitem__`,
    :meth:`Chain.__getitem__`, and
    :meth:`Expression.__getitem__`.
    """

    def __init__(self, root: Chain|Relationship|Expression, field: str|int|Concept, resolved: Field, resolved_ix:int):
        super().__init__(root._model)
        self._root = root
        self._field = field
        self._resolved = resolved
        self._resolved_ix = resolved_ix

    def _to_concept(self) -> Concept:
        return self._resolved.type

    @include_in_docs
    def __getattr__(self, item):
        """Return a chained relationship reference for attribute access.

        A :class:`FieldRef` behaves like a variable of its field's :class:`Concept`
        type. Accessing ``Person.works_for["company"].name`` returns a
        :class:`Chain` rooted at this field reference.

        If the attribute name starts with an underscore, this method defers to
        normal Python attribute access so internal attributes remain accessible.

        Parameters
        ----------
        item : str
            Relationship/property name to access on the field's concept type.

        Returns
        -------
        Chain
            A chained value representing the relationship reading.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised in the following cases:

            - The field's concept type does not declare the relationship and
              implicit properties are disabled (``model.implicit_properties`` config flag).
            - Relationships are accessed on a core concept,
              like :attr:`CoreConcepts.String` or
              :attr:`CoreConcepts.Integer`.
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        return Chain(self, self._resolved.type._dot(item))

    def to_dict_key(self) -> tuple:
        return (self._root.to_dict_key(), self._resolved._id)


#------------------------------------------------------
# MetaRef
#------------------------------------------------------

class MetaRef(Variable):
    def __init__(self, target: Concept|Relationship|Field):
        model = target._model if isinstance(target, DSLBase) else target.type._model
        super().__init__(model)
        self._target = target

#------------------------------------------------------
# TupleVariable
#------------------------------------------------------

@include_in_docs
class TupleVariable:
    """Group multiple DSL values into a single tuple-like argument.

    ``TupleVariable`` wraps a sequence of values so they can be passed as a
    single :data:`Value` (most commonly to satisfy list-valued relationship fields
    declared with :class:`Field` ``is_list=True``).

    During compilation, the wrapper is lowered to a tuple of its items.

    Notes
    -----
    Most users should not instantiate this class directly. It is primarily used
    by standard-library helpers (for example string/constraint utilities) and
    by internal compilation code.

    Parameters
    ----------
    items : Sequence[Value | Distinct]
        Values to group into a single argument.
    """
    def __init__(self, items: Sequence[Value|Distinct]):
        self._items = list(items)
        self._source = SourcePos.new()

#------------------------------------------------------
# AsBool
#------------------------------------------------------

@include_in_docs
class AsBool(Variable):
    """Represents a boolean-typed wrapper around a DSL expression.

    ``AsBool`` wraps another DSL value and forces its type to be :class:`CoreConcepts.Boolean`,
    so it can be selected as a column or composed like other values.

    At execution time, the result is ``True`` when the wrapped expression can
    succeed for the current bindings, and ``False`` otherwise.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Variable.as_bool` or selecting a boolean expression directly.
    """
    def __init__(self, item: Variable):
        super().__init__(item._model)
        self._item = item

    def _to_concept(self) -> Concept:
        return CoreConcepts["Boolean"]

#------------------------------------------------------
# Alias
#------------------------------------------------------

@include_in_docs
class Alias(Variable):
    """Represents a DSL value labeled with an output alias.

    ``Alias`` objects are typically created via :meth:`Variable.alias` and are used
    to name selected outputs (and enable indexing a fragment by that name, for
    example ``query[\"AGE\"]``).

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Variable.alias`.

    Parameters
    ----------
    source : Variable
        The underlying value being labeled.
    alias : str
        The name to attach to the value in query outputs.
    """
    def __init__(self, source: Variable, alias: str):
        super().__init__(source._model)
        self._source_item = source
        self._alias = alias

    def _to_concept(self) -> Concept|None:
        return self._source_item._to_concept()

    @include_in_docs
    def __format__(self, format_spec: str) -> str:
        """Return the placeholder used when formatting this alias in readings.

        This is mainly used when interpolating values into relationship/property
        reading strings (see :meth:`Concept.__format__`). When the aliased value
        is a :class:`Concept`, formatting the alias is equivalent to formatting
        the concept with the alias name as the field name.

        Parameters
        ----------
        format_spec : str
            Additional format spec. This is only allowed when the aliased value
            is not a concept.

        Returns
        -------
        str
            A placeholder string for use in a reading.

        Raises
        ------
        relationalai.util.error.RAIException
            If the aliased value is a concept and an additional ``format_spec``
            is provided.
        """
        if isinstance(self._source_item, Concept):
            if format_spec:
                exc("Invalid alias", f"Alias already specifies an alias for this concept, you can remove `:{format_spec}`")
            return self._source_item.__format__(self._alias)
        return super().__format__(format_spec)

#------------------------------------------------------
# Match
#------------------------------------------------------

@include_in_docs
class Match(DerivedTable):
    """Represents a left-to-right fallback across alternative branches.

    ``Match`` is the object created by the ``|`` operator for semantics DSL
    values and statements (see :meth:`Variable.__or__`). At execution
    time, it evaluates branches left-to-right and picks the first branch that
    can succeed for the current bindings.

    Each branch must return the same number of values; the result behaves like
    a derived table with that many columns.

    Nested matches are flattened, so ``(a | b) | c`` is equivalent to
    ``a | b | c``.

    Parameters
    ----------
    model : Model
        The model this match belongs to.
    *items : Statement
        Branches to try in order. Each branch must return the same number of
        values.

    Notes
    -----
    Most users should not instantiate this class directly. Use ``a | b`` for
    fallback/default behavior, or
    :meth:`Model.union` to combine results
    from multiple branches.
    """
    def __init__(self, model:Model, *items: Statement):
        super().__init__(model)
        t = type(self)
        self._items = [
            x
            for item in items
            for x in (item._items if (type(item) is t and isinstance(item, Match)) else (item,))
        ] # flatten nested Matches/Unions

    def _get_cols(self) -> list[DerivedColumn]:
        return [DerivedColumn(self, i) for i in range(self._arg_count())]

    def _arg_count(self) -> int:
        counts = []
        for item in self._items:
            if isinstance(item, DerivedTable):
                try:
                    counts.append(len(item._columns))
                except ValueError:
                    counts.append(0)
            else:
                # Expressions with no output and Not are filters, do not count as returning values
                is_filter = isinstance(item, Not) or (isinstance(item, Expression) and not item._has_output)
                counts.append(0 if is_filter else 1)
        if not counts:
            return 0
        first = counts[0]
        if any(c != first for c in counts[1:]):
            exc("Inconsistent branches",
                f"All branches in a {self.__class__.__name__} must have the same number of returned values",
                [source(self)])
        return first

    @include_in_docs
    def __getattr__(self, item) -> Chain:
        """Follow a relationship/property from the last returned value.

        This forwards attribute access to the last output column of the match,
        so you can write expressions like ``union(Cat, Dog).name`` or
        ``(a | b).name`` when the branches return an entity-like value.

        Names that start with an underscore are treated as normal Python
        attribute access.

        Parameters
        ----------
        item : str
            Relationship/property name to access.

        Returns
        -------
        Chain
            A chained value rooted at the last output column.
        """
        if item.startswith("_"):
            return object.__getattribute__(self, item)
        return getattr(self._columns[-1], item)

#------------------------------------------------------
# Union
#------------------------------------------------------

@include_in_docs
class Union(Match):
    """Represents a logical OR / union across multiple branches.

    ``Union`` is the object created by :meth:`Model.union`. Unlike :class:`Match`
    (created by the ``|`` operator), a union combines the results from *all*
    branches rather than picking the first branch that can succeed.

    Nested unions are flattened, so ``m.union(m.union(a, b), c)`` is equivalent
    to ``m.union(a, b, c)``.

    Parameters
    ----------
    model : Model
        The model this union belongs to.
    *items : Value
        Branches to combine. Each branch must return the same number of values.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer :meth:`Model.union`
    (or the convenience function :func:`~relationalai.semantics.union`).
    """
    def __init__(self, model:Model, *items: Value):
        # unions are always "distinct", so unwrap distinct nodes from any nested selects
        for item in items:
            if isinstance(item, Fragment) and item._select:
                args = item._select
                if any(isinstance(arg, Distinct) for arg in args):
                    if len(args) != 1:
                        exc("Invalid distinct", "Distinct must be applied to all arguments", [source(item)])
                    assert isinstance(args[0], Distinct)
                    item._select = list(args[0]._items)
        super().__init__(model, *items)

#------------------------------------------------------
# Not
#------------------------------------------------------

@include_in_docs
class Not(DSLBase):
    """Represents a negated condition for use in query filters.

    ``Not`` is the object created by :meth:`Model.not_` (and the convenience function
    :func:`~relationalai.semantics.not_`). Use it inside :meth:`Model.where` to exclude
    bindings that match a pattern; multiple items are treated as a single
    grouped condition (``NOT (a AND b)``).

    Parameters
    ----------
    *items : Statement
        One or more statements to negate.
    model : Model
        The model this negation belongs to.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Model.not_`.
    """
    def __init__(self, *items: Statement, model:Model):
        super().__init__(model)
        self._items = items

    @include_in_docs
    def __or__(self, other) -> Match:
        """Return a match expression (``self | other``).

        ``a | b`` builds a :class:`Match`
        with two branches. At execution time, the match picks the first branch
        (left-to-right) that can succeed for the current bindings.

        Parameters
        ----------
        other : Statement
            The right-hand branch.

        Returns
        -------
        Match
            A composable match expression.
        """
        return Match(self._model, self, other)

    @include_in_docs
    def __and__(self, other) -> Fragment:
        """Return a fragment that combines conditions with AND (``self & other``).

        Use ``&`` to explicitly conjoin conditions when building filters. This is
        equivalent to passing multiple conditions to :meth:`Model.where`.

        If ``other`` is a :class:`Fragment`, this returns a new fragment with ``self``
        added to ``other``'s filters (roughly ``other.where(self)``).

        Notes
        -----
        Do not use Python's boolean ``and`` with DSL values. Use ``&``
        (or pass multiple conditions to ``where``) instead.

        Parameters
        ----------
        other : Statement or Fragment
            The right-hand operand. This is typically a condition expression,
            but it may also be an existing fragment to extend.

        Returns
        -------
        Fragment
            A composable fragment representing the conjunction.

        Raises
        ------
        relationalai.util.error.RAIException
            If ``other`` is not a valid statement in a filter.
        NotImplementedError
            If a Python value cannot be converted to a supported literal.

        Examples
        --------
        Extend an existing fragment by adding another condition, then continue
        chaining:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> User = m.Concept("User")
        >>> m.define(
        ...     User.new(username="alice", age=30, status="active"),
        ...     User.new(username="bob", age=15, status="inactive"),
        ...     User.new(username="carol", age=40, status="active"),
        ... )
        >>> active_users = m.where(User.status == "active")
        >>> active_adults = active_users & m.not_(User.age < 18)
        >>> active_adults.select(User.username).to_df()
        """
        if isinstance(other, Fragment):
            new = other.where()
            new._where.insert(0, self)
            return new
        return self._model.where(self, other)

#------------------------------------------------------
# Distinct
#------------------------------------------------------

@include_in_docs
class Distinct(DSLBase):
    """Represents a "distinct" wrapper for values in a query.

    ``Distinct`` objects are created by :meth:`Model.distinct` (and the
    convenience function :func:`~relationalai.semantics.distinct`).
    Use them either as the *only* argument to :meth:`Model.select` to remove
    duplicate rows, or as an argument wrapper for aggregates (for example
    ``count(m.distinct(Person.name))``) to aggregate over unique values.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer :meth:`Model.distinct`.
    ``Distinct`` must be used in the contexts described above; using it
    elsewhere raises :class:`~relationalai.util.error.RAIException`.

    Parameters
    ----------
    *items : Value
        One or more :data:`Value` items to
        consider when removing duplicates.
    model : Model
        The model this wrapper belongs to.
    """
    def __init__(self, *items: Value, model:Model):
        super().__init__(model)
        self._items = items

#------------------------------------------------------
# Aggregate
#------------------------------------------------------

@include_in_docs
class Group(DSLBase):
    """Represent grouping keys for an aggregate.

    ``Group`` stores the values passed to :meth:`Aggregate.per` (for example
    ``count(Person).per(Person.city)``) and is used when the query is compiled.

    Notes
    -----
    Most users should not instantiate this class directly; prefer
    :meth:`Aggregate.per`.

    Parameters
    ----------
    *args : Value
        One or more values to group by.
    """
    def __init__(self, *args: Value):
        model = [arg._model for arg in args if isinstance(arg, Variable)][0] if args else None
        super().__init__(model) # type: ignore
        self._args = list(args)

    def _extend(self, args: Sequence[Value]) -> Group:
        new = Group(*self._args)
        new._args.extend(args)
        return new

    def _clone(self):
        return Group(*self._args)

@include_in_docs
class Aggregate(Variable):
    """Represents an aggregate computation in the DSL.

    ``Aggregate`` objects are returned by aggregate helpers such as
    :func:`~relationalai.semantics.count` and :func:`~relationalai.semantics.sum`.
    Use :meth:`per` to add grouping keys (like SQL ``GROUP BY``) and
    :meth:`where` to restrict which bindings contribute to the aggregate.

    To aggregate over unique values, wrap arguments with
    :meth:`Model.distinct` (or :func:`~relationalai.semantics.distinct`).

    Parameters
    ----------
    op : Relationship
        The aggregate operator (typically created by the standard aggregate
        helpers).
    *args : Value | Distinct
        Values to aggregate. Pass a single :class:`Distinct`
        to aggregate over unique values.
    distinct : bool, optional
        If True, treat the aggregate as distinct.

    Notes
    -----
    Most users should not instantiate this class directly.
    """
    def __init__(self, op: Relationship, *args: Value|Distinct, check_args: bool = True, distinct: bool = False):
        model = self._find_model(args) or op._model
        super().__init__(model)
        self._op = op
        self._where = Fragment(model)
        self._group = Group()
        self._args: list[Value] = []
        self._projection_args: list[Value] = []
        self._distinct = distinct
        if check_args:
            # unwrap distinct if present
            if any(isinstance(arg, Distinct) for arg in args):
                if len(args) != 1:
                    exc("Invalid distinct", "Distinct must be applied to all arguments", [source(self)])
                assert isinstance(args[0], Distinct)
                args = args[0]._items
                self._distinct = True

            args = cast(tuple[Value], args)

            num_inputs = sum(f.is_input for f in op._fields)
            if len(args) < num_inputs:
                need = [f.name for f in op._fields if f.is_input][len(args):]
                exc("Missing argument",
                    f"`{op._short_name or 'Relationship'}(..)` is missing: {', '.join(need)}",
                    [source(self)])

            self._projection_args = list(args[:-num_inputs] if num_inputs else args)
            supplied = iter(args[-num_inputs:] if num_inputs else [])

            self._args = [
                (next(supplied) if f.is_input else f.type.ref(f.name))
                for f in op._fields
            ]

    def _find_model(self, args: Sequence[Value|Distinct|TupleVariable]) -> Model|None:
        for arg in args:
            if isinstance(arg, (Variable, Distinct)):
                return arg._model
            elif isinstance(arg, TupleVariable):
                for item in arg._items:
                    if isinstance(item, (Variable, Distinct)):
                        return item._model
        return None


    @include_in_docs
    def where(self, *args: Statement) -> Aggregate:
        """Return a new aggregate restricted to bindings that satisfy the given
        conditions.

        This behaves like filtering *inside* the aggregate: only matches that
        satisfy the conditions contribute to the aggregate result. (It does not
        filter the rows of the surrounding query; use :meth:`Fragment.where` for that.)

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to use as filter conditions.

        Returns
        -------
        Aggregate
            A new aggregate with the additional filter(s) applied.

        Examples
        --------
        Count only employees older than 30 per department:

        >>> from relationalai.semantics import Integer, Model, count
        >>> m = Model()
        >>> Department = m.Concept("Department")
        >>> Employee = m.Concept("Employee")
        >>> Employee.age = m.Property(f"{Employee} is {Integer:age} years old")
        >>> Department.employees = m.Relationship(f"{Department} has {Employee:employees}")
        >>> employees = Department.employees
        >>> count(employees).per(Department).where(employees.age > 30)
        """
        new = self._clone()
        new._where = new._where.where(*args)
        return new

    @include_in_docs
    def per(self, *args: Value) -> Aggregate:
        """Return a new aggregate grouped by the given key(s).

        This behaves like adding SQL ``GROUP BY`` keys: the aggregate is
        computed separately for each unique combination of the grouping values.

        Parameters
        ----------
        *args : Value
            One or more values to group by.

        Returns
        -------
        Aggregate
            A new aggregate with the additional grouping key(s) applied.

        Examples
        --------
        Count employees per department:

        >>> from relationalai.semantics import Model, count
        >>> m = Model()
        >>> Department = m.Concept("Department")
        >>> Employee = m.Concept("Employee")
        >>> Department.employees = m.Relationship(f"{Department} has {Employee:employees}")
        >>> count(Department.employees).per(Department)
        """
        new = self._clone()
        new._group = new._group._extend(args)
        return new

    def _clone(self):
        agg = Aggregate(self._op, check_args=False)
        agg._args = self._args
        agg._projection_args = self._projection_args
        agg._where = self._where
        agg._group = self._group
        agg._distinct = self._distinct
        return agg

#------------------------------------------------------
# Data
#------------------------------------------------------

@include_in_docs
class Data(Table):
    """Represents a temporary table backed by in-memory data.

    ``Data`` objects are created by :meth:`Model.data` (and the convenience function
    :func:`~relationalai.semantics.data`). They behave like a :class:`Table`, so you
    can refer to their columns (for example ``d.name`` or ``d["name"]``) in queries.

    Notes
    -----
    Most users should not instantiate this class directly. Prefer
    :meth:`Model.data`.
    """

    def __init__(self, df:DataFrame, model:Model, schema: dict[str, Concept]|None = None):
        col_schema = {}
        for col in df.columns:
            col_name = f"col{col}" if isinstance(col, int) else col
            if schema and col_name in schema:
                col_schema[col_name] = _find_concept(model, schema[col_name], default=CoreConcepts["Any"])
            else:
                col_schema[col_name] = CoreConcepts[column_to_concept_name(df, col)]
        # Add schema-defined columns not present in the DataFrame (e.g. empty data [])
        if schema:
            for col_name, col_type in schema.items():
                if col_name not in col_schema:
                    col_schema[col_name] = _find_concept(model, col_type, default=CoreConcepts["Any"])
        # Ensure the DataFrame has all schema-declared columns — backends iterate
        # df.columns to build per-column IR nodes and will miss any that are absent.
        missing = [c for c in col_schema if c not in df.columns]
        if missing:
            df = df.reindex(columns=list(df.columns) + missing)
        super().__init__("Data", col_schema, model)
        self._data = df


    @staticmethod
    def raw_to_df(data: DataFrame | list[tuple] | list[dict], columns:list[str]|None) -> DataFrame:
        from pandas import DataFrame as PdDataFrame
        if isinstance(data, PdDataFrame):
            return data
        if not data:
            return PdDataFrame()
        if isinstance(data, list):
            if isinstance(data[0], tuple):
                # Named tuple check
                if hasattr(data[0], '_fields'):
                    return PdDataFrame([t._asdict() for t in data]) #type: ignore
                return PdDataFrame(data, columns=columns)
            elif isinstance(data[0], dict):
                return PdDataFrame(data)
        raise TypeError(f"Cannot convert {type(data)} to PdDataFrame. Use PdDataFrame, list of tuples, or list of dicts.")

#------------------------------------------------------
# Enum
#------------------------------------------------------

class ModelEnumMeta(EnumMeta):
    _concept: Concept
    def __setattr__(self, name: str, value: Any) -> None:
        if name.startswith("_") or isinstance(value, self):
            super().__setattr__(name, value)
        elif isinstance(value, (Relationship, Reading)):
            setattr(self._concept, name, value)
        else:
            raise AttributeError(f"Cannot set attribute {name} on {type(self).__name__}")

    def __format__(self, format_spec: str) -> str:
        return format(self._concept, format_spec)

class ModelEnum(Enum, metaclass=ModelEnumMeta):
    _model:Model

    def __init__(self, *args) -> None:
        super().__init__()
        self._source = SourcePos.new()

    def _compile_lookup(self):
        concept = getattr(self.__class__, "_concept")
        return concept.to_identity(name=self.name)

    @classmethod
    def lookup(cls, value:Variable|str):
        concept = cls._concept
        return concept.to_identity(name=value)

    # Python 3.10 doesn't correctly populate __members__ by the time it calls
    # __init_subclass__, so we need to initialize the members lazily when we
    # encounter the enum for the first time.
    @classmethod
    def _init_members(cls):
        if cls._has_inited_members:
            return
        c = cls._concept
        # Add the name and value attributes to the hashes we create for the enum
        members = [
            c.new(name=name, value=value.value)
            for name, value in cls.__members__.items()
        ]
        cls._model.define(*members)
        cls._has_inited_members = True

    def __format__(self, format_spec: str) -> str:
        return format(self._concept, format_spec)

def create_enum_class(model: Model):
    class AttachedModelEnum(ModelEnum):
        def __init_subclass__(cls, **kwargs):
            super().__init_subclass__(**kwargs)
            # this is voodoo black magic that is doing meta meta programming where
            # we are plugging into anytime a new subtype of this class is created
            # and then creating a concept to Represents the enum. This happens both
            # when you do `class Foo(Enum)` and when you do `Enum("Foo", [a, b, c])`
            cls._model = model
            c = model.Concept(
                cls.__name__,
                extends=[CoreConcepts["Enum"]],
                identify_by={"name": CoreConcepts["String"]}
            )
            model.enums.append(cls)
            model.enums_index[cls.__name__] = cls
            cls._has_inited_members = False
            cls._concept = c

    return AttachedModelEnum

#------------------------------------------------------
# Value
#------------------------------------------------------

Value = Variable|Primitive|Field|TupleVariable
"""Union of value types.

``Value`` covers the objects that can be used as operands in DSL expressions
(for example in relationship/property calls and in :class:`Expression`).
"""

#------------------------------------------------------
# Statement
#------------------------------------------------------

Statement = Value | Group | Not | Distinct | Aggregate
"""Union of statement types.

``Statement`` covers the objects accepted by most query-building methods (for
example :meth:`Model.where` and :meth:`Model.define`).
"""

StatementAndSchema = Statement | TableSchema
"""Union of statement and schema types.

``StatementAndSchema`` is accepted by ``select`` methods like
:meth:`Model.select` and :meth:`Fragment.select`.
"""

#------------------------------------------------------
# Fragment
#------------------------------------------------------

@include_in_docs
class Fragment(DerivedTable):
    """Represents a composable query or rule fragment.

    Fragments are returned by methods like :meth:`Model.where` and :meth:`Model.select`.
    You can chain :meth:`Fragment.where`, :meth:`Fragment.select`, :meth:`Fragment.define`,
    and :meth:`Fragment.require`, then materialize results with :meth:`Fragment.to_df`.

    Fragments are built lazily: chaining methods builds up a query, but nothing
    runs until you materialize or execute it. Calling ``to_df()`` compiles the
    fragment and executes it to produce a pandas DataFrame. To export results
    into a table, call :meth:`Fragment.into` to specify the destination and then
    call :meth:`Fragment.exec`.

    Notes
    -----
    Most users should not instantiate ``Fragment`` directly. Prefer starting
    from a :class:`Model` (or the convenience functions :func:`~relationalai.semantics.where`
    and :func:`~relationalai.semantics.select`).

    Parameters
    ----------
    model : Model
        The model this fragment belongs to.
    parent : Fragment, optional
        Internal: parent fragment used to accumulate chained clauses.

    Examples
    --------
    >>> from relationalai.semantics import Integer, Model
    >>> m = Model()
    >>> Person = m.Concept("Person",)
    >>> # A definition fragment that defines two people
    >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=30))
    >>> # A filter fragment that filters for people older than 21
    >>> q = m.where(Person.age > 21)
    >>> # A select fragment that extends the filter and selects the names of the filtered people
    >>> q.select(Person.name)
    >>> # Materialize the results as a DataFrame
    >>> q.to_df()
    """
    def __init__(self, model:Model, parent:Fragment|None=None):
        super().__init__(model)
        self._id = next(_global_id)
        self._select = []
        self._where = []
        self._require = []
        self._define = []
        self._procedures = []
        self._order_by = []
        self._limit = 0
        self._model = model
        self._into:Optional[Table] = None
        self._is_into_update = False
        self._has_executed = False
        assert self._model, "Fragment must have a model"

        self._parent = parent
        # self._source = runtime_env.get_source_pos()
        self._meta = {}
        self._annotations = []
        if parent is not None:
            self._select.extend(parent._select)
            self._where.extend(parent._where)
            self._require.extend(parent._require)
            self._define.extend(parent._define)
            self._order_by.extend(parent._order_by)
            self._limit = parent._limit


    def _add_items(self, items:Sequence[Statement], to_attr:list[Statement]):
        # TODO: ensure that you are _either_ a select, require, or then
        # not a mix of them
        model = self._model

        # remove any existing rules that this consumes
        for item in itertools.chain(items, [self._parent]):
            if isinstance(item, Fragment) and item._is_effect():
                model._remove_rule(item)

        to_attr.extend(items)
        if self._is_effect():
            model._add_rule(self)

        return self

    #------------------------------------------------------
    # Select arg handling
    #------------------------------------------------------

    def _check_select_args(self, args:Sequence[StatementAndSchema]) -> Sequence[Statement]:
        clean_args = []
        for arg in args:
            if isinstance(arg, Fragment) and len(arg._where) > 0 and len(arg._select) == 0:
                exc("Invalid Expression",
                    "Cannot use a `where` expression without a `select` expression, in this context",
                    [source(self)])
            # If you select x > y, treat that as AsBool(x > y)
            if isinstance(arg, Expression) and not arg._has_output:
                clean_args.append(AsBool(arg))
            elif isinstance(arg, TableSchema):
                for col in arg.get_columns():
                    clean_args.append(col(arg._table))
            else:
                clean_args.append(arg)
        return clean_args

    #------------------------------------------------------
    # Core API
    #------------------------------------------------------

    @include_in_docs
    def where(self, *args: Statement) -> Fragment:
        """Return a new fragment with additional filter conditions.

        Unlike :meth:`Model.where`, this extends an existing :class:`Fragment`.
        Multiple conditions are combined with AND.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to use as filter conditions.

        Returns
        -------
        Fragment
            A composable fragment representing the combined filter(s).
        """
        f = Fragment(self._model, parent=self)
        return f._add_items(args, f._where)

    @include_in_docs
    def select(self, *args: StatementAndSchema) -> Fragment:
        """Return a new fragment with selected outputs.

        Unlike :meth:`Model.select`, this extends an existing :class:`Fragment`.
        Passing a :class:`TableSchema` expands to selecting each column of the table.
        Selecting a comparison expression like ``x > y`` produces a boolean output column.

        Parameters
        ----------
        *args : StatementAndSchema
            Items to select. Each item may be a
            :data:`Statement` or a
            :class:`TableSchema`.

        Returns
        -------
        Fragment
            A composable fragment representing the combined selection.
        """
        f = Fragment(self._model, parent=self)
        return f._add_items(self._check_select_args(args), f._select)

    @include_in_docs
    def require(self, *args: Statement) -> Fragment:
        """Return a new fragment with additional requirements.

        Unlike :meth:`Model.require`, this extends an existing :class:`Fragment`.
        A requirement is an existence check (it fails when there are *no* matches).
        Scope a requirement with :meth:`where` to express per-entity constraints.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to enforce as requirements.

        Returns
        -------
        Fragment
            A composable fragment representing the combined requirement(s).
        """
        f = Fragment(self._model, parent=self)
        return f._add_items(args, f._require)

    @include_in_docs
    def define(self, *args: Statement) -> Fragment:
        """Return a new fragment with additional definitions.

        Unlike :meth:`Model.define`, this extends an existing :class:`Fragment`.
        It is most commonly used to write conditional definitions by chaining
        ``where(...).define(...)``.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to define.

        Returns
        -------
        Fragment
            A composable fragment representing the combined definition(s).
        """
        f = Fragment(self._model, parent=self)
        for arg in args:
            if isinstance(arg, Fragment) and len(arg._where) > 0 and len(arg._select) == 0:
                exc("Invalid Expression",
                    "Cannot use a `where` expression without a `select` expression, in this context",
                    [source(self)])
        return f._add_items(args, f._define)

    if not TYPE_CHECKING:
        def order_by(self, *args: Any) -> Fragment:
            exc("Feature unavailable", "The 'order_by' method is not yet available when querying RAI directly.", [
                source(self),
                "You can use [cyan]`relationalai.semantics.std.aggregates.rank`[/cyan] and select it as the first column as a temporary substitute.",
            ])
            f = Fragment(self._model, parent=self)
            return f._add_items(args, f._order_by)

        def limit(self, n:int) -> Fragment:
            exc("Feature unavailable", "The 'order_by' method is not yet available when querying RAI directly.", [
                source(self),
                "You can use [cyan]`relationalai.semantics.std.aggregates.limit`[/cyan] in a where clause as a temporary substitute.",
            ])
            f = Fragment(self._model, parent=self)
            f._limit = n
            return f

    @include_in_docs
    def meta(self, **kwargs: Any) -> Fragment:
        """Attach execution metadata to this fragment.

        The provided keyword arguments are forwarded as ``meta=...`` when the
        fragment is executed (for example via :meth:`Fragment.exec` or :meth:`Fragment.to_df`).
        This does not change the query's semantics; it only annotates the
        execution request. This is useful for tagging runs with context like a
        request id, job name, or other tracing/debugging information.

        Parameters
        ----------
        **kwargs
            Metadata key/value pairs to attach.

        Returns
        -------
        Fragment
            This fragment, to allow chaining.
        """
        self._meta.update(kwargs)
        return self

    @include_in_docs
    def annotate(self, *annos:Expression|Relationship) -> Fragment:
        """Attach one or more annotations to this fragment.

        An annotation is recorded on the fragment and emitted into the compiled
        query or rule. Annotations are primarily used by backends and internal
        tooling for debugging and support workflows (for example, to attach
        metadata such as tracking labels). In general you should not need to
        use annotations unless RelationalAI support instructs you to add them
        for diagnostic purposes.

        Parameters
        ----------
        *annos : Expression | Relationship
            One or more annotation nodes to attach.

        Returns
        -------
        Fragment
            This fragment (returned to enable fluent chaining).

        Raises
        ------
        relationalai.util.error.RAIException
            Raised when an annotation argument has an unsupported type.
        """
        self._annotations.extend(annos)
        return self

    #------------------------------------------------------
    # Loopy Procedure API
    #------------------------------------------------------

    def _procedure(self, algo: "ProcedureBuilder") -> Fragment:
        f = Fragment(self._model, parent=self)
        f._procedures.append(algo)
        if f._is_effect():
            self._model._add_rule(f)
        return f

    #------------------------------------------------------
    # into
    #------------------------------------------------------

    @include_in_docs
    def into(self, table: Table, update=False) -> Fragment:
        """Return a new fragment configured to export results into a table.

        Call this on a query fragment (typically after ``select(...)``) to set
        the export destination, then execute it with :meth:`Fragment.exec`.

        Parameters
        ----------
        table : Table
            Destination table reference.
        update : bool, optional
            If True, perform an update export (backend-dependent) rather than
            replacing the destination.

        Returns
        -------
        Fragment
            A new fragment that will export to ``table`` when executed.
        """
        f = Fragment(self._model, parent=self)
        f._into = table
        f._is_into_update = update
        self._model.exports.add(f)
        self._model._bump_version()
        return f

    #------------------------------------------------------
    # Execution
    #------------------------------------------------------

    @include_in_docs
    def exec(self):
        """Execute this fragment and export results into the table set by
        :meth:`Fragment.into`.

        Calling ``exec()`` runs the query once; subsequent calls are a no-op.
        Use :meth:`Fragment.to_df` to execute without exporting.

        Returns
        -------
        pandas.DataFrame | None
            The query results as a DataFrame, or None if the fragment was
            already executed.

        Raises
        ------
        relationalai.util.error.RAIException
            If no destination table was specified via :meth:`Fragment.into`.

        Notes
        -----
        Exporting query results does not avoid materializing results in Python,
        since the executor returns the results as a pandas DataFrame. This can
        have performance implications, especially for large result sets.
        """
        if self._has_executed:
            return
        if self._into is None:
            exc("Cannot execute", "Query must have an 'into' table specified to execute.", [source(self)])
        self._has_executed = True
        return self._model._get_executor().execute_query(self._model, self, export_to=self._into._name, update=self._is_into_update, meta=self._meta)

    @include_in_docs
    def to_df(self):
        """Materialize this fragment as a pandas DataFrame.

        Calling ``to_df()`` compiles the fragment and executes it, returning the
        query results as a DataFrame.

        Returns
        -------
        pandas.DataFrame
            The query results.

        Notes
        -----
        ``to_df`` executes the query and materializes the full result in memory.
        This can have performance implications, especially for large result sets.
        """
        return self._model._get_executor().execute_query(self._model, self, meta=self._meta)

    def inspect(self, end:str='\n'):
        print(self.to_df())
        print(end=end)

    #------------------------------------------------------
    # helpers
    #------------------------------------------------------

    def _is_effect(self) -> bool:
        return bool(self._define or self._require or self._procedures)

    def _is_where_only(self) -> bool:
        return not self._select and not self._define and not self._require and not self._order_by

    #------------------------------------------------------
    # And/Or
    #------------------------------------------------------

    @include_in_docs
    def __or__(self, other) -> Match:
        """Return a match expression (``self | other``).

        Use ``|`` to express fallback/default behavior between two query
        branches (for example, a selected value or a default literal). The
        returned :class:`Match` evaluates branches left-to-right and picks the
        first branch that can succeed for the current bindings.

        This is commonly used with fragments to provide defaults for optional
        subqueries that may have no match.

        Parameters
        ----------
        other : Statement
            The right-hand branch.

        Returns
        -------
        Match
            A composable match expression.

        Examples
        --------
        Use a default when an optional lookup has no match:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> User = m.Concept("User")
        >>> m.define(
        ...     User.new(id=1, name="Alice", status="active"),
        ...     User.new(id=2, name="Bob", status="inactive"),
        ...     User.new(id=3, name="Carol"),
        ... )
        >>> status = m.select(User.status) | "unknown"
        >>> m.where(User.id.in_([2, 3])).select(User.name, status).to_df()
        """
        return Match(self._model, self, other)

    @include_in_docs
    def __and__(self, other) -> Fragment:
        """Combine two filter fragments with AND (``self & other``).

        This operator is only supported for *where-only* fragments (fragments
        that contain only ``where(...)`` clauses). If ``other`` is not already a
        :class:`Fragment`, it is treated as a single filter condition and wrapped
        as ``Fragment(self._model).where(other)``.

        Parameters
        ----------
        other : Fragment | Statement
            A where-only fragment or a single filter condition.

        Returns
        -------
        Fragment
            A new where-only fragment with the combined conditions.

        Raises
        ------
        Exception
            Raised when either side is not a where-only fragment.

        Examples
        --------
        Combine an existing filter fragment with an additional condition:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> m.define(
        ...     Person.new(name="Alice", age=10),
        ...     Person.new(name="Bob", age=30)
        ... )
        >>> people_over_40 = m.where(Person.age > 40)
        >>> people_named_bob = m.where(Person.name == "Bob")
        >>> q = people_over_40 & people_named_bob
        >>> q.select(Person.name).to_df()
        """
        if not isinstance(other, Fragment):
            other = Fragment(self._model).where(other)
        if self._is_where_only() and other._is_where_only():
            return self.where(*other._where)
        elif self._is_where_only():
            return other.where(*self._where)
        elif other._is_where_only():
            return self.where(*other._where)
        else:
            raise Exception("Cannot AND two non-where-only fragments")

    #------------------------------------------------------
    # DerivedTable cols
    #------------------------------------------------------

    def _get_cols(self):
        return [DerivedColumn.from_value(self, i, col) for i, col in enumerate(self._select)]

    #------------------------------------------------------
    # Marterialize
    #------------------------------------------------------

    def to_metamodel(self):
        m = self._model
        return m._compiler.compile(self)

#------------------------------------------------------
# Model
#------------------------------------------------------

@include_in_docs
class Model:
    """Create and manage a semantic model.

    A :class:`Model` stores the concepts, relationships, definitions, and
    requirements that make up your semantic model. Create them using methods
    like :meth:`Model.Concept`, :meth:`Model.Relationship`, :meth:`Model.define`,
    and :meth:`Model.require`. Query methods like :meth:`Model.where` and
    :meth:`Model.select` return a composable :class:`Fragment` that you can further
    refine and materialize with :meth:`Fragment.to_df`.

    Many methods on :class:`Model` can be accessed as top-level functions from
    the :mod:`relationalai.semantics` module. For example, you can use
    :func:`~relationalai.semantics.define` instead of :meth:`Model.define` to
    add definitions to a model if you have a single active model (the common
    case in interactive use). However, if you have multiple models or want to
    be explicit about which model you're using, it's best to call methods
    directly on the model instance.

    Parameters
    ----------
    name : str, optional
        Name of the model.
    exclude_core : bool, optional
        If True, do not include the core library by default.
    is_library : bool, optional
        If True, this model is treated as a library and is not added to the
        global list of active models.
    config : relationalai.config.config.Config, optional
        Configuration used when executing queries. Defaults to `None` which
        means configuration is loaded from the filesystem (for example,
        from a ``raconfig.yaml`` file if present).

    Attributes
    ----------
    all_models : list[Model]
        Class-level list of models created in this Python session (excluding
        library-only models). Most users do not need this.
    name : str
        The model name (optional; used for identification/debugging).
    config : relationalai.config.config.Config
        Configuration settings used when executing queries against this model.
        If you have a ``raconfig.yaml`` file, or other valid configuration source,
        these settings are automatically loaded into new models by default.
        You can also override them by passing a custom config when creating the model.
        See the :mod:`relationalai.config` documentation for more details.
    defines
        Definitions added via :meth:`Model.define`.
    requires
        Requirements added via :meth:`Model.require`.
    exports
        Export artifacts created when exporting query results (advanced).
    libraries : list[type[Model]]
        Libraries included with this model (the core library is included by
        default unless you set ``exclude_core=True``).
    concepts : list[Concept]
        Concepts created in this model via :meth:`Model.Concept`.
    tables : list[Table]
        Table references created in this model via :meth:`Model.Table`.
    relationships : list[Relationship]
        Relationships and properties created in this model via :meth:`Model.Relationship`.
    enums : list[type[ModelEnum]]
        Enum types created in this model via :meth:`Model.Enum`.
    concept_index : dict[str, Concept]
        Dictionary mapping concept names to :class:`Concept` objects for quick lookup.
    table_index : dict[str, Table]
        Dictionary mapping table names to :class:`Table` objects for quick lookup.
    relationship_index : dict[str, Relationship]
        Dictionary mapping relationship/property names to :class:`Relationship` objects for quick lookup.
    enums_index : dict[str, type[ModelEnum]]
        Dictionary mapping enum names to enum types for quick lookup.
    Enum : type
        Helper used to create enums associated with this model.

    Examples
    --------
    Define a few entities and query them:

    >>> from relationalai.semantics import Model
    >>> m = Model()
    >>> Person = m.Concept("Person")
    >>> m.define(
    ...     Person.new(name="Alice", age=10),
    ...     Person.new(name="Bob", age=30),
    ... )
    >>> m.where(Person.age > 18).select(Person.name).to_df()
    """
    all_models:list[Model] = []

    def __init__(self, name: str = "", exclude_core: bool = False, is_library: bool = False, config:Config|None=None):
        self.name = name
        self.config: Config = config if config is not None else create_config()
        self.defines:KeyedSet[Fragment] = KeyedSet(dsl_key)
        self._procedures: KeyedSet[Fragment] = KeyedSet(dsl_key)
        self.requires:KeyedSet[Fragment] = KeyedSet(dsl_key)
        self.exports:KeyedSet[Fragment] = KeyedSet(dsl_key)
        self.libraries = []
        self.concepts: list[Concept] = []
        self.tables: list[Table] = []
        self.relationships: list[Relationship] = []
        self.enums: list[Type[ModelEnum]] = []
        self.concept_index: dict[str, Concept] = {}
        self.table_index: dict[str, Table] = {}
        self.relationship_index: dict[str, Relationship] = {}
        self.enums_index: dict[str, Type[ModelEnum]] = {}
        self._source = SourcePos.new()

        # Version tracking for caching
        self._version: int = 0
        # Lazy executor to run queries
        self._executor: Optional["Executor"] = None
        self.Enum = create_enum_class(self)

        if not is_library:
            Model.all_models.append(self)

        self.__compiler = None
        if not exclude_core:
            self.libraries.append(CoreLibrary)

    #------------------------------------------------------
    # Internal
    #------------------------------------------------------

    @property
    def _compiler(self):
        if not self.__compiler:
            from .front_compiler import FrontCompiler
            self.__compiler = FrontCompiler(self)
        return self.__compiler

    def _find_concept(self, name: str) -> Concept|None:
        if name in self.concept_index:
            return self.concept_index[name]
        if name in self.table_index:
            return self.table_index[name]
        for lib in self.libraries:
            found = lib._find_concept(name)
            if found is not None:
                return found

    def _remove_rule(self, fragment: Fragment) -> None:
        if fragment._define and fragment in self.defines:
            self.defines.remove(fragment)
            self._bump_version()
        elif fragment._require and fragment in self.requires:
            self.requires.remove(fragment)
            self._bump_version()
        elif fragment._procedures and fragment in self._procedures:
            self._procedures.remove(fragment)
        else:
            raise ValueError("Fragment must have either define, require, or procedure clauses to be removed")

    def _add_rule(self, fragment: Fragment) -> None:
        if fragment._define and fragment not in self.defines:
            self.defines.add(fragment)
            self._bump_version()
        elif fragment._require and fragment not in self.requires:
            self.requires.add(fragment)
            self._bump_version()
        elif fragment._procedures and fragment not in self._procedures:
            self._procedures.add(fragment)
        else:
            raise ValueError("Fragment must have either define, require, or procedure clauses to be added as a rule")

    def _bump_version(self) -> None:
        """Increment version on any model mutation."""
        self._version += 1

    def _get_executor(self) -> "Executor":
        if not self._executor:
            if self.config.install_mode:
                from relationalai.semantics.backends.sql_executor import SQLExecutor
                self._executor = SQLExecutor(self.config)
            elif self.config.reasoners.logic.use_lqp:
                from relationalai.semantics.backends.lqp.executor import LQPExecutor
                self._executor = LQPExecutor(self)
            else:
                from relationalai.semantics.backends.legacy_executor import LegacyExecutor
                self._executor = LegacyExecutor(self)
        return self._executor

    def exec(self):
        """Execute all export fragments registered on this model as a single batch transaction.

        Collects all fragments added via :meth:`Fragment.into` and submits them
        together. The backend may execute them in parallel.

        Examples
        --------
        >>> m = Model("SalesData")
        >>> Customer = m.Concept("Customer")
        >>> Product = m.Concept("Product")
        >>> m.select(Customer.id, Customer.name).into(customer_table)
        >>> m.select(Product.id, Product.name).into(product_table)
        >>> m.exec()
        """
        from relationalai.shims.executor import exec_batch
        exec_batch(list(self.exports))

    #------------------------------------------------------
    # Primary API
    #------------------------------------------------------

    @include_in_docs
    def Concept(self, name: str, extends: list[Concept] = [], identify_by: dict[str, Property|Concept] = {}, identity_includes_type: bool = False) -> Concept:
        """Create a concept type in this model.

        This is the primary way to introduce a new :class:`Concept` (entity type)
        into a :class:`Model`.

        Parameters
        ----------
        name : str
            Name of the concept.
        extends : list[Concept], optional
            Base concepts this concept extends. Relationships declared on base
            concepts are inherited by (and available on) the derived concept.
        identify_by : dict[str, Property | Concept], optional
            Identity fields for this concept. The keys are identity attribute
            names and the values specify their types (most commonly a :class:`Concept`
            such as :data:`~relationalai.semantics.Integer` or :data:`~relationalai.semantics.String`).

            Identity keys are used by :meth:`Concept.new`
            to refer to an entity by key (find-or-create).
        identity_includes_type : bool, optional
            If True, the subtype is part of an entity's key.
            Example: when `A` and `B` share a base identified by `id`,
            `A.new(id=1)` and `B.new(id=1)` create two different entities.
            If False, both calls refer to the same entity (same key fields),
            which can cause collisions across a type hierarchy.

        Returns
        -------
        Concept
            The newly created concept.

        Raises
        ------
        relationalai.util.error.RAIException
            If `extends` contains non-Concept values, or if `identify_by`
            contains values that are not Concepts or Properties.

        Examples
        --------
        Create a concept and define a couple of entities:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> m.define(
        ...     Person.new(name="Alice"),
        ...     Person.new(name="Bob"),
        ... )

        Create a derived concept:

        >>> Adult = m.Concept("Adult", extends=[Person])
        >>> m.where(Person.age >= 18).define(Adult(Person))

        Define an identity key:

        >>> from relationalai.semantics import Integer
        >>> Item = m.Concept("Item", identify_by={"id": Integer})
        >>> m.define(Item.new(id=1))
        """
        c = Concept(name, extends=extends, identify_by=identify_by, identity_includes_type=identity_includes_type, model=self)
        self.concepts.append(c)
        self.concept_index[name] = c
        self._bump_version()
        return c

    @include_in_docs
    def Table(self, path: str, schema: dict[str, Concept] = {}) -> Table:
        """Create a table reference in this model.

        A :class:`Table` represents an external table name (such as a SQL table)
        that you can use as a source of schema/data for defining entities, or as
        a target for exports via :meth:`Fragment.into`.

        Parameters
        ----------
        path : str
            Table name or path (for example `"MY_DB.MY_SCHEMA.CUSTOMERS"`).
        schema : dict[str, Concept], optional
            Optional mapping of column names to their types.

        Returns
        -------
        Table
            The created table reference.

        Examples
        --------
        Create a table reference for use as an export target:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"id": Integer})
        >>> Person.name = m.Property(f"{Person} is named {String:name}")
        >>> m.define(Person.new(id=1, name="Alice"))
        >>> out = m.Table("DB.SCHEMA.PERSONS_EXPORT")
        >>> m.select(Person.id, Person.name).into(out).exec()

        Use a table with a known schema as an input when defining entities:

        >>> from relationalai.semantics import Model, Integer, String
        >>> m = Model()
        >>> source = m.Table("DB.SCHEMA.CUSTOMERS", schema={"id": Integer, "name": String})
        >>> Customer = m.Concept("Customer", extends=[Person])
        >>> m.define(Customer.new(source.to_schema()))
        """
        t = Table(path, schema=schema, model=self)
        self.tables.append(t)
        self.table_index[path] = t
        self._bump_version()
        return t

    @include_in_docs
    def Relationship(self, reading: str = "", fields: list[Field] = [], short_name: str = "") -> Relationship:
        """Create a relationship in this model.

        This is the primary way to introduce a new :class:`Relationship` type into a
        :class:`Model`. The most common way to define a relationship is with a Python
        f-string ``reading`` that embeds concept/type placeholders. Each embedded
        placeholder becomes a field of the relationship, in order.

        Reading strings have the following format:

        - Use an f-string and embed types with ``{...}``, e.g. ``f"{Person} has {Pet}"``.
        - Name fields with ``{Type:name}``, e.g. ``f"{Person:owner} has {Pet:pet}"``.
        - For full control (including multi-field relationships), pass an
            explicit ``fields=[Field(...), ...]`` list instead of a reading.

        Parameters
        ----------
        reading : str, optional
            A reading that describes the relationship fields.
            Typically an f-string, for example ``f"{Person} has {Pet}"`` or
            ``f"{Integer:i} has {String:name}"``.
        fields : list[Field], optional
            Explicit fields for the relationship. If provided and ``reading`` is
            empty, a default reading is generated from the fields.
        short_name : str, optional
            Optional identifier used as the key in ``model.relationship_index``.

        Returns
        -------
        Relationship
            The created relationship.

        Raises
        ------
        ValueError
            If neither ``reading`` nor ``fields`` is provided.

        Examples
        --------
        Create a relationship between two concepts and define one fact:

        >>> from relationalai.semantics import Model, String, define, where
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Pet = m.Concept("Pet", identify_by={"name": String})
        >>> Person.name = m.Property(f"{Person} has {String:name}")
        >>> Person.pets = m.Relationship(f"{Person} has {Pet}")
        >>> define(
        ...     alice := Person.new(name="Alice"),
        ...     boots := Pet.new(),
        ...     alice.pets(boots),
        ... )
        >>> where(Person.pets).select(Person.name).to_df()

        Create a relationship by specifying fields directly and give it a
        stable name via ``short_name``:

        >>> from relationalai.semantics import Model, String, define
        >>> from relationalai.semantics.frontend.base import Field
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> has_pet = m.Relationship(
        ...     fields=[Field("person", Person), Field("pet_name", String)],
        ...     short_name="has_pet",
        ... )
        >>> define(alice := Person.new(name="Alice"), has_pet(alice, "boots"))
        >>> has_pet.to_df()
        """
        r = Relationship(self, reading, fields, short_name)
        self.relationships.append(r)
        self.relationship_index[short_name] = r
        self._bump_version()
        return r

    @include_in_docs
    def Property(self, reading: str = "", fields: list[Field] = [], short_name: str = "") -> Property:
        """Create a property in this model.

        This is the primary way to introduce a new :class:`Property` into a
        :class:`Model`. The most common way to define a property is with a Python
        f-string ``reading`` that embeds concept/type placeholders. Each embedded
        placeholder becomes a field of the property, in order.

        Reading strings have the following format:

        - Use an f-string and embed types with ``{...}``, e.g. ``f"{Person} has {Integer:age}"``.
        - Name fields with ``{Type:name}`` (the ``name`` becomes the attribute
            you access, e.g. ``Person.age``).
        - For full control, pass an explicit ``fields=[Field(...), ...]`` list.

        Parameters
        ----------
        reading : str, optional
            A reading that describes the property fields.
            Typically an f-string, for example
            ``f"{Person} has {Integer:age}"``.
        fields : list[Field], optional
            Explicit fields for the property. If provided and ``reading`` is
            empty, a default reading is generated from the fields.
        short_name : str, optional
            Optional identifier used as the key in ``model.relationship_index``.

        Returns
        -------
        Property
            The created property.

        Raises
        ------
        ValueError
            If neither ``reading`` nor ``fields`` is provided.

        Examples
        --------
        Declare a property on a concept, define a couple of entities, and query:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=30))
        >>> m.where(Person.age >= 18).select(Person.age).to_df()

        Create a property by specifying fields directly and give it a stable
        name via ``short_name``:

        >>> from relationalai.semantics import Integer, Model, String
        >>> from relationalai.semantics.frontend.base import Field
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> age = m.Property(
        ...     fields=[Field("person", Person), Field("age", Integer)],
        ...     short_name="age",
        ... )
        >>> Person.age = age
        """
        p = Property(self, reading, fields, short_name)
        self.relationships.append(p)
        self.relationship_index[short_name] = p
        self._bump_version()
        return p

    @include_in_docs
    def select(self, *args: StatementAndSchema) -> Fragment:
        """Return a selection fragment in this model.

        This starts a new :class:`Fragment` with a ``select`` clause. The
        returned fragment is composable and can be chained with
        :meth:`Fragment.where`, :meth:`Fragment.define`, and
        :meth:`Fragment.require`, then materialized via :meth:`Fragment.to_df`.

        You can select most values and expressions, including:

        - Concepts, relationships/properties, and fields
        - Variables and expressions (including arithmetic and comparisons)
        - Aggregates and grouped expressions (for example ``per(x).sum(y)``)
        - Python literals like strings, integers, floats, and booleans

        Passing a :class:`TableSchema`
        (typically created via :meth:`Table.to_schema`)
        expands to selecting each column of the table. Selecting a comparison
        expression like ``Person.age >= 18`` produces a boolean output column.

        Parameters
        ----------
        *args : StatementAndSchema
            Items to select. Each item may be a
            :data:`Statement` or a
            :class:`TableSchema`.

        Returns
        -------
        Fragment
            A composable query fragment representing the selection.

        Examples
        --------
        Select a value and a computed boolean column:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=30))
        >>> m.select(Person.name, Person.age >= 18).to_df()
        """
        f = Fragment(self)
        return f.select(*args)

    @include_in_docs
    def where(self, *args: Statement) -> Fragment:
        """Return a fragment filtered by the given conditions.

        This starts a new :class:`Fragment` with a ``where`` clause. The
        returned fragment is composable and can be chained with
        :meth:`Fragment.select`, :meth:`Fragment.define`, and
        :meth:`Fragment.require`, then materialized via :meth:`Fragment.to_df`.

        Multiple conditions passed in a single call are combined with AND. You
        can add additional filters by chaining
        :meth:`Fragment.where`.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to use as filter conditions.

        Returns
        -------
        Fragment
            A composable query fragment representing the filter(s).

        Examples
        --------
        Filter and select:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=30))
        >>> m.where(Person.age > 21).select(Person.name).to_df()
        """
        f = Fragment(self)
        return f.where(*args)

    @include_in_docs
    def require(self, *args: Statement) -> Fragment:
        """Return a fragment that adds requirements (constraints).

        This starts a new :class:`Fragment` with a ``require`` clause. The
        returned fragment is composable and can be chained with
        :meth:`Fragment.where` to make a requirement conditional, or with
        :meth:`Fragment.require` to add additional requirements.

        A requirement is an existence check: it fails when there are *no* matches.
        Roughly speaking, ``m.require(expr)`` asserts ``EXISTS(expr)`` and fails
        when ``NOT EXISTS(expr)``.

        For example, ``m.require(Person.age > 0)`` fails in the following cases:

        - There are no ``Person`` entities.
        - There are ``Person`` entities, but none have an ``age`` value.
        - There are ``age`` values, but none are greater than zero.

        It does *not* fail just because some ``Person`` entities are missing an
        ``age`` value, or because some ages are non-positive, as long as there
        is at least one match for ``Person.age > 0``.

        To enforce a per-entity requirement, scope it with
        :meth:`Model.where` (for example,
        ``where(Person).require(Person.age > 0)``) or use :meth:`Concept.require`
        (for example, ``Person.require(Person.age > 0)``). Both of those forms are
        equivalent and the choice is mostly a matter of style.

        Multiple requirements passed in a single call are enforced together.
        You can also apply requirements to an existing fragment by calling
        :meth:`Fragment.require`.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to enforce as requirements.

        Returns
        -------
        Fragment
            A composable fragment representing the requirement(s).

        Notes
        -----
        Requirements can affect performance:

        - Adding more requirements generally increases compilation and execution
          work, especially if a requirement introduces large joins, broad
          disjunctions (for example via
          :meth:`Model.union`), or
          aggregates over large domains.
        - A well-chosen requirement can also improve performance by narrowing
          what needs to be considered (for example by reducing the number
          of matches that flow into later joins or aggregates).
        - Prefer scoped requirements (for example ``m.require(...).where(...)``)
          when the constraint should only apply under certain conditions.

        Examples
        --------
        Require that at least one match exists:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=1), Person.new(name="Bob", age=2))
        >>> # Passes because there exists a Person with age >= 0.
        >>> m.require(Person.age >= 0)

        Scope a requirement to enforce it per entity:

        >>> # Requires every Person to have an age >= 0.
        >>> m.where(Person).require(Person.age >= 0)
        """
        f = Fragment(self)
        return f.require(*args)

    @include_in_docs
    def define(self, *args: Statement) -> Fragment:
        """Return a fragment that adds definitions (facts or logic).

        This starts a new :class:`Fragment` with a ``define`` clause. The
        returned fragment is composable and can be chained with
        :meth:`Fragment.where` to make definitions conditional, or with
        :meth:`Fragment.define` to add additional definitions.

        Calling :meth:`Model.define` also
        registers the resulting definition fragment with the model, so the
        definitions are included when the model is compiled and evaluated by
        later queries.

        Parameters
        ----------
        *args : Statement
            One or more :data:`Statement`
            items to define. Common examples include entity creations
            (``Person.new(...)``) and relationship/property facts.

        Returns
        -------
        Fragment
            A composable fragment representing the definition(s).

        Examples
        --------
        Define some base facts:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=30))

        Define a derived concept under a condition:

        >>> Adult = m.Concept("Adult", extends=[Person])
        >>> m.where(Person.age >= 18).define(Adult(Person))
        """
        f = Fragment(self)
        return f.define(*args)

    @include_in_docs
    def union(self, *items: Value) -> Union:
        """Combine items with a logical OR or a set-style union.

        This constructs a :class:`Union` object that represents a disjunction over multiple *branches*.

        Unlike the ``|`` operator (see :meth:`Variable.__or__`), which picks the
        first branch that can succeed, ``union`` combines the results from *all* branches.

        `union` is commonly used in two ways:

        - As an OR-combinator in :meth:`Model.where`
          clauses, for example ``m.where(m.union(cond1, cond2))``.
        - As a union of multiple derived tables/fragments that return the same
          number of values. The resulting union behaves like a derived table
          whose columns can be selected or unpacked.

        Nested unions are flattened, so ``m.union(m.union(a, b), c)`` is
        equivalent to ``m.union(a, b, c)``.

        Parameters
        ----------
        *items : Value
            One or more :data:`Value`
            items to union. Each branch must return the same number of values.
            Branches used purely as filters (for example comparisons, or
            :class:`Not`) contribute
            zero output values and are intended to be used as conditions in a
            ``where`` clause.

        Returns
        -------
        Union
            A composable union object.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised if the union branches return different numbers of values.

        Examples
        --------
        Use `union` to OR conditions in a ``where`` clause:

        >>> from relationalai.semantics import Integer, Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Person.age = m.Property(f"{Person} has {Integer:age}")
        >>> m.define(Person.new(name="Alice", age=10), Person.new(name="Bob", age=70))
        >>> m.where(m.union(Person.age < 18, Person.age >= 65)).select(Person.name).to_df()

        Union two fragments that select the same number of values:

        >>> Edge = m.Concept("Edge")
        >>> Edge.src = m.Property(f"{Edge} has {Integer:src}")
        >>> Edge.dst = m.Property(f"{Edge} has {Integer:dst}")
        >>> m.define(Edge.new(src=1, dst=2), Edge.new(src=4, dst=3))
        >>> e, src, dst = m.union(
        ...     m.where(Edge, Edge.src <= Edge.dst).select(Edge, Edge.src, Edge.dst),
        ...     m.where(Edge, Edge.src > Edge.dst).select(Edge, Edge.dst, Edge.src),
        ... )
        >>> m.select(src, dst).to_df()
        """
        return Union(self, *items)

    @include_in_docs
    def data(self, data: DataFrame | list[tuple] | list[dict], columns: list[str]|None = None, schema: dict[str, Concept]|None = None) -> Data:
        """Create a temporary table from in-memory Python or pandas data.

        This converts the provided data into a pandas `DataFrame` and wraps it
        as a :class:`Data` object. The resulting object behaves like a table in
        the semantics DSL: you can refer to its columns in :meth:`Model.select`,
        :meth:`Model.where`, and :meth:`Model.define`, or convert it to a schema
        via :meth:`Table.to_schema`.

        This method is the underlying implementation used by the convenience
        wrapper :func:`~relationalai.semantics.data`.

        Parameters
        ----------
        data : pandas.DataFrame | list[tuple] | list[dict]
            The input data. Supported forms are:

            - A pandas `DataFrame`
            - A list of tuples (each tuple is a row)
            - A list of dicts (each dict is a row)

        columns : list[str], optional
            Column names to use when `data` is a list of tuples.
            If omitted, the data will use default integer column labels `0`, `1`,
            `2`, ... . Those default labels are exposed as `col0`, `col1`, `col2`,
            ... so you can write ``d.col0``, ``d.col1``, etc.

        schema : dict[str, Concept], optional
            Explicit column type overrides. Keys are column names and values are
            :class:`Concept` types (for example ``Integer`` or ``String``). When
            provided, the type for each named column is taken from this mapping
            instead of being inferred from the data. Columns not present in the
            mapping are still inferred from the data as usual.

        Returns
        -------
        Data
            A table-like object backed by the provided data.

        Raises
        ------
        TypeError
            If `data` cannot be converted into a pandas `DataFrame`.

        Examples
        --------
        Create data from a list of dicts and use it directly:

        >>> from relationalai.semantics import Model
        >>> m = Model()
        >>> people_rows = m.data([
        ...     {"name": "Alice", "age": 10},
        ...     {"name": "Bob", "age": 30},
        ... ])
        >>> m.select(people_rows.name, people_rows.age).to_df()

        Provide column names when loading tuple rows:

        >>> measurements = m.data([(0, 72.5), (1, 71.9)], columns=["minute", "temperature"])
        >>> m.select(measurements.minute, measurements.temperature).to_df()
        """
        df = Data.raw_to_df(data, columns)
        return Data(df, self, schema=schema)

    @include_in_docs
    def not_(self, *items: Statement) -> Not:
        """Return a negated condition for use in query filters.

        Use :meth:`Model.not_` inside :meth:`Model.where` (or :meth:`Fragment.where`)
        to say “exclude anything that matches this pattern”. For example,
        ``m.not_(Person.pets)`` can be read as “people with no pets”.

        If you pass multiple statements, they are treated as a single grouped
        condition and the *whole group* is negated. In other words,
        ``m.not_(a, b)`` means “NOT (a AND b)”, not “(NOT a) AND (NOT b)”.
        Note that ``NOT (a AND b)`` is equivalent to ``(NOT a) OR (NOT b)``.

        Parameters
        ----------
        *items : Statement
            One or more :data:`Statement`
            items to negate (for example, comparisons, relationship/property
            patterns, or other filter statements).

        Returns
        -------
        Not
            A negation expression.

        Examples
        --------
        Filter to people who have no pets:

        >>> from relationalai.semantics import Model, String
        >>> m = Model()
        >>> Person = m.Concept("Person", identify_by={"name": String})
        >>> Pet = m.Concept("Pet", identify_by={"name": String})
        >>> Person.pets = m.Relationship(f"{Person} has {Pet}")
        >>> m.define(
        ...     alice := Person.new(name="Alice"),
        ...     bob := Person.new(name="Bob"),
        ...     boots := Pet.new(name="boots"),
        ...     bob.pets(boots),
        ... )
        >>> m.where(Person, m.not_(Person.pets)).select(Person.name).to_df()

        Exclude people who have a pet named "boots":

        >>> m.where(Person, m.not_(Person.pets.name == "boots")).select(Person.name).to_df()

        Contrast grouped vs separate negation:

        >>> # Exclude people people who have no pets and people who have a pet named "boots".
        >>> m.where(Person, m.not_(Person.pets, Person.pets.name == "boots"))
        >>> # Exclude only people who have no pets.
        >>> m.where(Person, m.not_(Person.pets))
        """
        return Not(*items, model=self)

    @include_in_docs
    def distinct(self, *items: Value) -> Distinct:
        """Mark values as distinct (unique) for use in :meth:`Model.select` and
        aggregates.

        This constructs a :class:`Distinct` wrapper tied to this model. Use it
        to request “unique values only” in two common situations:

        - As a SQL `SELECT DISTINCT`-style wrapper by passing it as the *only*
          argument to :meth:`Model.select`, e.g. ``m.select(m.distinct(x, y))``.
        - As an argument wrapper for aggregates to make them count/sum/etc. over
          unique values, e.g. ``count(m.distinct(Person.name))``.

        Parameters
        ----------
        *items : Value
            One or more :data:`Value` items
            to consider when removing duplicates.

        Returns
        -------
        Distinct
            An object you pass to :meth:`Model.select`
            or to aggregate functions to request “unique values only”.

        Raises
        ------
        relationalai.util.error.RAIException
            Raised when the returned object is used incorrectly (for example, not
            applied to the entire ``select(...)``, or not applied to all aggregate
            arguments).

        Examples
        --------
        Count unique names (like SQL ``COUNT(DISTINCT name)``):

        >>> from relationalai.semantics import Model, String, count
        >>> m = Model()
        >>> Person = m.Concept("Person")
        >>> Person.name = m.Property(f"{Person} has {String:name}")
        >>> m.define(
        ...     Person.new(name="Chris"),
        ...     Person.new(name="Chris"),
        ...     Person.new(name="Joe"),
        ... )
        >>> m.select(count(m.distinct(Person.name))).to_df()

        Select unique rows by wrapping all selected values:

        >>> name_counts = count(Person).per(Person.name)
        >>> m.select(m.distinct(Person.name, name_counts)).to_df()
        """
        return Distinct(*items, model=self)

    #------------------------------------------------------
    # Procedure (experimental)
    #------------------------------------------------------

    def _procedure(self, *globals: Relationship):
        from ..experimental.loopy.builder import procedure as _procedure
        return _procedure(*globals, model=self)

    def _procedure_fragment(self, algo: "ProcedureBuilder") -> Fragment:
        f = Fragment(self)
        return f._procedure(algo)

    #------------------------------------------------------
    # Meta
    #------------------------------------------------------

    def to_metamodel(self) -> mModel:
        return self._compiler.compile_model(self)

#------------------------------------------------------
# Library
#------------------------------------------------------

class Library(Model):
    def __init__(self, name: str, exclude_core: bool = False):
        super().__init__(name=name, exclude_core=exclude_core, is_library=True, config=NoOpConfig())

    def Type(self, name: str, super_types: list[Concept] = []) -> Concept:
        c = self.Concept(name, extends=super_types)
        self._register_builtin(c)
        return c

    def Relation(self, name: str, fields: list[Field], overloads: list[list[Concept]]|None = None, annotations: list[Any]|None = None) -> Relationship:
        r = Relationship(self, "", fields, name, allow_no_fields=True, overloads=overloads)
        self.relationships.append(r)
        self.relationship_index[name] = r
        self._register_builtin(r)
        # TODO - deal with annotations
        return r

    def _register_builtin(self, builtin: Concept|Relationship) -> None:
        """Register a builtin concept or relationship in the global builtins
        registry."""
        builtins.register(self.name, self._compiler.to_relation(builtin))


#------------------------------------------------------
# Core Library
#------------------------------------------------------

# Library is populated when relationalai.semantics.frontend.core is imported
CoreLibrary: Library = include_in_docs(Library("core", exclude_core=True))
CoreRelationships = CoreLibrary.relationship_index
CoreConcepts = CoreLibrary.concept_index


#------------------------------------------------------
# todo
#------------------------------------------------------
"""
x new/to_identity/filter_by
x capture all defines/requires
x identify_by
x figure out Table/DerivedTable
x data node
x in_
x error on invalid iteration/sum/min/max
x Aggregates
x fill in std.agg functions when Thiago's lib stuff lands
- Error?
x rank stuff
- validation checks for identify_by
- unique
- Enum?
"""
