"""Abstract base class for metadata storage backends."""

from __future__ import annotations

from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, Sequence
from contextlib import AbstractContextManager, contextmanager
from datetime import datetime, timezone
from types import TracebackType
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypeVar, cast, overload

import narwhals as nw
from narwhals.typing import Frame, FrameT, IntoFrame
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing_extensions import Self

from metaxy._decorators import public
from metaxy._utils import switch_implementation_to_polars
from metaxy.config import MetaxyConfig
from metaxy.metadata_store.exceptions import (
    FeatureNotFoundError,
    StoreNotOpenError,
    SystemDataNotFoundError,
    VersioningEngineMismatchError,
)
from metaxy.metadata_store.system.keys import METAXY_SYSTEM_KEY_PREFIX
from metaxy.metadata_store.types import AccessMode
from metaxy.metadata_store.utils import (
    empty_frame_like,
)
from metaxy.metadata_store.warnings import (
    MetaxyColumnMissingWarning,
    PolarsMaterializationWarning,
)
from metaxy.models.constants import (
    ALL_SYSTEM_COLUMNS,
    METAXY_CREATED_AT,
    METAXY_DATA_VERSION,
    METAXY_DATA_VERSION_BY_FIELD,
    METAXY_DELETED_AT,
    METAXY_FEATURE_VERSION,
    METAXY_MATERIALIZATION_ID,
    METAXY_PROJECT_VERSION,
    METAXY_PROVENANCE,
    METAXY_PROVENANCE_BY_FIELD,
    METAXY_UPDATED_AT,
)
from metaxy.models.feature import current_graph
from metaxy.models.plan import FeaturePlan
from metaxy.models.types import (
    CoercibleToFeatureKey,
    FeatureKey,
    ValidatedFeatureKeyAdapter,
)
from metaxy.versioning import VersioningEngine
from metaxy.versioning.polars import PolarsVersioningEngine
from metaxy.versioning.types import HashAlgorithm, Increment, LazyIncrement

if TYPE_CHECKING:
    pass


def _is_map_column(df: Frame, col_name: str) -> bool:
    """Check if a column in a narwhals frame is a Map type.

    Returns False immediately if enable_map_datatype is not set.
    Delegates to find_map_columns for backend-specific detection.
    """
    if not MetaxyConfig.get().enable_map_datatype:
        return False

    from metaxy._utils import find_map_columns

    return col_name in find_map_columns(df)


# TypeVar for config types - used for typing from_config method
MetadataStoreConfigT = TypeVar("MetadataStoreConfigT", bound="MetadataStoreConfig")


@public
class MetadataStoreConfig(BaseSettings):
    """Base configuration class for metadata stores.

    This class defines common configuration fields shared by all metadata store types.
    Store-specific config classes should inherit from this and add their own fields.

    Example:
        ```toml title="metaxy.toml"
        [stores.dev]
        type = "metaxy.ext.metadata_stores.duckdb.DuckDBMetadataStore"

        [stores.dev.config]
        database = "metadata.db"
        hash_algorithm = "md5"
        ```
    """

    model_config = SettingsConfigDict(frozen=True, extra="forbid")

    fallback_stores: list[str] = Field(
        default_factory=list,
        description="List of fallback store names to search when features are not found in the current store.",
    )

    hash_algorithm: HashAlgorithm | None = Field(
        default=None,
        description="Hash algorithm for versioning. If None, uses store's default.",
    )

    versioning_engine: Literal["auto", "native", "polars"] = Field(
        default="auto",
        description="Which versioning engine to use: 'auto' (prefer native), 'native', or 'polars'.",
    )


VersioningEngineOptions: TypeAlias = Literal["auto", "native", "polars"]

# Mapping of system columns to their expected Narwhals dtypes
# Used to cast Null-typed columns to correct types
# Note: Struct columns (METAXY_PROVENANCE_BY_FIELD, METAXY_DATA_VERSION_BY_FIELD) are not cast
_SYSTEM_COLUMN_DTYPES = {
    METAXY_PROVENANCE: nw.String,
    METAXY_FEATURE_VERSION: nw.String,
    METAXY_PROJECT_VERSION: nw.String,
    METAXY_DATA_VERSION: nw.String,
    METAXY_CREATED_AT: nw.Datetime(time_zone="UTC"),
    METAXY_UPDATED_AT: nw.Datetime(time_zone="UTC"),
    METAXY_DELETED_AT: nw.Datetime(time_zone="UTC"),
    METAXY_MATERIALIZATION_ID: nw.String,
}


def _cast_present_system_columns(
    df: nw.DataFrame[Any] | nw.LazyFrame[Any],
) -> nw.DataFrame[Any] | nw.LazyFrame[Any]:
    """Cast system columns with Null/Unknown dtype to their correct types.

    This handles edge cases where empty DataFrames or certain operations
    result in Null-typed columns (represented as nw.Unknown in Narwhals)
    that break downstream processing.

    Args:
        df: Narwhals DataFrame or LazyFrame

    Returns:
        DataFrame with system columns cast to correct types
    """
    schema = df.collect_schema()
    columns_to_cast = []

    for col_name, expected_dtype in _SYSTEM_COLUMN_DTYPES.items():
        if col_name in schema and schema[col_name] == nw.Unknown:
            columns_to_cast.append(nw.col(col_name).cast(expected_dtype))

    if columns_to_cast:
        df = df.with_columns(columns_to_cast)

    return df


@public
class MetadataStore(ABC):
    """
    Abstract base class for metadata storage backends.
    """

    # Subclasses can override this to disable auto_create_tables warning
    # Set to False for stores where table creation is not applicable
    _should_warn_auto_create_tables: bool = True

    # Subclasses must define the versioning engine class to use
    versioning_engine_cls: type[VersioningEngine]

    def __init__(
        self,
        *,
        name: str | None = None,
        hash_algorithm: HashAlgorithm | None = None,
        versioning_engine: VersioningEngineOptions = "auto",
        fallback_stores: Sequence[MetadataStore | str] | None = None,
        auto_create_tables: bool | None = None,
        materialization_id: str | None = None,
    ):
        """
        Initialize the metadata store.

        Args:
            name: Optional name for this store. For representation purposes only.
                Is typically included into `.display()`.
            hash_algorithm: Hash algorithm to use for the versioning engine.
            versioning_engine: Which versioning engine to use.

                - "auto": Prefer the store's native engine and fall back to Polars if needed

                - "native": Always use the store's native engine, raise `VersioningEngineMismatchError`
                    if at some point the metadata store has to switch to Polars

                - "polars": Always use the Polars engine

            fallback_stores: Ordered list of read-only fallback stores.
                Used when upstream features are not in this store.
                `VersioningEngineMismatchError` is not raised when reading from fallback stores.
            auto_create_tables: Whether to automatically create tables when writing new features.
                If not provided, the global Metaxy configuration `auto_created_tables` option will
                be used (which can be set via `METAXY_AUTO_CREATE_TABLES` env var).

                !!! warning
                    This is intended for development/testing purposes.
                    Use proper database migration tools like Alembic to manage table infrastructure in production.

            materialization_id: Optional external orchestration ID.
                If provided, all metadata writes will include this ID in the `metaxy_materialization_id` column.
                Can be overridden per [`MetadataStore.write`][metaxy.MetadataStore.write] call.

        Raises:
            ValueError: If fallback stores use different hash algorithms or truncation lengths
            VersioningEngineMismatchError: If the versioning engine is attempted to be switched
                to Polars and `versioning_engine` is set to `native`.
        """
        self._name = name
        self._is_open = False
        self._mode_stack: list[AccessMode] = []
        self._versioning_engine = versioning_engine
        self._materialization_id = materialization_id
        self._exit_stack: list[AbstractContextManager[Any]] = []
        self._transaction_timestamp: datetime | None = None
        self._soft_delete_in_progress: bool = False

        if auto_create_tables is None:
            from metaxy.config import MetaxyConfig

            self.auto_create_tables = MetaxyConfig.get().auto_create_tables
        else:
            self.auto_create_tables = auto_create_tables

        if hash_algorithm is None:
            hash_algorithm = self._get_default_hash_algorithm()
        self.hash_algorithm = hash_algorithm

        from metaxy.metadata_store.fallback import FallbackStoreList

        self.fallback_stores: FallbackStoreList = (
            fallback_stores
            if isinstance(fallback_stores, FallbackStoreList)
            else FallbackStoreList(fallback_stores or [])
        )

    @overload
    def resolve_update(
        self,
        feature: CoercibleToFeatureKey,
        *,
        samples: IntoFrame | Frame | None = None,
        filters: Mapping[CoercibleToFeatureKey, Sequence[nw.Expr]] | None = None,
        global_filters: Sequence[nw.Expr] | None = None,
        target_filters: Sequence[nw.Expr] | None = None,
        lazy: Literal[False] = False,
        versioning_engine: Literal["auto", "native", "polars"] | None = None,
        skip_comparison: bool = False,
        **kwargs: Any,
    ) -> Increment: ...

    @overload
    def resolve_update(
        self,
        feature: CoercibleToFeatureKey,
        *,
        samples: IntoFrame | Frame | None = None,
        filters: Mapping[CoercibleToFeatureKey, Sequence[nw.Expr]] | None = None,
        global_filters: Sequence[nw.Expr] | None = None,
        target_filters: Sequence[nw.Expr] | None = None,
        lazy: Literal[True],
        versioning_engine: Literal["auto", "native", "polars"] | None = None,
        skip_comparison: bool = False,
        **kwargs: Any,
    ) -> LazyIncrement: ...

    def resolve_update(
        self,
        feature: CoercibleToFeatureKey,
        *,
        samples: IntoFrame | Frame | None = None,
        filters: Mapping[CoercibleToFeatureKey, Sequence[nw.Expr]] | None = None,
        global_filters: Sequence[nw.Expr] | None = None,
        target_filters: Sequence[nw.Expr] | None = None,
        lazy: bool = False,
        versioning_engine: Literal["auto", "native", "polars"] | None = None,
        skip_comparison: bool = False,
        staleness_predicates: Sequence[nw.Expr] | None = None,
        **kwargs: Any,
    ) -> Increment | LazyIncrement:
        """Calculate an incremental update for a feature.

        This is the main workhorse in Metaxy.

        Args:
            feature: Feature class to resolve updates for
            samples: A dataframe with joined upstream metadata and `"metaxy_provenance_by_field"` column set.
                When provided, `MetadataStore` skips loading upstream feature metadata and provenance calculations.

                !!! info "Required for root features"
                    Metaxy doesn't know how to populate input metadata for root features,
                    so `samples` argument **must** be provided for them.

                !!! tip
                    For non-root features, use `samples` to customize the automatic upstream loading and field provenance calculation.
                    For example, it can be used to requires processing for specific sample IDs.

                Setting this parameter during normal operations is not required.

            filters: A mapping from feature keys to lists of Narwhals filter expressions.
                Keys can be feature classes, FeatureKey objects, or string paths.
                Applied at read-time. May filter the current feature,
                in this case it will also be applied to `samples` (if provided).
                Example: `{UpstreamFeature: [nw.col("x") > 10], ...}`
            global_filters: A list of Narwhals filter expressions applied to all features
                (both upstream and target). These filters are combined with any feature-specific
                filters from `filters`. Must reference columns that exist in ALL features.
                Useful for filtering by common columns like `sample_uid` across all features.
                Example: `[nw.col("sample_uid").is_in(["s1", "s2"])]`
            target_filters: A list of Narwhals filter expressions applied ONLY to the target
                feature (not to upstream features). Use this when filtering by columns that
                only exist in the target feature.
                Example: `[nw.col("height").is_null()]`
            lazy: Whether to return a [`LazyIncrement`][metaxy.versioning.types.LazyIncrement] or a [`Increment`][metaxy.versioning.types.Increment].
            versioning_engine: Override the store's versioning engine for this operation.
            skip_comparison: If True, skip the increment comparison logic and return all
                upstream samples in `increment.new`. The `changed` and `removed` frames will
                be empty.
            staleness_predicates: Narwhals expressions to treat samples as stale even if their provenance is up-to-date.
                These expressions are evaluated against existing feature metadata.
                When multiple predicates are provided, a sample is considered stale if it matches any of them.

        Raises:
            ValueError: If no `samples` dataframe has been provided when resolving an update for a root feature.
            VersioningEngineMismatchError: If `versioning_engine` has been set to `"native"`
                and a dataframe of a different implementation has been encountered during `resolve_update`.

        !!! note
            This method automatically loads feature definitions from the metadata store
            before computing the update. This ensures that any external feature dependencies
            are resolved with their actual definitions from the store, preventing incorrect
            version calculations from stale external feature definitions.

        !!! example "With a root feature"

            ```py
            import narwhals as nw
            import polars as pl

            samples = pl.DataFrame(
                {
                    "id": ["x", "y", "z"],
                    "metaxy_provenance_by_field": [
                        {"part_1": "h1", "part_2": "h2"},
                        {"part_1": "h3", "part_2": "h4"},
                        {"part_1": "h5", "part_2": "h6"},
                    ],
                }
            )
            with store.open(mode="w"):
                result = store.resolve_update(MyFeature, samples=nw.from_native(samples))
            ```

        !!! example "With staleness predicates"

            Force reprocessing of failed or incomplete samples,
            regardless of whether their upstream provenance has changed:

            <!-- skip: next -->
            ```py
            import narwhals as nw

            with store.open(mode="w"):
                result = store.resolve_update(
                    MyFeature,
                    staleness_predicates=[
                        nw.col("status") == "failed",
                        nw.col("embedding_dim").is_null(),
                    ],
                )

                # result.stale includes both version-stale and predicate-matched samples
            ```
        """
        import narwhals as nw

        import metaxy as mx
        from metaxy.config import MetaxyConfig

        # Sync external feature definitions from the store to replace any external feature placeholders.
        # This ensures version hashes are computed correctly against actual stored definitions.
        # it is acceptable to call this here automatically for three reasons:
        # 1. `resolve_update` is typically only called once at the start of the workflow
        # 2. `resolve_update` is already doing heavy computations so an extra little call won't hurt performance
        # 3. it is extremely important to get the result right
        if MetaxyConfig.get(_allow_default_config=True).sync:
            mx.sync_external_features(self)

        # Convert samples to Narwhals frame if not already
        samples_nw: nw.DataFrame[Any] | nw.LazyFrame[Any] | None = None
        if samples is not None:
            if isinstance(samples, (nw.DataFrame, nw.LazyFrame)):
                samples_nw = samples
            else:
                samples_nw = nw.from_native(samples)  # ty: ignore[invalid-assignment]

        # Normalize filter keys to FeatureKey
        normalized_filters: dict[FeatureKey, list[nw.Expr]] = {}
        if filters:
            for key, exprs in filters.items():
                feature_key = self._resolve_feature_key(key)
                normalized_filters[feature_key] = list(exprs)

        # Convert global_filters and target_filters to lists for easy concatenation
        global_filter_list = list(global_filters) if global_filters else []
        target_filter_list = list(target_filters) if target_filters else []

        feature_key = self._resolve_feature_key(feature)
        if self._is_system_table(feature_key):
            raise NotImplementedError("Delete operations are not yet supported for system tables.")
        graph = current_graph()
        plan = graph.get_feature_plan(feature_key)

        # Root features without samples: error (samples required)
        if not plan.deps and samples_nw is None:
            raise ValueError(
                f"Feature {feature_key} has no upstream dependencies (root feature). "
                f"Must provide 'samples' parameter with sample_uid and {METAXY_PROVENANCE_BY_FIELD} columns. "
                f"Root features require manual {METAXY_PROVENANCE_BY_FIELD} computation."
            )

        # Combine feature-specific filters, global filters, and target filters for current feature
        # target_filters are ONLY applied to the current feature, not to upstream features
        current_feature_filters = [
            *normalized_filters.get(feature_key, []),
            *global_filter_list,
            *target_filter_list,
        ]

        # Read current metadata with deduplication (with_sample_history=False by default)
        # Use allow_fallback=False since we only want metadata from THIS store
        # to determine what needs to be updated locally
        try:
            current_metadata: nw.LazyFrame[Any] | None = self.read(
                feature_key,
                filters=current_feature_filters if current_feature_filters else None,
                allow_fallback=False,
                with_feature_history=False,  # filters by current feature_version
                with_sample_history=False,  # deduplicates by id_columns, keeping latest
            )
        except FeatureNotFoundError:
            current_metadata = None

        upstream_by_key: dict[FeatureKey, nw.LazyFrame[Any]] = {}
        filters_by_key: dict[FeatureKey, list[nw.Expr]] = {}

        # if samples are provided, use them as source of truth for upstream data
        if samples_nw is not None:
            # Apply filters to samples if any
            filtered_samples = samples_nw
            if current_feature_filters:
                filtered_samples = samples_nw.filter(current_feature_filters)

            # fill in METAXY_PROVENANCE column if it's missing (e.g. for root features)
            samples_nw = self.hash_struct_version_column(
                plan,
                df=filtered_samples,
                struct_column=METAXY_PROVENANCE_BY_FIELD,
                hash_column=METAXY_PROVENANCE,
            )

            # For root features, add data_version columns if they don't exist
            # (root features have no computation, so data_version equals provenance)
            # Use collect_schema().names() to avoid PerformanceWarning on lazy frames
            if METAXY_DATA_VERSION_BY_FIELD not in samples_nw.collect_schema().names():
                samples_nw = samples_nw.with_columns(
                    nw.col(METAXY_PROVENANCE_BY_FIELD).alias(METAXY_DATA_VERSION_BY_FIELD),
                    nw.col(METAXY_PROVENANCE).alias(METAXY_DATA_VERSION),
                )
        else:
            for upstream_spec in plan.deps or []:
                # Combine feature-specific filters with global filters for upstream
                upstream_filters = [
                    *normalized_filters.get(upstream_spec.key, []),
                    *global_filter_list,
                ]
                upstream_feature_metadata = self.read(
                    upstream_spec.key,
                    filters=upstream_filters if upstream_filters else None,
                )
                if upstream_feature_metadata is not None:
                    upstream_by_key[upstream_spec.key] = upstream_feature_metadata

        # determine which implementation to use for resolving the increment
        # consider (1) whether all upstream metadata has been loaded with the native implementation
        # (2) if samples have native implementation

        # Use parameter if provided, otherwise use store default
        engine_mode = versioning_engine if versioning_engine is not None else self._versioning_engine

        # If "polars" mode, force Polars immediately
        if engine_mode == "polars":
            implementation = nw.Implementation.POLARS
            switched_to_polars = True
        else:
            implementation = self.native_implementation()
            switched_to_polars = False

            for upstream_key, df in upstream_by_key.items():
                if df.implementation != implementation:
                    switched_to_polars = True
                    # Only raise error in "native" mode if no fallback stores configured.
                    # If fallback stores exist, the implementation mismatch indicates data came
                    # from fallback (different implementation), which is legitimate fallback access.
                    # If data were local, it would have the native implementation.
                    if engine_mode == "native" and not self.fallback_stores:
                        raise VersioningEngineMismatchError(
                            f"versioning_engine='native' but upstream feature `{upstream_key.to_string()}` "
                            f"has implementation {df.implementation}, expected {self.native_implementation()}"
                        )
                    elif engine_mode == "auto" or (engine_mode == "native" and self.fallback_stores):
                        PolarsMaterializationWarning.warn_on_implementation_mismatch(
                            expected=self.native_implementation(),
                            actual=df.implementation,
                            message=f"Using Polars for resolving the increment instead. This was caused by upstream feature `{upstream_key.to_string()}`.",
                        )
                    implementation = nw.Implementation.POLARS
                    break

            if samples_nw is not None and samples_nw.implementation != self.native_implementation():
                if not switched_to_polars:
                    if engine_mode == "native":
                        # Always raise error for samples with wrong implementation, regardless
                        # of fallback stores, because samples come from user argument, not from fallback
                        raise VersioningEngineMismatchError(
                            f"versioning_engine='native' but provided `samples` have implementation {samples_nw.implementation}, "
                            f"expected {self.native_implementation()}"
                        )
                    elif engine_mode == "auto":
                        PolarsMaterializationWarning.warn_on_implementation_mismatch(
                            expected=self.native_implementation(),
                            actual=samples_nw.implementation,
                            message=f"Provided `samples` have implementation {samples_nw.implementation}. Using Polars for resolving the increment instead.",
                        )
                implementation = nw.Implementation.POLARS
                switched_to_polars = True

        if switched_to_polars:
            if current_metadata:
                current_metadata = switch_implementation_to_polars(current_metadata)
            if samples_nw:
                samples_nw = switch_implementation_to_polars(samples_nw)
            for upstream_key, df in upstream_by_key.items():
                upstream_by_key[upstream_key] = switch_implementation_to_polars(df)

        with self.create_versioning_engine(plan=plan, implementation=implementation) as engine:
            if skip_comparison:
                # Skip comparison: return all upstream samples as added
                if samples_nw is not None:
                    # Root features or user-provided samples: use samples directly
                    # Note: samples already has metaxy_provenance computed
                    added = samples_nw.lazy()
                    input_df = None  # Root features have no upstream input
                else:
                    # Non-root features: load all upstream with provenance
                    added = engine.load_upstream_with_provenance(
                        upstream=upstream_by_key,
                        hash_algo=self.hash_algorithm,
                        filters=filters_by_key,
                    )
                    input_df = added  # Input is the same as added when skipping comparison
                changed = None
                removed = None
            else:
                added, changed, removed, input_df = engine.resolve_increment_with_provenance(
                    current=current_metadata,
                    upstream=upstream_by_key,
                    hash_algorithm=self.hash_algorithm,
                    filters=filters_by_key,
                    sample=samples_nw.lazy() if samples_nw is not None else None,
                    staleness_predicates=tuple(staleness_predicates) if staleness_predicates else (),
                )

        # Convert None to empty DataFrames
        if changed is None:
            changed = empty_frame_like(added)
        if removed is None:
            removed = empty_frame_like(added)

        if lazy:
            return LazyIncrement(
                new=added if isinstance(added, nw.LazyFrame) else nw.from_native(added),
                stale=changed if isinstance(changed, nw.LazyFrame) else nw.from_native(changed),
                orphaned=removed if isinstance(removed, nw.LazyFrame) else nw.from_native(removed),
                input=input_df if input_df is None or isinstance(input_df, nw.LazyFrame) else nw.from_native(input_df),
            )
        else:
            return Increment(
                new=added.collect() if isinstance(added, nw.LazyFrame) else added,
                stale=changed.collect() if isinstance(changed, nw.LazyFrame) else changed,
                orphaned=removed.collect() if isinstance(removed, nw.LazyFrame) else removed,
            )

    def compute_provenance(
        self,
        feature: CoercibleToFeatureKey,
        df: FrameT,
    ) -> FrameT:
        """Compute provenance columns for a DataFrame with pre-joined upstream data.

        !!! note
            This method may be useful in very rare cases.
            Rely on [`MetadataStore.resolve_update`][metaxy.metadata_store.base.MetadataStore.resolve_update] instead.

        Use this method when you perform custom joins outside of Metaxy's auto-join
        system but still want Metaxy to compute provenance. The method computes
        metaxy_provenance_by_field, metaxy_provenance, metaxy_data_version_by_field,
        and metaxy_data_version columns based on the upstream metadata.

        !!! info
            The input DataFrame must contain the renamed metaxy_data_version_by_field
            columns from each upstream feature. The naming convention follows the pattern
            `metaxy_data_version_by_field__<feature_key.to_column_suffix()>`. For example, for an
            upstream feature with key `["video", "raw"]`, the column should be named
            `metaxy_data_version_by_field__video_raw`.

        Args:
            feature: The feature to compute provenance for.
            df: A DataFrame containing pre-joined upstream data with renamed
                metaxy_data_version_by_field columns from each upstream feature.

        Returns:
            The input DataFrame with provenance columns added. Returns the same
            frame type as the input, either an eager DataFrame or a LazyFrame.

        Raises:
            StoreNotOpenError: If the store is not open.
            ValueError: If required upstream `metaxy_data_version_by_field` columns
                are missing from the DataFrame.

        Example:
            <!-- skip next -->
            ```py

                # Read upstream metadata
                video_df = store.read(VideoFeature).collect()
                audio_df = store.read(AudioFeature).collect()

                # Rename data_version_by_field columns to the expected convention
                video_df = video_df.rename({
                    "metaxy_data_version_by_field": "metaxy_data_version_by_field__video_raw"
                })
                audio_df = audio_df.rename({
                    "metaxy_data_version_by_field": "metaxy_data_version_by_field__audio_raw"
                })

                # Perform custom join
                joined = video_df.join(audio_df, on="sample_uid", how="inner")

                # Compute provenance
                with_provenance = store.compute_provenance(MyFeature, joined)

                # Pass to resolve_update
                increment = store.resolve_update(MyFeature, samples=with_provenance)
            ```
        """
        self._check_open()

        feature_key = self._resolve_feature_key(feature)
        graph = current_graph()
        plan = graph.get_feature_plan(feature_key)

        # Use native implementation if DataFrame matches, otherwise fall back to Polars
        implementation = self.native_implementation()
        if df.implementation != implementation:
            implementation = nw.Implementation.POLARS
            df = switch_implementation_to_polars(df)  # ty: ignore[no-matching-overload]

        with self.create_versioning_engine(plan=plan, implementation=implementation) as engine:
            # Validate required upstream columns exist
            expected_columns = {
                dep.feature: engine.get_renamed_data_version_by_field_col(dep.feature)
                for dep in (plan.feature_deps or [])
            }

            df_columns = set(df.collect_schema().names())  # ty: ignore[invalid-argument-type]
            missing_columns = [
                f"{col} (from upstream feature {key.to_string()})"
                for key, col in expected_columns.items()
                if col not in df_columns
            ]

            if missing_columns:
                raise ValueError(
                    f"DataFrame is missing required upstream columns for computing "
                    f"provenance of feature {feature_key.to_string()}. "
                    f"Missing columns: {missing_columns}. "
                    f"Make sure to rename metaxy_data_version_by_field columns from "
                    f"each upstream feature using the pattern "
                    f"metaxy_data_version_by_field__<feature_key.table_name>."
                )

            return engine.compute_provenance_columns(df, hash_algo=self.hash_algorithm)  # ty: ignore[invalid-argument-type]

    @overload
    def read(
        self,
        feature: CoercibleToFeatureKey,
        *,
        feature_version: str | None = None,
        filters: Sequence[nw.Expr] | None = None,
        columns: Sequence[str] | None = None,
        allow_fallback: bool = True,
        with_feature_history: bool = False,
        with_sample_history: bool = False,
        include_soft_deleted: bool = False,
        with_store_info: Literal[False] = False,
    ) -> nw.LazyFrame[Any]: ...

    @overload
    def read(
        self,
        feature: CoercibleToFeatureKey,
        *,
        feature_version: str | None = None,
        filters: Sequence[nw.Expr] | None = None,
        columns: Sequence[str] | None = None,
        allow_fallback: bool = True,
        with_feature_history: bool = False,
        with_sample_history: bool = False,
        include_soft_deleted: bool = False,
        with_store_info: Literal[True],
    ) -> tuple[nw.LazyFrame[Any], MetadataStore]: ...

    def read(
        self,
        feature: CoercibleToFeatureKey,
        *,
        feature_version: str | None = None,
        filters: Sequence[nw.Expr] | None = None,
        columns: Sequence[str] | None = None,
        allow_fallback: bool = True,
        with_feature_history: bool = False,
        with_sample_history: bool = False,
        include_soft_deleted: bool = False,
        with_store_info: bool = False,
    ) -> nw.LazyFrame[Any] | tuple[nw.LazyFrame[Any], MetadataStore]:
        """
        Read metadata with optional fallback to upstream stores.

        By default, does not include:
           - rows from historical feature versions (configured via `with_feature_history=False`)
           - rows that have been overwritten by subsequent writes with the same feature version (configured via `with_sample_history=False`)
           - soft-deleted with `metaxy_deleted_at` set to a non-null value (configured via `include_soft_deleted=False`)

        Args:
            feature: Feature to read metadata for
            feature_version: Explicit feature_version to filter by (mutually exclusive with with_feature_history=False)
            filters: Sequence of Narwhals filter expressions to apply to this feature.
                Example: `[nw.col("x") > 10, nw.col("y") < 5]`
            columns: Subset of columns to include. Metaxy's system columns are always included.
            allow_fallback: Whether to allow fallback to upstream stores if the requested feature is not found in the current store.
            with_feature_history: If True, include rows from all historical feature versions.
                By default (False), only returns rows with the currently registered feature version.
            with_sample_history: If True, include all historical materializations per sample.
                By default (False), deduplicates samples within `id_columns` groups ordered by `metaxy_created_at`.
            include_soft_deleted: If `True`, include soft-deleted rows in the result. Previous historical materializations of the same feature version will be effectively removed from the output otherwise.
            with_store_info: If `True`, return a tuple of (LazyFrame, MetadataStore) where
                the MetadataStore is the store that actually contained the feature (which
                may be a fallback store if allow_fallback=True).

        Returns:
            Narwhals `LazyFrame` with metadata, or, if `with_store_info=True`, a tuple of (`LazyFrame`, `MetadataStore`) containing the metadata store which has been actually used to retrieve the feature (may be a fallback store).

        Raises:
            FeatureNotFoundError: If feature not found in any store
            SystemDataNotFoundError: When attempting to read non-existent Metaxy system data
            ValueError: If both feature_version and with_feature_history=False are provided

        !!! info
            When this method is called with default arguments, it will return the latest (by `metaxy_created_at`)
            metadata for the current feature version excluding soft-deleted rows. Therefore, it's perfectly suitable
            for most use cases.

        !!! warning
            The order of rows is not guaranteed.
        """
        import metaxy as mx
        from metaxy.config import MetaxyConfig

        self._check_open()

        filters = filters or []
        columns = columns or []

        feature_key = self._resolve_feature_key(feature)
        is_system_table = self._is_system_table(feature_key)

        # Sync external features if auto-sync is enabled (default)
        # This call is a no-op most of the time and is very lightweight when it's not
        # Skip for system tables to avoid infinite recursion (sync_external_features reads system tables)
        if not is_system_table and MetaxyConfig.get(_allow_default_config=True).sync:
            mx.sync_external_features(self)

        # If caller wants soft-deleted records, do not filter them out later
        filter_deleted = not include_soft_deleted and not is_system_table

        # Validate mutually exclusive parameters
        if feature_version is not None and not with_feature_history:
            raise ValueError(
                "Cannot specify both feature_version and with_feature_history=False. "
                "Use with_feature_history=True with feature_version parameter."
            )

        # Separate system filters (applied before dedup) from user filters (applied after dedup)
        # System filters like feature_version need to be applied early to reduce data volume
        # User filters should see the deduplicated view of the data
        system_filters: list[nw.Expr] = []
        user_filters = list(filters) if filters else []

        # Add feature_version filter only when needed (this is a system filter)
        if not with_feature_history or feature_version is not None and not is_system_table:
            version_filter = nw.col(METAXY_FEATURE_VERSION) == (
                current_graph().get_feature_version(feature_key) if not with_feature_history else feature_version
            )
            system_filters.append(version_filter)

        # If user filters are provided, we need to read all columns since filters may
        # reference columns not in the requested columns list. Column selection happens
        # after filtering
        if user_filters:
            read_columns = None
        elif columns and not is_system_table:
            # Add only system columns that aren't already in the user's columns list
            columns_set = set(columns)
            missing_system_cols = [c for c in ALL_SYSTEM_COLUMNS if c not in columns_set]
            read_columns = [*columns, *missing_system_cols]
        else:
            read_columns = None

        lazy_frame = None
        try:
            # Only pass system filters to _read_feature
            # User filters will be applied after deduplication
            lazy_frame = self._read_feature(
                feature, filters=system_filters if system_filters else None, columns=read_columns
            )
        except FeatureNotFoundError as e:
            # do not read system features from fallback stores
            if is_system_table:
                raise SystemDataNotFoundError(
                    f"System Metaxy data with key {feature_key} is missing in {self}. Invoke `metaxy push` before attempting to read system data."
                ) from e

        # Handle case where _read_feature returns None (no exception raised)
        if lazy_frame is None and is_system_table:
            raise SystemDataNotFoundError(
                f"System Metaxy data with key {feature_key} is missing in {self}. Invoke `metaxy push` before attempting to read system data."
            )

        if lazy_frame is not None and not is_system_table:
            # Deduplicate first, then filter soft-deleted rows
            if not with_sample_history:
                id_cols = list(self._resolve_feature_plan(feature_key).feature.id_columns)
                # Treat soft-deletes like hard deletes by ordering on the
                # most recent lifecycle timestamp.
                lazy_frame = self.versioning_engine_cls.keep_latest_by_group(
                    df=lazy_frame,
                    group_columns=id_cols,
                    timestamp_columns=[METAXY_DELETED_AT, METAXY_UPDATED_AT],
                )

            if filter_deleted:
                lazy_frame = lazy_frame.filter(nw.col(METAXY_DELETED_AT).is_null())

            # Apply user filters AFTER deduplication so they see the latest version of each row
            for user_filter in user_filters:
                lazy_frame = lazy_frame.filter(user_filter)

        # For system tables, apply user filters directly (no dedup needed)
        if lazy_frame is not None and is_system_table:
            for user_filter in user_filters:
                lazy_frame = lazy_frame.filter(user_filter)

        if lazy_frame is not None:
            # After dedup and user filters, filter to requested columns if specified
            if columns:
                lazy_frame = lazy_frame.select(columns)

            if with_store_info:
                return lazy_frame, self
            return lazy_frame

        # Try fallback stores (opened on demand)
        if allow_fallback:
            for store in self.fallback_stores:
                try:
                    # Open fallback store on demand for reading
                    with store:
                        # Use full read to handle nested fallback chains
                        if with_store_info:
                            return store.read(
                                feature,
                                feature_version=feature_version,
                                filters=filters,
                                columns=columns,
                                allow_fallback=True,
                                with_feature_history=with_feature_history,
                                with_sample_history=with_sample_history,
                                include_soft_deleted=include_soft_deleted,
                                with_store_info=True,
                            )
                        return store.read(
                            feature,
                            feature_version=feature_version,
                            filters=filters,
                            columns=columns,
                            allow_fallback=True,
                            with_feature_history=with_feature_history,
                            with_sample_history=with_sample_history,
                            include_soft_deleted=include_soft_deleted,
                        )
                except FeatureNotFoundError:
                    # Try next fallback store
                    continue

        # Not found anywhere
        raise FeatureNotFoundError(
            f"Feature {feature_key.to_string()} not found in store" + (" or fallback stores" if allow_fallback else "")
        )

    def write(
        self,
        feature: CoercibleToFeatureKey,
        df: IntoFrame,
        materialization_id: str | None = None,
        preserve_feature_version: bool = False,
    ) -> None:
        """
        Write metadata for a feature (append-only by design).

        Automatically adds the Metaxy system columns, unless they already exist in the DataFrame.

        Args:
            feature: Feature to write metadata for
            df: Metadata DataFrame of any type supported by [Narwhals](https://narwhals-dev.github.io/narwhals/).
                Must have `metaxy_provenance_by_field` column of type Struct with fields matching feature's fields.
                Optionally, may also contain `metaxy_data_version_by_field`.
            materialization_id: Optional external orchestration ID for this write.
                Overrides the store's default `materialization_id` if provided.
                Useful for tracking which orchestration run produced this metadata.
            preserve_feature_version: If `True`, keep the `metaxy_feature_version`
                and `metaxy_project_version` already present in `df` instead of
                overwriting them with the current graph's versions.
                Is useful when writing [rebased](/guide/concepts/metadata-stores.md#rebases) metadata.

        Raises:
            MetadataSchemaError: If DataFrame schema is invalid
            StoreNotOpenError: If store is not open
        Note:
            - Must be called within a `MetadataStore.open(mode="w")` context manager.

            - Metaxy always performs an "append" operation. Metadata is never deleted or mutated.

            - Fallback stores are never used for writes.

        """
        self._check_write_mode()

        feature_key = self._resolve_feature_key(feature)
        is_system_table = self._is_system_table(feature_key)

        # Convert Polars to Narwhals to Polars if needed
        # if isinstance(df_nw, (pl.DataFrame, pl.LazyFrame)):
        df_nw = nw.from_native(df)

        assert isinstance(df_nw, (nw.DataFrame, nw.LazyFrame)), f"df must be a Narwhals DataFrame, got {type(df_nw)}"

        # For system tables, write directly without feature_version tracking
        if is_system_table:
            self._validate_schema_system_table(df_nw)
            self._write_feature(feature_key, df_nw)
            return

        # Use collect_schema().names() to avoid PerformanceWarning on lazy frames
        if METAXY_PROVENANCE_BY_FIELD not in df_nw.collect_schema().names():
            from metaxy.metadata_store.exceptions import MetadataSchemaError

            raise MetadataSchemaError(f"DataFrame must have '{METAXY_PROVENANCE_BY_FIELD}' column")

        # Add all required system columns
        # warning: for dataframes that do not match the native MetadataStore implementation
        # and are missing the METAXY_DATA_VERSION column, this call will lead to materializing the equivalent Polars DataFrame
        # while calculating the missing METAXY_DATA_VERSION column
        df_nw = self._add_system_columns(
            df_nw, feature, materialization_id=materialization_id, preserve_feature_version=preserve_feature_version
        )

        self._validate_schema(df_nw)
        self._write_feature(feature_key, df_nw)

    def write_multi(
        self,
        metadata: Mapping[Any, IntoFrame],
        materialization_id: str | None = None,
    ) -> None:
        """
        Write metadata for multiple features in reverse topological order.

        Processes features so that dependents are written before their dependencies.
        This ordering ensures that downstream features are written first, which can
        be useful for certain data consistency requirements or when features need
        to be processed in a specific order.

        Args:
            metadata: Mapping from feature keys to metadata DataFrames.
                Keys can be any type coercible to FeatureKey (string, sequence,
                FeatureKey, or BaseFeature class). Values must be DataFrames
                compatible with Narwhals, containing required system columns.
            materialization_id: Optional external orchestration ID for all writes.
                Overrides the store's default `materialization_id` if provided.
                Applied to all feature writes in this batch.

        Raises:
            MetadataSchemaError: If any DataFrame schema is invalid
            StoreNotOpenError: If store is not open
            ValueError: If writing to a feature from a different project than expected

        Note:
            - Must be called within a `MetadataStore.open(mode="w")` context manager.
            - Empty mappings are handled gracefully (no-op).
            - Each feature's metadata is written via `write`, so all
              validation and system column handling from that method applies.

        Example:
            <!-- skip next -->
            ```py
            with store.open(mode="w"):
                store.write_multi(
                    {
                        ChildFeature: child_df,
                        ParentFeature: parent_df,
                    }
                )
            # Features are written in reverse topological order:
            # ChildFeature first, then ParentFeature
            ```
        """
        if not metadata:
            return

        # Build mapping from resolved keys to dataframes in one pass
        resolved_metadata = {self._resolve_feature_key(key): df for key, df in metadata.items()}

        # Get reverse topological order (dependents first)
        graph = current_graph()
        sorted_keys = graph.topological_sort_features(list(resolved_metadata.keys()), descending=True)

        # Write metadata in reverse topological order
        for feature_key in sorted_keys:
            self.write(
                feature_key,
                resolved_metadata[feature_key],
                materialization_id=materialization_id,
            )

    @classmethod
    @abstractmethod
    def config_model(cls) -> type[MetadataStoreConfig]:
        """Return the configuration model class for this store type.

        Subclasses must override this to return their specific config class.

        Returns:
            The config class type (e.g., DuckDBMetadataStoreConfig)

        Note:
            Subclasses override this with a more specific return type.
            Type checkers may show a warning about incompatible override,
            but this is intentional - each store returns its own config type.
        """
        ...

    @classmethod
    def from_config(cls, config: MetadataStoreConfig, **kwargs: Any) -> Self:
        """Create a store instance from a configuration object.

        This method creates a store by:
        1. Converting the config to a dict
        2. Resolving fallback store names to actual store instances
        3. Calling the store's __init__ with the config parameters

        Args:
            config: Configuration object (should be the type returned by config_model())
            **kwargs: Additional arguments passed directly to the store constructor
                (e.g., materialization_id for runtime parameters not in config)

        Returns:
            A new store instance configured according to the config object

        Example:
            <!-- skip next -->
            ```python
            from metaxy.ext.metadata_stores.duckdb import (
                DuckDBMetadataStore,
                DuckDBMetadataStoreConfig,
            )

            config = DuckDBMetadataStoreConfig(
                database="metadata.db",
                fallback_stores=["prod"],
            )

            store = DuckDBMetadataStore.from_config(config)
            ```
        """
        from metaxy.metadata_store.fallback import FallbackStoreList

        config_dict = config.model_dump(exclude_unset=True)
        fallback_store_names = config_dict.pop("fallback_stores", [])
        store = cls(**config_dict, **kwargs)
        if fallback_store_names:
            store.fallback_stores = FallbackStoreList(
                fallback_store_names,
                config=MetaxyConfig.get(),
                parent_hash_algorithm=store.hash_algorithm,
            )
        return store

    @property
    def hash_truncation_length(self) -> int:
        return MetaxyConfig.get().hash_truncation_length

    @property
    def materialization_id(self) -> str | None:
        """The external orchestration ID for this store instance.

        If set, all metadata writes include this ID in the `metaxy_materialization_id` column,
        allowing filtering of rows written during a specific materialization run.
        """
        return self._materialization_id

    @property
    def name(self) -> str | None:
        """The configured name of this store, if any."""
        return self._name

    @property
    def qualified_class_name(self) -> str:
        """The fully qualified class name (module.classname)."""
        cls = self.__class__
        return f"{cls.__module__}.{cls.__qualname__}"

    def _format_display_with_name(self, display_str: str) -> str:
        """Format display string with optional name prefix."""
        if self._name is not None:
            return f"[{self._name}] {display_str}"
        return display_str

    @abstractmethod
    def _get_default_hash_algorithm(self) -> HashAlgorithm:
        """Get the default hash algorithm for this store type.

        Returns:
            Default hash algorithm
        """
        pass

    def native_implementation(self) -> nw.Implementation:
        """Get the native Narwhals implementation for this store's backend."""
        return self.versioning_engine_cls.implementation()

    @abstractmethod
    @contextmanager
    def _create_versioning_engine(self, plan: FeaturePlan) -> Iterator[VersioningEngine]:
        """Create provenance engine for this store as a context manager.

        Args:
            plan: Feature plan for the feature we're tracking provenance for

        Yields:
            VersioningEngine instance appropriate for this store's backend.
            - For SQL stores (DuckDB, ClickHouse): Returns IbisVersioningEngine
            - For in-memory/Polars stores: Returns PolarsVersioningEngine

        Raises:
            NotImplementedError: If provenance tracking not supported by this store

        Example:
            <!-- skip next -->
            ```python
            with self._create_versioning_engine(plan) as engine:
                result = engine.resolve_update(...)
            ```
        """
        ...

    @contextmanager
    def _create_polars_versioning_engine(self, plan: FeaturePlan) -> Iterator[PolarsVersioningEngine]:
        yield PolarsVersioningEngine(plan=plan)

    @contextmanager
    def create_versioning_engine(
        self, plan: FeaturePlan, implementation: nw.Implementation
    ) -> Iterator[VersioningEngine | PolarsVersioningEngine]:
        """
        Creates an appropriate provenance engine.

        Falls back to Polars implementation if the required implementation differs from the store's native implementation.

        Args:
            plan: The feature plan.
            implementation: The desired engine implementation.

        Returns:
            An appropriate provenance engine.
        """

        if implementation == nw.Implementation.POLARS:
            cm = self._create_polars_versioning_engine(plan)
        elif implementation == self.native_implementation():
            cm = self._create_versioning_engine(plan)
        else:
            cm = self._create_polars_versioning_engine(plan)

        with cm as engine:
            yield engine

    def hash_struct_version_column(
        self,
        plan: FeaturePlan,
        df: Frame,
        struct_column: str,
        hash_column: str,
    ) -> Frame:
        with self.create_versioning_engine(plan, df.implementation) as engine:
            if isinstance(engine, PolarsVersioningEngine) and df.implementation != nw.Implementation.POLARS:
                PolarsMaterializationWarning.warn_on_implementation_mismatch(
                    self.native_implementation(),
                    df.implementation,
                    message=f"`{hash_column}` will be calculated in Polars.",
                )
                df = switch_implementation_to_polars(nw.from_native(df.lazy().collect()))

            return cast(
                Frame,
                engine.hash_struct_version_column(
                    df,  # ty: ignore[invalid-argument-type]
                    hash_algorithm=self.hash_algorithm,
                    struct_column=struct_column,
                    hash_column=hash_column,
                ),
            )

    @contextmanager
    def open(self, mode: AccessMode = "r") -> Iterator[Self]:
        """Open/initialize the store for operations.

        Context manager that opens the store with specified access mode.
        Manages a mode stack to support nested contexts with mode escalation.

        Args:
            mode: Access mode for this connection session.

        Yields:
            Self: The store instance with connection open

        Note:
            Users should prefer using `with store:` pattern except when write access mode is needed.
        """
        self._mode_stack.append(mode)
        try:
            if len(self._mode_stack) == 1:
                self._open(mode)
                self._is_open = True
                self._validate_after_open()
            yield self
        finally:
            self._mode_stack.pop()
            if not self._mode_stack:
                self._close()
                self._is_open = False

    @abstractmethod
    def _open(self, mode: AccessMode) -> None:
        """Open backend-specific connection/resources.

        Args:
            mode: Access mode for this connection session.
        """
        ...

    @abstractmethod
    def _close(self) -> None:
        """Close backend-specific connection/resources."""
        ...

    def __enter__(self) -> Self:
        """Enter context manager - opens store in READ mode by default.

        Use [`MetadataStore.open`][metaxy.metadata_store.base.MetadataStore.open] for write access mode instead.

        Returns:
            Self: The opened store instance
        """
        cm = self.open("r")
        cm.__enter__()
        self._exit_stack.append(cm)
        return self

    def _validate_after_open(self) -> None:
        """Validate hash algorithm compatibility after store is opened.

        When fallback stores are lazy (not yet resolved), hash algorithm checks
        are deferred to FallbackStoreList._resolve() at resolution time.
        """
        stores_resolved = self.fallback_stores.all_resolved
        self.validate_hash_algorithm(check_fallback_stores=stores_resolved)

        if stores_resolved:
            for i, fallback_store in enumerate(self.fallback_stores):
                if fallback_store.hash_algorithm != self.hash_algorithm:
                    raise ValueError(
                        f"Fallback store {i} uses hash_algorithm='{fallback_store.hash_algorithm.value}' "
                        f"but this store uses '{self.hash_algorithm.value}'. "
                        f"All stores in a fallback chain must use the same hash algorithm."
                    )

    def __exit__(
        self,
        exc_type: type[BaseException] | None,
        exc_val: BaseException | None,
        exc_tb: TracebackType | None,
    ) -> None:
        if self._exit_stack:
            self._exit_stack.pop().__exit__(exc_type, exc_val, exc_tb)

    @property
    def _access_mode(self) -> AccessMode:
        return self._mode_stack[-1] if self._mode_stack else "r"

    def _check_open(self) -> None:
        """Check if store is open, raise error if not.

        Raises:
            StoreNotOpenError: If store is not open
        """
        if not self._is_open:
            raise StoreNotOpenError(
                f"{self.__class__.__name__} must be opened before use. "
                'Use it as a context manager: `with store: ...` or `with store.open(mode="w"): ...`'
            )

    def _check_write_mode(self) -> None:
        """Check if store is open in write mode.

        Raises:
            StoreNotOpenError: If store is not open or not in write mode
        """
        self._check_open()
        if self._access_mode != "w":
            raise StoreNotOpenError(
                f"{self.__class__.__name__} is open in read mode. "
                'Write operations require write mode: `with store.open(mode="w"): ...`'
            )

    # ========== Hash Algorithm Validation ==========

    def validate_hash_algorithm(
        self,
        check_fallback_stores: bool = True,
    ) -> None:
        """Validate that hash algorithm is supported by this store's components.

        Public method - can be called to verify hash compatibility.

        Args:
            check_fallback_stores: If True, also validate hash is supported by
                fallback stores (ensures compatibility for future cross-store operations)

        Raises:
            ValueError: If hash algorithm not supported by components or fallback stores
        """
        # Validate hash algorithm support without creating a full engine
        # (engine creation requires a graph which isn't available during store init)
        self._validate_hash_algorithm_support()

        # Check fallback stores
        if check_fallback_stores:
            for fallback in self.fallback_stores:
                fallback.validate_hash_algorithm(check_fallback_stores=False)

    def _validate_hash_algorithm_support(self) -> None:
        """Validate that the configured hash algorithm is supported.

        Default implementation does nothing (assumes all algorithms supported).
        Subclasses can override to check algorithm support.

        Raises:
            Exception: If hash algorithm is not supported
        """
        # Default: no validation (assume all algorithms supported)
        pass

    # ========== Helper Methods ==========

    def _is_system_table(self, feature_key: FeatureKey) -> bool:
        """Check if feature key is a system table."""
        return len(feature_key) >= 1 and feature_key[0] == METAXY_SYSTEM_KEY_PREFIX

    def _resolve_feature_key(self, feature: CoercibleToFeatureKey) -> FeatureKey:
        """Resolve various types to FeatureKey.

        Accepts types that can be converted into a FeatureKey.

        Args:
            feature: Feature to resolve to FeatureKey

        Returns:
            FeatureKey instance
        """
        return ValidatedFeatureKeyAdapter.validate_python(feature)

    def _resolve_feature_plan(self, feature: CoercibleToFeatureKey) -> FeaturePlan:
        """Resolve to FeaturePlan for dependency resolution."""
        # First resolve to FeatureKey
        feature_key = self._resolve_feature_key(feature)
        # Then get the plan
        graph = current_graph()
        return graph.get_feature_plan(feature_key)

    # ========== Core CRUD Operations ==========

    @contextmanager
    def _shared_transaction_timestamp(self, *, soft_delete: bool = False) -> Iterator[datetime]:
        """Context manager that establishes a shared timestamp for a write transaction.

        All write operations (write, delete with soft=True) within
        this context share the same timestamp for metaxy_created_at and metaxy_deleted_at
        columns. This ensures consistency when a single logical operation affects
        multiple system columns.

        If already within a transaction, returns the existing timestamp without
        creating a new one (reentrant).

        Args:
            soft_delete: If True, preserves metaxy_updated_at during writes within this context.

        Yields:
            datetime: The transaction timestamp (UTC)
        """
        if self._transaction_timestamp is not None:
            # Already in a transaction, reuse existing timestamp
            yield self._transaction_timestamp
        else:
            # Start new transaction
            self._transaction_timestamp = datetime.now(timezone.utc)
            if soft_delete:
                self._soft_delete_in_progress = True
            try:
                yield self._transaction_timestamp
            finally:
                self._transaction_timestamp = None
                self._soft_delete_in_progress = False

    @abstractmethod
    def _write_feature(
        self,
        feature_key: FeatureKey,
        df: Frame,
        **kwargs: Any,
    ) -> None:
        """
        Internal write implementation (backend-specific).

        Backends may convert to their specific type if needed (e.g., Polars, Ibis).

        Args:
            feature_key: Feature key to write to
            df: [Narwhals](https://narwhals-dev.github.io/narwhals/)-compatible DataFrame with metadata to write
            **kwargs: Backend-specific parameters

        Note: Subclasses implement this for their storage backend.
        """
        pass

    def _add_system_columns(
        self,
        df: Frame,
        feature: CoercibleToFeatureKey,
        materialization_id: str | None = None,
        preserve_feature_version: bool = False,
    ) -> Frame:
        """Add all required system columns to the DataFrame.

        Args:
            df: Narwhals DataFrame/LazyFrame
            feature: Feature class or key
            materialization_id: Optional external orchestration ID for this write.
                Overrides the store's default if provided.

        Returns:
            DataFrame with all system columns added
        """
        feature_key = self._resolve_feature_key(feature)

        # Use collect_schema().names() to avoid PerformanceWarning on lazy frames
        columns = df.collect_schema().names()

        # Check if version columns already exist in DataFrame
        has_feature_version = METAXY_FEATURE_VERSION in columns
        has_project_version = METAXY_PROJECT_VERSION in columns

        # When preserving feature versions (e.g. rebases), use existing values as-is
        if preserve_feature_version and has_feature_version and has_project_version:
            pass  # Use existing values (e.g. rebases)
        else:
            # Drop any existing version columns (e.g., from SQLModel with null values)
            # and add current versions
            columns_to_drop = []
            if has_feature_version:
                columns_to_drop.append(METAXY_FEATURE_VERSION)
            if has_project_version:
                columns_to_drop.append(METAXY_PROJECT_VERSION)
            if columns_to_drop:
                df = df.drop(*columns_to_drop)

            # Get current feature version and project_version from graph
            from metaxy.models.feature import FeatureGraph

            graph = FeatureGraph.get_active()

            current_feature_version = graph.get_feature_version(feature_key)
            current_project_version = graph.project_version

            df = df.with_columns(
                [
                    nw.lit(current_feature_version).alias(METAXY_FEATURE_VERSION),
                    nw.lit(current_project_version).alias(METAXY_PROJECT_VERSION),
                ]
            )

        # These should normally be added by the provenance engine during resolve_update
        from metaxy.models.constants import (
            METAXY_CREATED_AT,
            METAXY_DATA_VERSION,
            METAXY_DATA_VERSION_BY_FIELD,
            METAXY_UPDATED_AT,
        )

        # Re-fetch columns since df may have been modified above
        columns = df.collect_schema().names()

        if METAXY_PROVENANCE_BY_FIELD not in columns:
            raise ValueError(
                f"Metadata is missing a required column `{METAXY_PROVENANCE_BY_FIELD}`. It should have been created by a prior `MetadataStore.resolve_update` call. Did you drop it on the way?"
            )

        if METAXY_PROVENANCE not in columns:
            plan = self._resolve_feature_plan(feature_key)

            # Only warn for non-root features (features with dependencies).
            # Root features don't have upstream dependencies, so they don't go through
            # resolve_update() - they just need metaxy_provenance_by_field to be set.
            if plan.deps:
                MetaxyColumnMissingWarning.warn_on_missing_column(
                    expected=METAXY_PROVENANCE,
                    df=df,
                    message=f"It should have been created by a prior `MetadataStore.resolve_update` call. Re-crearing it from `{METAXY_PROVENANCE_BY_FIELD}` Did you drop it on the way?",
                )

            df = self.hash_struct_version_column(
                plan=plan,
                df=df,
                struct_column=METAXY_PROVENANCE_BY_FIELD,
                hash_column=METAXY_PROVENANCE,
            )

        # Re-fetch columns since df may have been modified
        columns = df.collect_schema().names()

        # Use shared transaction timestamp to ensure consistency across
        # metaxy_created_at and metaxy_updated_at columns
        with self._shared_transaction_timestamp() as ts:
            if METAXY_CREATED_AT not in columns:
                df = df.with_columns(nw.lit(ts).alias(METAXY_CREATED_AT))

            # metaxy_updated_at: set to current transaction time unless soft delete is in progress
            # Soft delete preserves the original updated_at to reflect when data was last changed
            if not self._soft_delete_in_progress:
                df = df.with_columns(nw.lit(ts).alias(METAXY_UPDATED_AT))

        if METAXY_DELETED_AT not in columns:
            df = df.with_columns(nw.lit(None, dtype=nw.Datetime(time_zone="UTC")).alias(METAXY_DELETED_AT))

        # Add materialization_id if not already present
        from metaxy.models.constants import METAXY_MATERIALIZATION_ID

        df = df.with_columns(
            nw.lit(materialization_id or self._materialization_id, dtype=nw.String).alias(METAXY_MATERIALIZATION_ID)
        )

        # data_version defaults to provenance but can be user-overridden.
        # When the column is missing, add it from provenance.
        # When the column is present, fill null rows from provenance (partial override).
        # Re-fetch columns since df may have been modified
        columns = df.collect_schema().names()

        if METAXY_DATA_VERSION_BY_FIELD not in columns:
            df = df.with_columns(nw.col(METAXY_PROVENANCE_BY_FIELD).alias(METAXY_DATA_VERSION_BY_FIELD))
            df = df.with_columns(nw.col(METAXY_PROVENANCE).alias(METAXY_DATA_VERSION))
        else:
            # Fill null data_version_by_field rows from provenance
            df = df.with_columns(
                nw.when(nw.col(METAXY_DATA_VERSION_BY_FIELD).is_null())
                .then(nw.col(METAXY_PROVENANCE_BY_FIELD))
                .otherwise(nw.col(METAXY_DATA_VERSION_BY_FIELD))
                .alias(METAXY_DATA_VERSION_BY_FIELD)
            )
            if METAXY_DATA_VERSION not in columns:
                df = self.hash_struct_version_column(
                    plan=self._resolve_feature_plan(feature_key),
                    df=df,
                    struct_column=METAXY_DATA_VERSION_BY_FIELD,
                    hash_column=METAXY_DATA_VERSION,
                )
            else:
                # Fill null data_version rows from provenance
                df = df.with_columns(
                    nw.when(nw.col(METAXY_DATA_VERSION).is_null())
                    .then(nw.col(METAXY_PROVENANCE))
                    .otherwise(nw.col(METAXY_DATA_VERSION))
                    .alias(METAXY_DATA_VERSION)
                )

        # Cast system columns with Null dtype to their correct types
        # This handles edge cases where empty DataFrames or certain operations
        # result in Null-typed columns that break downstream processing
        df = _cast_present_system_columns(df)

        return df

    def _validate_schema(self, df: Frame) -> None:
        """
        Validate that DataFrame has required schema.

        Args:
            df: Narwhals DataFrame or LazyFrame to validate

        Raises:
            MetadataSchemaError: If schema is invalid
        """
        from metaxy.metadata_store.exceptions import MetadataSchemaError

        schema = df.collect_schema()

        # Check for metaxy_provenance_by_field column
        if METAXY_PROVENANCE_BY_FIELD not in schema.names():
            raise MetadataSchemaError(f"DataFrame must have '{METAXY_PROVENANCE_BY_FIELD}' column")

        # Check that metaxy_provenance_by_field is a struct or map
        provenance_dtype = schema[METAXY_PROVENANCE_BY_FIELD]
        if not isinstance(provenance_dtype, nw.Struct) and not _is_map_column(df, METAXY_PROVENANCE_BY_FIELD):
            raise MetadataSchemaError(
                f"'{METAXY_PROVENANCE_BY_FIELD}' column must be a Struct or Map, got {provenance_dtype}"
            )

        # Note: metaxy_provenance is auto-computed if missing, so we don't validate it here

        # Check for feature_version column
        if METAXY_FEATURE_VERSION not in schema.names():
            raise MetadataSchemaError(f"DataFrame must have '{METAXY_FEATURE_VERSION}' column")

        # Check for project_version column
        if METAXY_PROJECT_VERSION not in schema.names():
            raise MetadataSchemaError(f"DataFrame must have '{METAXY_PROJECT_VERSION}' column")

    def _validate_schema_system_table(self, df: Frame) -> None:
        """Validate schema for system tables (minimal validation).

        Args:
            df: Narwhals DataFrame to validate
        """
        # System tables don't need metaxy_provenance_by_field column
        pass

    @abstractmethod
    def _drop_feature(self, feature_key: FeatureKey) -> None:
        """Drop/delete all metadata for a feature.

        Backend-specific implementation for dropping feature metadata.

        Args:
            feature_key: The feature key to drop metadata for
        """
        pass

    @abstractmethod
    def _delete_feature(
        self,
        feature_key: FeatureKey,
        filters: Sequence[nw.Expr] | None,
        *,
        with_feature_history: bool,
    ) -> None:
        """Backend-specific hard delete implementation.

        Args:
            feature_key: Feature to delete from
            filters: Optional Narwhals expressions to filter records; None or empty deletes all
            with_feature_history: If True, delete across all feature versions. If False, only current version.
        """
        raise NotImplementedError(f"{self.__class__.__name__} does not yet support hard delete.")

    def drop_feature_metadata(self, feature: CoercibleToFeatureKey) -> None:
        """Drop all metadata for a feature.

        This removes all stored metadata for the specified feature from the store.
        Useful for cleanup in tests or when re-computing feature metadata from scratch.

        Warning:
            This operation is irreversible and will **permanently delete all metadata** for the specified feature.

        Args:
            feature: Feature class or key to drop metadata for

        Example:
            ```py
            with store_with_data.open(mode="w"):
                assert store_with_data.has_feature(MyFeature)
                store_with_data.drop_feature_metadata(MyFeature)
            ```
        """
        self._check_write_mode()
        feature_key = self._resolve_feature_key(feature)
        if self._is_system_table(feature_key):
            raise NotImplementedError(f"{self.__class__.__name__} does not support deletes for system tables")
        self._drop_feature(feature_key)

    def delete(
        self,
        feature: CoercibleToFeatureKey,
        filters: Sequence[nw.Expr] | nw.Expr | None,
        *,
        soft: bool = True,
        with_feature_history: bool = False,
        with_sample_history: bool = False,
    ) -> None:
        """Delete records matching provided filters.

        Performs a soft delete by default. This is achieved by setting metaxy_deleted_at to the current timestamp.
        Subsequent [[MetadataStore.read]] calls would ignore these records by default.

        Args:
            feature: Feature to delete from.
            filters: One or more Narwhals expressions or a sequence of expressions that determine which records to delete.
                If `None`, deletes all records (subject to `with_feature_history` and `with_sample_history` constraints).
            soft: Whether to perform a soft delete.
            with_feature_history: If True, delete across all historical feature versions. If False (default), only current version.
            with_sample_history: If True, include all historical materializations. If False (default), deduplicate to latest rows.

        !!! critical
            By default, deletions target historical records. Even when `with_feature_history` is `False`,
            records with the same feature version but an older `metaxy_created_at` would be targeted as
            well. Consider adding additional conditions to `filters` if you want to avoid that.
        """
        self._check_write_mode()
        feature_key = self._resolve_feature_key(feature)

        # Normalize filters to list
        if filters is None:
            filter_list: list[nw.Expr] = []
        elif isinstance(filters, nw.Expr):
            filter_list = [filters]
        else:
            filter_list = list(filters)

        if soft:
            # Soft delete: mark records with deletion timestamp, preserving original updated_at
            lazy = self.read(
                feature_key,
                filters=filter_list,
                include_soft_deleted=False,
                with_feature_history=with_feature_history,
                with_sample_history=with_sample_history,
                allow_fallback=True,
            )
            with self._shared_transaction_timestamp(soft_delete=True) as ts:
                soft_deletion_marked = lazy.with_columns(
                    nw.lit(ts).alias(METAXY_DELETED_AT),
                )
                self.write(feature_key, soft_deletion_marked.to_native())
        else:
            # Hard delete: add version filter if needed, then delegate to backend
            if not with_feature_history and not self._is_system_table(feature_key):
                version_filter = nw.col(METAXY_FEATURE_VERSION) == current_graph().get_feature_version(feature_key)
                filter_list = [version_filter, *filter_list]

            self._delete_feature(feature_key, filter_list, with_feature_history=with_feature_history)

    @public
    def rebase(
        self,
        feature: CoercibleToFeatureKey,
        df: FrameT,
        *,
        to_feature_version: str | None = None,
    ) -> FrameT:
        """Rebase metadata from one feature version to another.

        Recalculates Metaxy versioning columns for the desired feature version.
        User-provided `metaxy_data_version` values are preserved.

        The caller is responsible for providing a `df` with the right historical feature version.
        The returned frame includes `metaxy_feature_version` and `metaxy_project_version` columns
        set to the target version; pass `preserve_feature_version=True` to
        [`MetadataStore.write`][metaxy.MetadataStore.write] to keep them.

        Args:
            feature: Feature to rebase.
            df: Existing metadata to rebase (must contain `metaxy_provenance_by_field`).
            to_feature_version: Target feature version. Defaults to the current feature version.
                When different from the current version, the historical feature graph
                is loaded from the metadata store's system storage (must be pushed beforehand).

        Returns:
            Rebased frame with recalculated provenance, feature version, and project version.

        Raises:
            ValueError: If feature is a root feature (no upstream dependencies).
            ValueError: If `to_feature_version` is provided but not found in
                the store's system storage.
        """
        feature_key = self._resolve_feature_key(feature)
        plan = current_graph().get_feature_plan(feature_key)

        if not plan.deps:
            raise ValueError(
                f"Cannot rebase root feature {feature_key.to_string()}. "
                f"Root features have user-defined provenance that cannot be automatically rebased. "
                f"Re-run the computation pipeline to generate new data."
            )

        current = current_graph()
        current_version = current.get_feature_version(feature_key)

        if to_feature_version is None or to_feature_version == current_version:
            target_graph = current
            to_feature_version = current_version
            target_project_version = current.project_version
        else:
            from metaxy.metadata_store.system import FEATURE_VERSIONS_KEY
            from metaxy.metadata_store.system.storage import SystemTableStorage

            # Look up the project_version associated with to_feature_version
            storage = SystemTableStorage(self)
            versions_lazy = storage._read_system_metadata(FEATURE_VERSIONS_KEY)
            project_version_df = (
                versions_lazy.filter(
                    nw.col(METAXY_FEATURE_VERSION) == to_feature_version,
                    nw.col("feature_key") == feature_key.to_string(),
                )
                .select(METAXY_PROJECT_VERSION)
                .unique()
                .collect()
                .to_polars()
            )

            if project_version_df.height == 0:
                raise ValueError(
                    f"Feature version '{to_feature_version}' for '{feature_key.to_string()}' "
                    f"not found in system storage. Run `metaxy push` first."
                )

            target_project_version = project_version_df[METAXY_PROJECT_VERSION][0]

            # Load the feature graph for the target version
            target_graph = storage.load_graph_from_snapshot(target_project_version)

        # Null out data_version where it matches provenance (default, not user-overridden)
        # so _add_system_columns fills them from new provenance after write
        working = df.with_columns(  # ty: ignore[invalid-argument-type]
            nw.when(nw.col(METAXY_DATA_VERSION) == nw.col(METAXY_PROVENANCE))
            .then(nw.lit(None, dtype=nw.String))
            .otherwise(nw.col(METAXY_DATA_VERSION))
            .alias(METAXY_DATA_VERSION),
            nw.when(nw.col(METAXY_DATA_VERSION) == nw.col(METAXY_PROVENANCE))
            .then(nw.lit(None))
            .otherwise(nw.col(METAXY_DATA_VERSION_BY_FIELD))
            .alias(METAXY_DATA_VERSION_BY_FIELD),
        )

        # Drop all provenance/version columns — resolve_update will recalculate
        # provenance_by_field from the target graph's upstream metadata.
        columns_to_drop = [
            c
            for c in working.collect_schema().names()
            if c in {METAXY_PROVENANCE, METAXY_PROVENANCE_BY_FIELD, METAXY_FEATURE_VERSION, METAXY_PROJECT_VERSION}
        ]
        working = working.drop(*columns_to_drop)

        # Recalculate provenance under the target feature graph.
        # samples=None so resolve_update loads upstream and builds fresh provenance_by_field.
        with target_graph.use():
            increment = self.resolve_update(feature_key, skip_comparison=True, lazy=True)

        # Join fresh provenance onto the user data from the original frame.
        # Only take provenance columns from increment.new; everything else comes from working.
        id_columns = list(plan.feature.id_columns)
        provenance_columns = [METAXY_PROVENANCE, METAXY_PROVENANCE_BY_FIELD]
        rebased = working.lazy().join(
            increment.new.select([*id_columns, *provenance_columns]),
            on=id_columns,
            how="inner",
        )

        return rebased.with_columns(
            nw.lit(to_feature_version).alias(METAXY_FEATURE_VERSION),
            nw.lit(target_project_version).alias(METAXY_PROJECT_VERSION),
        )

    @abstractmethod
    def _read_feature(
        self,
        feature: CoercibleToFeatureKey,
        *,
        filters: Sequence[nw.Expr] | None = None,
        columns: Sequence[str] | None = None,
        **kwargs: Any,
    ) -> nw.LazyFrame[Any] | None:
        """
        Read metadata from THIS store only without using any fallbacks stores.

        Args:
            feature: Feature to read metadata for
            filters: List of Narwhals filter expressions for this specific feature.
            columns: Subset of columns to return
            **kwargs: Backend-specific parameters

        Returns:
            Narwhals LazyFrame with metadata, or None if feature not found in the store
        """
        pass

    def read_feature_schema_from_store(
        self,
        feature: CoercibleToFeatureKey,
    ) -> nw.Schema:
        """Read the schema for a feature from the store.

        Args:
            feature: Feature to read schema for

        Returns:
            Narwhals schema for the feature

        Raises:
            FeatureNotFoundError: If feature not found in the store
        """
        lazy = self.read(
            feature,
            allow_fallback=False,
        )
        return lazy.collect_schema()

    # ========== Feature Existence ==========

    def has_feature(
        self,
        feature: CoercibleToFeatureKey,
        *,
        check_fallback: bool = False,
    ) -> bool:
        """
        Check if feature exists in store.

        Args:
            feature: Feature to check
            check_fallback: If True, also check fallback stores

        Returns:
            True if feature exists, False otherwise
        """
        self._check_open()

        if self._read_feature(feature) is not None:
            return True

        # Check fallback stores
        if not check_fallback:
            return self._has_feature_impl(feature)
        else:
            for store in self.fallback_stores:
                if store.has_feature(feature, check_fallback=True):
                    return True

        return False

    @abstractmethod
    def _has_feature_impl(self, feature: CoercibleToFeatureKey) -> bool:
        """Implementation of _has_feature.

        Args:
            feature: Feature to check

        Returns:
            True if feature exists, False otherwise
        """
        pass

    @abstractmethod
    def display(self) -> str:
        """Return a human-readable display string for this store.

        Used in warnings, logs, and CLI output to identify the store.

        Returns:
            Display string (e.g., "DuckDBMetadataStore(database=/path/to/db.duckdb)")
        """
        pass

    def __repr__(self) -> str:
        """Return the display string with optional name prefix."""
        return self._format_display_with_name(self.display())

    def find_store_for_feature(
        self,
        feature_key: CoercibleToFeatureKey,
        *,
        check_fallback: bool = True,
    ) -> MetadataStore | None:
        """Find the store that contains the given feature.

        Args:
            feature_key: Feature to find
            check_fallback: Whether to check fallback stores when the feature
                is not found in the current store

        Returns:
            The store containing the feature, or None if not found
        """
        self._check_open()

        # Check if feature exists in this store
        if self.has_feature(feature_key):
            return self

        # Try fallback stores if enabled (opened on demand)
        if check_fallback:
            for store in self.fallback_stores:
                with store:
                    found = store.find_store_for_feature(feature_key, check_fallback=True)
                    if found is not None:
                        return found

        return None

    def get_store_metadata(
        self,
        feature_key: CoercibleToFeatureKey,
        *,
        check_fallback: bool = True,
    ) -> dict[str, Any]:
        """Arbitrary key-value pairs with useful metadata for logging purposes (like a path in storage).

        This method should not expose sensitive information.

        Args:
            feature_key: Feature to get metadata for
            check_fallback: Whether to check fallback stores when the feature
                is not found in the current store

        Returns:
            Dictionary with store-specific metadata. Contains:
            - `name`, `display`: The queried store (self)
            - `resolved_from`: Where the feature was actually found (may be a fallback store),
                includes `name`, `display`, and store-specific fields like `table_name` or `uri`
        """
        resolved_store = self.find_store_for_feature(feature_key, check_fallback=check_fallback)
        result: dict[str, Any] = {
            "name": self.name,
            "display": self.display(),
        }
        if resolved_store is not None:
            result["resolved_from"] = resolved_store.get_store_info(feature_key)
        return result

    def _get_store_metadata_impl(self, feature_key: CoercibleToFeatureKey) -> dict[str, Any]:
        """Implementation of get_store_metadata for this specific store type.

        Override in subclasses to return store-specific metadata.

        Args:
            feature_key: Feature to get metadata for

        Returns:
            Dictionary with store-specific metadata
        """
        return {}

    def get_store_info(self, feature_key: CoercibleToFeatureKey) -> dict[str, Any]:
        """Build a dictionary with store identification and feature-specific metadata.

        Args:
            feature_key: Feature to get metadata for

        Returns:
            Dictionary containing:
            - `name`: The store name
            - `type`: The fully qualified class name (module.classname)
            - `display`: Human-readable store description
            - Store-specific metadata from `_get_store_metadata_impl` (e.g., `table_name`, `uri`)
        """
        return {
            "name": self.name,
            "type": self.qualified_class_name,
            "display": self.display(),
            **self._get_store_metadata_impl(feature_key),
        }

    def calculate_input_progress(
        self,
        lazy_increment: LazyIncrement,
        feature_key: CoercibleToFeatureKey,
    ) -> float | None:
        """Calculate progress percentage from lazy increment.

        Uses the `input` field from LazyIncrement to count total input units
        and compares with `added` to determine how many are missing.

        Progress represents the percentage of input units that have been processed
        at least once. Stale samples (in `changed`) are counted as processed since
        they have existing metadata, even though they may need re-processing due to
        upstream changes.

        Args:
            lazy_increment: The lazy increment containing input and added dataframes.
            feature_key: The feature key to look up lineage information.

        Returns:
            Progress percentage (0-100), or None if input is not available.
        """
        if lazy_increment.input is None:
            return None

        key = self._resolve_feature_key(feature_key)
        graph = current_graph()
        plan = graph.get_feature_plan(key)

        # Get the columns that define input units from the feature plan
        input_id_columns = plan.input_id_columns

        # Count distinct input units using two separate queries
        # We can't use concat because input and added may have different schemas
        # (e.g., nullable vs non-nullable columns)
        total_units: int = lazy_increment.input.select(input_id_columns).unique().select(nw.len()).collect().item()

        if total_units == 0:
            return None  # No input available from upstream

        missing_units: int = lazy_increment.new.select(input_id_columns).unique().select(nw.len()).collect().item()

        processed_units = total_units - missing_units
        return (processed_units / total_units) * 100

    def copy_metadata(
        self,
        from_store: MetadataStore,
        features: Sequence[CoercibleToFeatureKey] | None = None,
        *,
        filters: Mapping[str, Sequence[nw.Expr]] | None = None,
        global_filters: Sequence[nw.Expr] | None = None,
        with_feature_history: bool = True,
        with_sample_history: bool = False,
    ) -> dict[str, int]:
        """Copy metadata from another store.

        Args:
            from_store: Source metadata store to copy from (must be opened for reading)
            features: Features to copy. Can be:

                - `None`: copies all features from the active graph

                - Sequence of specific features to copy

            filters: Dict mapping feature keys (as strings) to sequences of Narwhals filter expressions.
                These filters are applied when reading from the source store.
                Example: {"feature/key": [nw.col("x") > 10], "other/feature": [...]}
            global_filters: Sequence of Narwhals filter expressions applied to all features.
                These filters are combined with any feature-specific filters from `filters`.
                Example: [nw.col("sample_uid").is_in(["s1", "s2"])]
            with_feature_history: If True (default), include rows from all historical feature versions.
                If False, only copy rows with the current feature_version.
            with_sample_history: If True, include all historical materializations per sample.
                If False (default), deduplicate within `id_columns` groups by keeping latest.

        Returns:
            Dict with statistics: {"features_copied": int, "rows_copied": int}

        Raises:
            ValueError: If source or destination store is not open
            FeatureNotFoundError: If a specified feature doesn't exist in source store

        Examples:
            <!-- skip next -->
            ```py
            # Copy all features
            with source_store, dest_store.open("w"):
                stats = dest_store.copy_metadata(from_store=source_store)
            ```

            <!-- skip next -->
            ```py
            # Copy specific features
            with source_store, dest_store.open("w"):
                stats = dest_store.copy_metadata(
                    from_store=source_store,
                    features=[mx.FeatureKey("my_feature")],
                )
            ```

            <!-- skip next -->
            ```py
            # Copy with global filters applied to all features
            with source_store, dest_store.open("w"):
                stats = dest_store.copy_metadata(
                    from_store=source_store,
                    global_filters=[nw.col("id").is_in(["a", "b"])],
                )
            ```

            <!-- skip next -->
            ```py
            # Copy specific features with per-feature filters
            with source_store, dest_store.open("w"):
                stats = dest_store.copy_metadata(
                    from_store=source_store,
                    features=[
                        mx.FeatureKey("feature_a"),
                        mx.FeatureKey("feature_b"),
                    ],
                    filters={
                        "feature_a": [nw.col("field_a") > 10],
                        "feature_b": [nw.col("field_b") < 30],
                    },
                )
            ```
        """
        import logging

        logger = logging.getLogger(__name__)

        # Validate both stores are open
        if not self._is_open:
            raise ValueError('Destination store must be opened with store.open("w") before use')
        if not from_store._is_open:
            raise ValueError("Source store must be opened with store before use")

        return self._copy_metadata_impl(
            from_store=from_store,
            features=features,
            filters=filters,
            global_filters=global_filters,
            with_feature_history=with_feature_history,
            with_sample_history=with_sample_history,
            logger=logger,
        )

    def _copy_metadata_impl(
        self,
        from_store: MetadataStore,
        features: Sequence[CoercibleToFeatureKey] | None,
        filters: Mapping[str, Sequence[nw.Expr]] | None,
        global_filters: Sequence[nw.Expr] | None,
        with_feature_history: bool,
        with_sample_history: bool,
        logger,
    ) -> dict[str, int]:
        """Internal implementation of copy_metadata."""
        # Determine which features to copy
        features_to_copy: list[FeatureKey]
        if features is None:
            # Copy all features from active graph (features defined in current project)
            from metaxy.models.feature import FeatureGraph

            graph = FeatureGraph.get_active()
            features_to_copy = graph.list_features(only_current_project=True)
            logger.info(f"Copying all features from active graph: {len(features_to_copy)} features")
        else:
            # Convert all to FeatureKey using the adapter
            features_to_copy = [self._resolve_feature_key(item) for item in features]
            logger.info(f"Copying {len(features_to_copy)} specified features")

        # Copy metadata for each feature
        total_rows = 0
        features_copied = 0

        for feature_key in features_to_copy:
            try:
                # Build combined filters for this feature
                feature_filters: list[nw.Expr] = []

                # Add global filters
                if global_filters:
                    feature_filters.extend(global_filters)

                # Add feature-specific filters
                if filters:
                    feature_key_str = feature_key.to_string()
                    if feature_key_str in filters:
                        feature_filters.extend(filters[feature_key_str])

                # Read metadata from source with all filters applied
                source_lazy = from_store.read(
                    feature_key,
                    filters=feature_filters if feature_filters else None,
                    allow_fallback=False,
                    with_feature_history=with_feature_history,
                    with_sample_history=with_sample_history,
                )

                # Collect to narwhals DataFrame to get row count
                source_df = source_lazy.collect()
                row_count = len(source_df)

                if row_count == 0:
                    logger.warning(f"No rows found for {feature_key.to_string()}, skipping")
                    continue

                # Write to destination (preserving project_version and feature_version)
                self.write(feature_key, source_df, preserve_feature_version=True)

                features_copied += 1
                total_rows += row_count
                logger.info(f"Copied {row_count} rows for {feature_key.to_string()}")

            except FeatureNotFoundError:
                logger.warning(f"Feature {feature_key.to_string()} not found in source store, skipping")
                continue
            except Exception as e:
                logger.error(f"Error copying {feature_key.to_string()}: {e}", exc_info=True)
                raise

        logger.info(f"Copy complete: {features_copied} features, {total_rows} total rows")

        return {"features_copied": features_copied, "rows_copied": total_rows}
