"""
aaiclick.data.data_context - Function-based context management for ClickHouse client and Object lifecycle.

This module provides a context manager that manages the lifecycle of Objects created
within its scope, automatically cleaning up tables when the context exits.
"""

from __future__ import annotations

import re
import warnings
import weakref
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from contextvars import ContextVar
from dataclasses import replace
from datetime import datetime
from typing import TYPE_CHECKING, Any, cast

import pyarrow as pa

from aaiclick.locks import load_advisory_id, table_insert_lock
from aaiclick.oplog.oplog_api import oplog_record

from ..models import (
    AAI_ID_COLUMN,
    AAI_ID_INFO,
    ENGINE_MEMORY,
    FIELDTYPE_ARRAY,
    FIELDTYPE_DICT,
    FIELDTYPE_SCALAR,
    ColumnInfo,
    EngineType,
    FieldSpec,
    Schema,
    ValueDictType,
    ValueListType,
    ValueScalarType,
    ValueType,
    build_order_by_clause,
)
from ..scope import NamedScope, make_persistent_table_name
from ..sql_utils import quote_identifier
from .ch_client import ChClient, _ch_client_var, create_ch_client, get_ch_client
from .lifecycle import LocalLifecycleHandler, _lifecycle_var, get_data_lifecycle, register_table

if TYPE_CHECKING:
    from ..object import Object

# Per-resource ContextVars — each set by data_context() on entry, reset on exit.
# Resources owned by their respective modules:
#   ChClient        → ch_client.py  (_ch_client_var / get_ch_client)
#   LifecycleHandler→ lifecycle.py  (_lifecycle_var  / get_data_lifecycle)
#   Oplog           → oplog/collector.py (delegates to lifecycle handler)
_engine_var: ContextVar[EngineType] = ContextVar("engine", default=ENGINE_MEMORY)
_objects_var: ContextVar[dict[int, weakref.ref]] = ContextVar("objects")


def get_engine() -> EngineType:
    """Return the table engine for the active data context."""
    return _engine_var.get()


@asynccontextmanager
async def _maybe_insert_lock(table: str, name: str | None) -> AsyncIterator[None]:
    """Acquire the per-table insert lock when writing to a named (persistent)
    destination; no-op for fresh/temp destinations that cannot race."""
    if name is None:
        yield
        return
    async with table_insert_lock(await load_advisory_id(table)):
        yield


def incref(table_name: str) -> None:
    """Increment the reference count for a table, keeping it alive.

    Called automatically when a new Object is derived from an existing one
    (e.g. via operators or copy). Only needed when managing Object lifecycles
    manually outside of `data_context()`.

    Args:
        table_name: ClickHouse table name to retain.
    """
    lifecycle = get_data_lifecycle()
    if lifecycle is not None:
        lifecycle.incref(table_name)


def decref(table_name: str) -> None:
    """Decrement the reference count for a table, dropping it when it reaches zero.

    Called automatically when a derived Object is garbage collected or the
    enclosing `data_context()` exits. Only needed when managing Object
    lifecycles manually outside of `data_context()`.

    Args:
        table_name: ClickHouse table name to release.
    """
    lifecycle = get_data_lifecycle()
    if lifecycle is not None:
        lifecycle.decref(table_name)


def register_object(obj: Object) -> None:
    """Register an Object so it is marked stale when the enclosing context exits.

    Called automatically by `create_object()` and `create_object_from_value()`.
    There is no need to call this directly in normal usage.

    Args:
        obj: Object instance to track.
    """
    try:
        objects = _objects_var.get()
    except LookupError:
        return
    objects[id(obj)] = weakref.ref(obj)


async def delete_object(obj: Object) -> None:
    """Delete an Object's underlying ClickHouse table and mark the Object stale.

    After calling this, any further operations on `obj` will raise `RuntimeError`.
    Existing Python references to `obj` remain valid but unusable.

    Args:
        obj: Object instance to delete.
    """
    obj._stale = True
    obj._registered = False
    try:
        objects = _objects_var.get()
    except LookupError:
        objects = {}
    obj_id = id(obj)
    if obj_id in objects:
        del objects[obj_id]
    lifecycle = get_data_lifecycle()
    if lifecycle is not None:
        lifecycle.decref(obj.table)


@asynccontextmanager
async def data_context(
    engine: EngineType | None = None,
) -> AsyncIterator[None]:
    """Async context manager for standalone data operations.

    Sets per-resource ContextVars for the duration of the block:
    - ChClient (ch_client.py)
    - LocalLifecycleHandler (lifecycle.py)
    - EngineType and object registry (data_context.py)

    Always creates and owns a LocalLifecycleHandler. For orchestration job
    execution use orch_context() + task_scope() instead. Persistent named
    objects (``scope="global"``/``"job"``, ``open_object()``) require an
    orch_context — ``data_context()`` does not provide the SQL session
    that ``table_registry`` reads/writes need.

    Args:
        engine: ClickHouse table engine. Defaults to Memory (RAM).
    """
    ch_client = await create_ch_client()
    effective_engine = engine if engine is not None else ENGINE_MEMORY

    lifecycle = LocalLifecycleHandler(ch_client)
    await lifecycle.start()

    objects: dict[int, weakref.ref] = {}

    ch_token = _ch_client_var.set(ch_client)
    lc_token = _lifecycle_var.set(lifecycle)
    eng_token = _engine_var.set(effective_engine)
    obj_token = _objects_var.set(objects)

    try:
        yield
    finally:
        # Decref and stale-mark all tracked objects still alive.
        # Dead weakrefs (GC'd during context) were already decreffed by __del__.
        # Setting _stale prevents a later __del__ from double-decrefing.
        for obj_ref in objects.values():
            obj = obj_ref()
            if obj is not None:
                obj._stale = True
                if obj._registered and not obj.persistent:
                    if obj._owns_lifecycle_ref:
                        decref(obj.table)
                    obj._registered = False
        objects.clear()

        await lifecycle.stop()

        # Reset all ContextVars
        _objects_var.reset(obj_token)
        _engine_var.reset(eng_token)
        _lifecycle_var.reset(lc_token)
        _ch_client_var.reset(ch_token)


def get_engine_clause(engine: EngineType, order_by: str = "tuple()") -> str:
    """Get the ENGINE clause for table creation."""
    if engine == "Memory":
        return "ENGINE = Memory"
    return f"ENGINE = {engine} ORDER BY {order_by}"


_VALID_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")


def _validate_persistent_name(name: str) -> None:
    """Validate a persistent object name.

    Raises:
        ValueError: If name doesn't match [a-zA-Z_][a-zA-Z0-9_]*
    """
    if not _VALID_NAME_RE.match(name):
        raise ValueError(f"Invalid persistent name '{name}': must match [a-zA-Z_][a-zA-Z0-9_]*")


def _resolve_scope(name: str | None, scope: NamedScope | None) -> NamedScope | None:
    """Resolve the effective scope for a named object.

    Rules:
    - ``name is None``: unnamed temp table; ``scope`` must also be ``None``.
    - ``name`` set: requires an active ``orch_context()`` — persistent
      tables need the SQL ``table_registry`` row, which only the orch
      lifecycle handler writes. ``data_context()`` alone is rejected.
    - ``scope="job"``: requires an active orch job_id (raises ValueError
      otherwise — see scope.py).
    - ``name`` set, ``scope=None``: default to ``"job"`` (the only scope
      reachable inside an orch job).
    """
    if name is None:
        if scope is not None:
            raise ValueError("scope can only be set together with name")
        return None

    from aaiclick.orchestration.sql_context import _sql_engine_var

    if _sql_engine_var.get() is None:
        raise RuntimeError(
            "Persistent objects (name=...) require an active orch_context() — "
            "data_context() does not provide the SQL session that table_registry needs. "
            "Wrap your code in 'async with orch_context(): async with task_scope(...):' "
            "or drop the name= argument for a temp table."
        )

    if scope is not None:
        return scope
    lifecycle = get_data_lifecycle()
    if lifecycle is not None and lifecycle.current_job_id() is not None:
        return "job"
    return "global"


def _build_persistent_table(name: str, scope: NamedScope) -> str:
    """Validate ``name`` and build the full CH table name for a scoped object."""
    _validate_persistent_name(name)
    job_id = None
    if scope == "job":
        lifecycle = get_data_lifecycle()
        job_id = lifecycle.current_job_id() if lifecycle is not None else None
    return make_persistent_table_name(scope, name, job_id=job_id)


async def create_object(
    schema: Schema,
    engine: EngineType | None = None,
    name: str | None = None,
    scope: NamedScope | None = None,
) -> Object:
    """Create a new Object with a ClickHouse table using the specified schema.

    Args:
        schema: Schema dataclass with fieldtype, columns, engine, and order_by.
        engine: Table engine override. Priority (highest to lowest):
                ``name`` (named) → this param → ``schema.engine`` → context default.
                Ignored when ``name`` is set — named tables always use MergeTree.
        name: Optional name. When set, ``scope`` is required and selects the
              persistence tier — see ``scope`` below.
        scope: Required when ``name`` is set, must be ``None`` otherwise.
              ``"global"`` → ``p_<name>`` (user-managed, removed only by
              ``delete_persistent_object()``). ``"job"`` → ``j_<job_id>_<name>``
              (lives only as long as the active orch job; raises if no job is
              active).

    Returns:
        Object: New Object instance with created table
    """
    from ..object import Object

    effective_scope = _resolve_scope(name, scope)
    if effective_scope is not None:
        assert name is not None
        table_name = _build_persistent_table(name, effective_scope)
        obj = Object(table=table_name, schema=schema)
    else:
        obj = Object(schema=schema)

    # Fieldtype metadata for these columns lives in table_registry.schema_doc
    # (written by register_table below) rather than ClickHouse COMMENTs.
    def _column_ddl(col_name: str, col_def: ColumnInfo) -> str:
        ddl = f"{quote_identifier(col_name)} {col_def.ch_type()}"
        if col_def.default:
            ddl += f" DEFAULT {col_def.default}"
        return ddl

    column_defs = [_column_ddl(name, info) for name, info in schema.columns.items()]

    # Persistent tables always use MergeTree regardless of engine param or schema.engine.
    if obj.persistent:
        effective_engine = "MergeTree"
    else:
        effective_engine = engine or schema.engine or get_engine()

    order_by = schema.order_by or "tuple()"

    # Memory engine ignores ORDER BY — warn when both are combined.
    if effective_engine == "Memory" and schema.order_by:
        warnings.warn(
            f"order_by={schema.order_by!r} has no effect with Memory engine. "
            "Use engine='MergeTree' for ORDER BY support.",
            UserWarning,
            stacklevel=2,
        )
    engine_clause = get_engine_clause(effective_engine, order_by=order_by)

    create_or = "CREATE TABLE IF NOT EXISTS" if obj.persistent else "CREATE TABLE"
    create_query = f"""
    {create_or} {obj.table} (
        {", ".join(column_defs)}
    ) {engine_clause}
    """

    if not obj.persistent:
        obj._register()  # Write-ahead incref: register before CREATE TABLE
    register_object(obj)  # Object lifecycle: track for stale marking on exit
    await get_ch_client().command(create_query)

    # Register table in table_registry for cleanup worker.
    # In orch mode this records the job_id so all tables (including persistent)
    # are scoped to their job and cleaned up when the job expires. The
    # schema_doc carries the serialised SchemaView that _get_table_schema
    # reads back — replaces the per-column ClickHouse COMMENT YAML.
    # operation_log entries are recorded by higher-level callers (operators, ingest, etc.)
    from ..view_models import schema_to_view

    register_table(obj.table, schema_doc=schema_to_view(schema).model_dump_json())

    # Flush the lifecycle queue so the registry row is committed before the
    # caller reads schema_doc (e.g. via Object.data() → _get_table_schema).
    # The queue is async and order-sensitive for incref/decref, but registry
    # writes are standalone and idempotent; a synchronous flush here is
    # cheaper than forcing every read path to flush.
    lifecycle = get_data_lifecycle()
    if lifecycle is not None:
        await lifecycle.flush()

    return obj


def _infer_array_clickhouse_type(value: list) -> ColumnInfo:
    """Infer Array(T) ClickHouse type from a Python list for use as an Array column."""
    element_def = _infer_clickhouse_type(value)
    return ColumnInfo(element_def.type, array=True)


def _is_list_of_dicts(value: object) -> bool:
    """Check if a value is a non-empty list of dicts (nested array-of-objects)."""
    return isinstance(value, list) and bool(value) and isinstance(value[0], dict)


def _has_nested_dicts(record: dict) -> bool:
    """Check if a dict contains any list-of-dicts values (nested structures)."""
    return any(_is_list_of_dicts(v) for v in record.values())


def _flatten_nested_schema(sample: dict, prefix: str = "", array_depth: int = 0) -> dict[str, ColumnInfo]:
    """Recursively infer flat column schema from a nested record.

    Uses dot-star notation for nested array-of-objects levels.
    Each ``*`` level adds one Array() wrapper to the leaf column type.

    Args:
        sample: A sample record to infer schema from
        prefix: Column name prefix (e.g., ``"b.*."`` for nested fields)
        array_depth: Number of ``*`` nesting levels above this point

    Returns:
        Dict mapping flat column names to ColumnInfo
    """
    columns: dict[str, ColumnInfo] = {}
    for key, val in sample.items():
        col_name = f"{prefix}{key}"

        if _is_list_of_dicts(val):
            sub_cols = _flatten_nested_schema(val[0], f"{col_name}.*.", array_depth + 1)
            columns.update(sub_cols)
        elif isinstance(val, list):
            col_info = _infer_array_clickhouse_type(val)
            columns[col_name] = ColumnInfo(
                col_info.type,
                array=int(col_info.array) + array_depth,
                low_cardinality=col_info.low_cardinality,
            )
        else:
            col_info = _infer_clickhouse_type(val)
            if array_depth:
                columns[col_name] = ColumnInfo(
                    col_info.type,
                    array=array_depth,
                    low_cardinality=col_info.low_cardinality,
                )
            else:
                columns[col_name] = col_info
    return columns


def _flatten_nested_record(record: dict, prefix: str = "") -> dict:
    """Flatten a single nested record into dot-star notation.

    Converts nested list-of-dicts into parallel arrays (one row per record).

    Args:
        record: A dict possibly containing list-of-dicts values
        prefix: Column name prefix

    Returns:
        Flat dict with dot-star column names and array values
    """
    result: dict = {}
    for key, val in record.items():
        col_name = f"{prefix}{key}"

        if _is_list_of_dicts(val):
            sub_records = [_flatten_nested_record(item, f"{col_name}.*.") for item in val]
            if sub_records:
                for sub_key in sub_records[0]:
                    result[sub_key] = [sr[sub_key] for sr in sub_records]
        else:
            result[col_name] = val
    return result


def _infer_clickhouse_type(value: ValueScalarType | ValueListType) -> ColumnInfo:
    """Infer ClickHouse column type from Python value using pyarrow.

    Returns a ColumnInfo with nullable=False. Nullable columns must be
    created explicitly via Schema with ColumnInfo(type, nullable=True).
    String types default to LowCardinality for better storage and query performance.
    Datetime types map to DateTime64(3, 'UTC') for millisecond-precision UTC storage.
    """
    if isinstance(value, list):
        if not value:
            return ColumnInfo("String")

        arr = pa.array(value)
        pa_type = arr.type

        if pa.types.is_boolean(pa_type):
            return ColumnInfo("Bool")
        elif pa.types.is_timestamp(pa_type):
            return ColumnInfo("DateTime64(3, 'UTC')")
        elif pa.types.is_integer(pa_type):
            return ColumnInfo("Int64")
        elif pa.types.is_floating(pa_type):
            return ColumnInfo("Float64")
        else:
            return ColumnInfo("String")

    if isinstance(value, bool):
        return ColumnInfo("Bool")
    elif isinstance(value, datetime):
        return ColumnInfo("DateTime64(3, 'UTC')")
    elif isinstance(value, int):
        return ColumnInfo("Int64")
    elif isinstance(value, float):
        return ColumnInfo("Float64")
    elif isinstance(value, str):
        return ColumnInfo("String")
    else:
        return ColumnInfo("String")


def _apply_field_spec(col: ColumnInfo, spec: FieldSpec) -> ColumnInfo:
    """Apply a FieldSpec override to an inferred ColumnInfo."""
    return replace(
        col,
        type=spec.type if spec.type is not None else col.type,
        nullable=spec.nullable,
        low_cardinality=spec.low_cardinality,
    )


def _apply_field_specs(
    columns: dict[str, ColumnInfo],
    fields: dict[str, FieldSpec] | None,
) -> dict[str, ColumnInfo]:
    """Apply field specs to inferred columns, validating field names."""
    if not fields:
        return columns
    unknown = set(fields) - set(columns)
    if unknown:
        raise ValueError(f"fields references unknown columns: {sorted(unknown)}. Available columns: {sorted(columns)}")
    return {name: _apply_field_spec(col, fields[name]) if name in fields else col for name, col in columns.items()}


def _find_non_empty_nested_sample(records: list, key: str) -> dict:
    """Find a non-empty sample for a nested list-of-dicts field across records.

    When the first record has an empty list for a nested field, searches
    subsequent records for a non-empty sample to infer schema from.
    """
    for record in records:
        val = record[key]
        if _is_list_of_dicts(val):
            return val[0]
    return {}


async def _create_nested_object(
    val: dict,
    ch: ChClient,
    name: str | None,
    scope: NamedScope | None = None,
) -> Object:
    """Create an Object from a single dict with nested list-of-dicts values.

    Stores nested list-of-dicts as parallel Array columns using dot-star
    notation. For example:
    ``{a: 2, b: [{c: [1,2,3], d: 5}, {c: [4,5,6], d: 10}]}``
    becomes 1 row with columns ``a`` (Int64), ``b.*.c`` (Array(Array(Int64))),
    ``b.*.d`` (Array(Int64)).
    """
    flat = _flatten_nested_record(val)
    nested_cols = _flatten_nested_schema(val)

    columns = {}
    columns.update(nested_cols)

    schema = Schema(fieldtype=FIELDTYPE_DICT, columns=columns)
    obj = await create_object(schema, name=name, scope=scope)

    keys = list(flat.keys())
    async with _maybe_insert_lock(obj.table, name):
        await ch.insert(
            obj.table,
            [[v] for v in flat.values()],
            column_names=keys,
            column_oriented=True,
            column_type_names=[columns[k].ch_type() for k in keys],
        )

    return obj


async def _create_nested_records_object(
    val: list,
    ch: ChClient,
    name: str | None,
    scope: NamedScope | None = None,
) -> Object:
    """Create an Object from a list of dicts with nested list-of-dicts values.

    Each input record becomes one row. Nested list-of-dicts are stored as
    parallel Array columns using dot-star notation.
    """
    first_keys = set(val[0].keys())
    for i, record in enumerate(val[1:], 1):
        if set(record.keys()) != first_keys:
            raise ValueError(
                f"All records must have identical keys. "
                f"Record 0 has {sorted(first_keys)}, "
                f"record {i} has {sorted(record.keys())}"
            )

    # Infer schema from first record, using non-empty samples for nested fields
    sample = dict(val[0])
    for key in sample:
        if _is_list_of_dicts(sample[key]) and not sample[key]:
            found = _find_non_empty_nested_sample(val[1:], key)
            if found:
                sample[key] = [found]

    nested_cols = _flatten_nested_schema(sample)
    # List-of-records is dict-of-arrays — each column carries N values across
    # the input records, so ColumnInfo.fieldtype must be ARRAY (default is
    # SCALAR, which would make data() return only the first row).
    columns = {name_: ci.with_fieldtype(FIELDTYPE_ARRAY) for name_, ci in nested_cols.items()}

    schema = Schema(fieldtype=FIELDTYPE_DICT, columns=columns)
    obj = await create_object(schema, name=name, scope=scope)

    all_flat = [_flatten_nested_record(record) for record in val]
    keys = list(all_flat[0].keys())
    col_data = [[flat[key] for flat in all_flat] for key in keys]
    async with _maybe_insert_lock(obj.table, name):
        await ch.insert(
            obj.table,
            col_data,
            column_names=keys,
            column_oriented=True,
            column_type_names=[columns[k].ch_type() for k in keys],
        )

    return obj


async def create_object_from_value(
    val: ValueType,
    name: str | None = None,
    order_by: list[str] | None = None,
    fields: dict[str, FieldSpec] | None = None,
    scope: NamedScope | None = None,
    *,
    aai_id: bool = False,
) -> Object:
    """Create a new Object from Python values with automatic schema inference.

    Args:
        val: Value to create object from. Can be:
            - Object or View: Returned directly without modification
            - Scalar (int, float, bool, str): Creates single row
            - List of scalars: Creates multiple rows
            - Dict of scalars: Single row with columns per key
            - Dict of arrays: Multiple rows with columns per key
            - Dict/List with nested list-of-dicts: Flattened with dot-star notation
        name: Optional name. When set, ``scope`` is required and selects
              the persistence tier — see ``scope`` below.
        order_by: Optional list of column names for the table ORDER BY clause.
                  Empty input yields ``tuple()`` (ClickHouse no-sort form).
                  Example: ``order_by=['date']`` → ``ORDER BY (date)``
        fields: Optional per-column overrides for inferred types. Maps column
                names to ``FieldSpec`` instances that control nullable,
                low_cardinality, and type override.
                Example: ``fields={"name": FieldSpec(low_cardinality=True),
                "score": FieldSpec(nullable=True)}``
        scope: Required when ``name`` is set, must be ``None`` otherwise.
              ``"global"`` → ``p_<name>`` (user-managed, removed only by
              ``delete_persistent_object()``). ``"job"`` → ``j_<job_id>_<name>``
              (lives only as long as the active orch job; raises when called
              from pure ``data_context()``).
        aai_id: When ``True``, add an ``aai_id`` column (``UInt64`` with
              ``DEFAULT generateSnowflakeID()``). Each row gets a unique,
              monotonically-increasing 64-bit Snowflake assigned per-row by
              ClickHouse at insert time. Use ``view(order_by="aai_id").data()``
              to recover insertion order — handy for cross-table arithmetic
              that needs pair-stable row ordering.

              When the LEFT operand of a binary operator carries ``aai_id``,
              the result table propagates the LEFT side's ``aai_id`` values,
              so ``(a + b).view(order_by="aai_id")`` recovers the original
              row order.

    Returns:
        Object: New Object instance with data
    """
    from ..object import Object, View

    if isinstance(val, (Object, View)):
        return val

    ch = get_ch_client()
    order_by_clause = build_order_by_clause(order_by) if order_by is not None else None

    def _maybe_add_aai_id(columns: dict[str, ColumnInfo]) -> dict[str, ColumnInfo]:
        if not aai_id:
            return columns
        if AAI_ID_COLUMN in columns:
            raise ValueError(f"aai_id=True conflicts with user column '{AAI_ID_COLUMN}'")
        return {**columns, AAI_ID_COLUMN: AAI_ID_INFO}

    if isinstance(val, dict):
        if _has_nested_dicts(val):
            result = await _create_nested_object(val, ch, name, scope=scope)
            oplog_record(result.table, "create_from_value")
            return result

        has_arrays = any(isinstance(v, list) for v in val.values())

        if has_arrays:
            columns = {}
            array_len = None

            for key, value in val.items():
                if isinstance(value, list):
                    if array_len is None:
                        array_len = len(value)
                    elif len(value) != array_len:
                        raise ValueError(
                            f"All arrays must have same length. Expected {array_len}, got {len(value)} for key '{key}'"
                        )
                    col_def = _infer_clickhouse_type(value)
                else:
                    raise ValueError(
                        f"Dict of arrays requires all values to be lists. Key '{key}' has type {type(value).__name__}"
                    )
                columns[key] = col_def.with_fieldtype(FIELDTYPE_ARRAY)

            columns = _maybe_add_aai_id(_apply_field_specs(columns, fields))
            schema = Schema(fieldtype=FIELDTYPE_DICT, columns=columns, order_by=order_by_clause)
            obj = await create_object(schema, name=name, scope=scope)

            if array_len and array_len > 0:
                keys = list(val.keys())
                array_cols: list[list[Any]] = [cast("list[Any]", val[k]) for k in keys]
                async with _maybe_insert_lock(obj.table, name):
                    await ch.insert(
                        obj.table,
                        array_cols,
                        column_names=keys,
                        column_oriented=True,
                        column_type_names=[columns[k].ch_type() for k in keys],
                    )

        else:
            columns = {}
            for key, value in val.items():
                columns[key] = _infer_clickhouse_type(value)

            columns = _maybe_add_aai_id(_apply_field_specs(columns, fields))
            schema = Schema(fieldtype=FIELDTYPE_DICT, columns=columns, order_by=order_by_clause)
            obj = await create_object(schema, name=name, scope=scope)

            keys = list(val.keys())
            async with _maybe_insert_lock(obj.table, name):
                await ch.insert(
                    obj.table,
                    [[v] for v in val.values()],
                    column_names=keys,
                    column_oriented=True,
                    column_type_names=[columns[k].ch_type() for k in keys],
                )

    elif isinstance(val, list):
        if val and isinstance(val[0], dict):
            # Narrow: list-of-dicts (ValueRecordType). pyright can't infer this
            # from isinstance(val[0], dict) alone.
            records = cast("list[ValueDictType]", val)
            if _has_nested_dicts(records[0]):
                result = await _create_nested_records_object(records, ch, name)
                oplog_record(result.table, "create_from_value")
                return result

            # Records format: list of dicts with possible Array fields
            first_keys = set(records[0].keys())
            for i, record in enumerate(records[1:], 1):
                if set(record.keys()) != first_keys:
                    raise ValueError(
                        f"All records must have identical keys. "
                        f"Record 0 has {sorted(first_keys)}, "
                        f"record {i} has {sorted(record.keys())}"
                    )

            columns = {}
            keys = list(records[0].keys())
            for key in keys:
                sample: Any = records[0][key]
                if isinstance(sample, list):
                    # Find a non-empty sample for better type inference
                    if not sample:
                        for record in records[1:]:
                            candidate = record[key]
                            if isinstance(candidate, list) and candidate:
                                sample = candidate
                                break
                    columns[key] = _infer_array_clickhouse_type(sample).with_fieldtype(FIELDTYPE_ARRAY)
                else:
                    columns[key] = _infer_clickhouse_type(sample).with_fieldtype(FIELDTYPE_ARRAY)

            columns = _maybe_add_aai_id(_apply_field_specs(columns, fields))
            schema = Schema(fieldtype=FIELDTYPE_DICT, columns=columns, order_by=order_by_clause)
            obj = await create_object(schema, name=name, scope=scope)

            col_data: list[list[Any]] = [[record[key] for record in records] for key in keys]
            async with _maybe_insert_lock(obj.table, name):
                await ch.insert(
                    obj.table,
                    col_data,
                    column_names=keys,
                    column_oriented=True,
                    column_type_names=[columns[k].ch_type() for k in keys],
                )
        else:
            # Narrow: list of scalars (ValueListType).
            scalars = cast(ValueListType, val)
            col_def = _infer_clickhouse_type(scalars)
            columns = {"value": col_def.with_fieldtype(FIELDTYPE_ARRAY)}
            columns = _maybe_add_aai_id(_apply_field_specs(columns, fields))
            col_def = columns["value"]
            schema = Schema(
                fieldtype=FIELDTYPE_ARRAY,
                columns=columns,
                order_by=order_by_clause,
            )
            obj = await create_object(schema, name=name, scope=scope)

            if scalars:
                async with _maybe_insert_lock(obj.table, name):
                    await ch.insert(
                        obj.table,
                        [scalars],
                        column_names=["value"],
                        column_oriented=True,
                        column_type_names=[col_def.ch_type()],
                    )

    else:
        col_def = _infer_clickhouse_type(val)
        columns = {"value": col_def}
        columns = _apply_field_specs(columns, fields)
        col_def = columns["value"]
        schema = Schema(
            fieldtype=FIELDTYPE_SCALAR,
            columns=columns,
            order_by=order_by_clause,
        )
        obj = await create_object(schema, name=name, scope=scope)

        async with _maybe_insert_lock(obj.table, name):
            await ch.insert(
                obj.table,
                [[val]],
                column_names=["value"],
                column_oriented=True,
                column_type_names=[col_def.ch_type()],
            )

    oplog_record(obj.table, "create_from_value")
    return obj


async def open_object(name: str, scope: NamedScope = "global") -> Object:
    """Open an existing persistent Object by name.

    Args:
        name: Persistent name (without prefix).
        scope: Persistence tier the object was created with — ``"global"`` →
               looks up ``p_<name>``; ``"job"`` → looks up
               ``j_<job_id>_<name>`` using the active orch job. Defaults to
               ``"global"`` since that was the historical behavior.

    Returns:
        Object with schema loaded from ClickHouse.

    Raises:
        ValueError: If name is invalid.
        RuntimeError: If table does not exist.
    """
    from ..object import Object
    from ..object.ingest import _get_table_schema

    table_name = _build_persistent_table(name, scope)
    ch = get_ch_client()

    result = await ch.command(f"EXISTS TABLE {table_name}")
    if not result:
        raise RuntimeError(f"Persistent object '{name}' does not exist (table {table_name})")

    fieldtype, columns = await _get_table_schema(table_name, ch)
    schema = Schema(fieldtype=fieldtype, columns=columns)
    obj = Object(table=table_name, schema=schema)
    register_object(obj)
    return obj


async def delete_persistent_object(name: str, scope: NamedScope = "global") -> None:
    """Drop a persistent table by name.

    Args:
        name: Persistent name (without prefix).
        scope: Tier the object was created with — ``"global"`` drops
               ``p_<name>``; ``"job"`` drops ``j_<job_id>_<name>``.

    Raises:
        ValueError: If name is invalid.
    """
    table_name = _build_persistent_table(name, scope)
    await get_ch_client().command(f"DROP TABLE IF EXISTS {table_name}")


async def delete_persistent_objects(
    after: datetime | None = None,
    before: datetime | None = None,
) -> list[str]:
    """Drop persistent tables filtered by creation time.

    Uses ClickHouse ``system.tables.metadata_modification_time`` to
    determine when each table was created.

    Args:
        after: Drop tables created at or after this time (inclusive).
        before: Drop tables created before this time (exclusive).

    Returns:
        List of deleted persistent names (without ``p_`` prefix).

    Raises:
        ValueError: If neither ``after`` nor ``before`` is specified.
    """
    if after is None and before is None:
        raise ValueError(
            "At least one of 'after' or 'before' must be specified "
            "to prevent accidental deletion of all persistent objects"
        )
    ch = get_ch_client()
    conditions = [
        "database = currentDatabase()",
        r"name LIKE 'p\_%'",
    ]
    if after is not None:
        after_str = after.strftime("%Y-%m-%d %H:%M:%S")
        conditions.append(f"metadata_modification_time >= '{after_str}'")
    if before is not None:
        before_str = before.strftime("%Y-%m-%d %H:%M:%S")
        conditions.append(f"metadata_modification_time < '{before_str}'")

    where = " AND ".join(conditions)
    result = await ch.query(f"SELECT name FROM system.tables WHERE {where}")
    names = [row[0] for row in result.result_rows]

    for table_name in names:
        await ch.command(f"DROP TABLE IF EXISTS {table_name}")

    return [n[2:] for n in names]


async def list_persistent_objects() -> list[str]:
    """List all persistent object names.

    Returns:
        List of persistent names (without ``p_`` prefix).
    """
    result = await get_ch_client().query(
        "SELECT name FROM system.tables "
        "WHERE database = currentDatabase() "
        r"AND name LIKE 'p\_%'"
    )
    return [row[0][2:] for row in result.result_rows]
