# ======================================================================
# (A) FILE PATH & IMPORT PATH
# depths/core/logger.py  →  import path: depths.core.logger
# ======================================================================

# ======================================================================
# (B) FILE OVERVIEW (concept & significance in v0.1.0)
# High-level façade that wires together the six OTel tables (spans,
# span_events, span_links, logs, metrics_points, metrics_hist) with:
#   • One LogProducer + LogAggregator per table (validation → Delta writes)
#   • OTLPMapper for JSON → row dict projection
#   • A background “shipper” that seals a completed UTC day and uploads
#     to S3, then verifies remote rowcounts (optional)
#   • Reader helpers returning Polars LazyFrames from local disk or S3
#
# Responsibilities:
#   - Own “instance” directories on local disk (<instance_root>/…)
#   - Manage lifecycle (start/stop) for all aggregators
#   - Handle UTC day rollover (retarget writers, enqueue yesterday for ship)
#   - Provide OTLP JSON ingest convenience APIs
#   - Provide named-predicate readers with pushdown-friendly predicates
#
# Why this matters:
#   DepthsLogger is the single entry-point most users code against. It
#   binds configuration, ingestion, storage, shipping, and lightweight
#   analytics into one place for v0.1.0.
# ======================================================================

# ======================================================================
# (C) IMPORTS & GLOBALS (what & why)
# atexit, signal, threading, asyncio, time → lifecycle & background services
# pathlib.Path, os                         → instance directory scaffolding
# datetime as _dt                          → UTC day, timestamps, timers
# polars as pl                             → LazyFrame readers
# re                                       → day-format validation
# depths.core.schema                       → EventSchema + concrete table schemas
# depths.core.config                       → options & configs (+ JSON save/load; S3Config)
# depths.core.producer / aggregator        → per-table ingestion & persistence
# depths.core.shipper                      → day sealing/upload/verify pipeline
# depths.core.otlp_mapper                  → OTLP JSON → rows
#
# Globals defined in this file:
#   _DAY_RE: regex for 'YYYY-MM-DD' validation for day-directory names
# ======================================================================

from __future__ import annotations

import atexit, asyncio, signal, threading, time, re, queue
from pathlib import Path
from typing import Any, Dict, Mapping, Optional, Tuple, Literal, Iterable

import os
import datetime as _dt
import polars as pl

from depths.core.schema import (
    EventSchema,
    SPAN_SCHEMA, SPAN_EVENT_SCHEMA, SPAN_LINK_SCHEMA,
    LOG_SCHEMA, METRIC_POINT_SCHEMA, METRIC_HIST_SCHEMA,
)
from depths.core.config import (
    DepthsLoggerOptions, LogProducerConfig, LogAggregatorConfig,
    _save_json, _load_json, S3Config,
)
from depths.core.producer import LogProducer
from depths.core.aggregator import LogAggregator
from depths.core.shipper import ship_day, ShipperOptions
from depths.core.otlp_mapper import OTLPMapper

# Doc:
# Strict regex validator for UTC day strings in ISO format 'YYYY-MM-DD'.
# Used to guard shipping and discovery over day directories.

# --- DEVELOPER NOTES -----------------------------------------------------
# - Keep the format stable across v0.1.x; readers and shipper logic depend on it.

_DAY_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")

def _today_utc_str() -> str:
    """
    Get today’s date in UTC as 'YYYY-MM-DD'.

    Returns:
        ISO date string (UTC) used for staging-day directory names and rollover checks.
    """
    # --- DEVELOPER NOTES -------------------------------------------------
    # - Centralized so any future change in formatting is one place.

    return _dt.datetime.now(tz=_dt.UTC).strftime("%Y-%m-%d")

def _path_startswith(p: str, prefix: str) -> bool:
    """
    Portable check that absolute path `p` is under `prefix`.

    Args:
        p:      Candidate path (file/dir).
        prefix: Directory prefix to test containment against.

    Returns:
        True when p resolves under prefix (case-normalized on Windows).
    """
    # --- DEVELOPER NOTES -------------------------------------------------
    # - Avoids naive string startswith issues across platforms and slashes.

    p_norm = str(Path(p).resolve())
    pref_norm = str(Path(prefix).resolve())
    if os.name == "nt":
        p_norm = p_norm.lower()
        pref_norm = pref_norm.lower()
    return p_norm.startswith(pref_norm.rstrip("/\\") + os.sep)

class DepthsLogger:
    """
    Unified OTel-native, S3-native telemetry logger
    Covering ingestion to eventual persistence on S3.

    Overview (v0.1.0 role):
        Owns one Producer+Aggregator per table under a single “instance”
        directory tree. On construction, prepares today’s UTC day staging
        paths, merges options (persisted vs provided), wires schemas into
        per-table configs, installs atexit/signal hooks, optionally starts
        aggregators and the background shipper, and replays unshipped days.

    Attributes (selected):
        _instance_id:    Logical instance name (namespace for S3 layout).
        _instance_root:  Filesystem root for configs/index/staging.
        _schemas:        Table name → EventSchema mapping.
        _producers/_aggs:Per-table Producer/Aggregator instances.
        _opts:           DepthsLoggerOptions effective configuration.
        _current_day:    Current UTC day string. Rollover retargets aggregators.
        _mapper:         Stateless OTLP JSON → rows mapper.
        _shipper_*:      Background shipping queue/thread coordination.
    """
    # --- DEVELOPER NOTES -------------------------------------------------
    # - No asyncio in aggregators; `astop` uses a simple thread hop.
    # - All paths are UTC-day rooted; current day is never shipped.
    # - Readers default to local tables, but will fall back to S3 when requested.

    @staticmethod
    def _instance_dirs(root: Path) -> Dict[str, Path]:
        """
        Compute canonical subpaths under an instance root.

        Args:
            root: Base directory for this Depths instance.

        Returns:
            Mapping with keys: root, configs, index_dir, index_json, index_jsonl, staging_days.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Paths are used widely; keep names stable for cross-module references.

        return {
            "root": root,
            "configs": root / "configs",
            "index_dir": root / "index",
            "index_json": root / "index" / "day.json",
            "index_jsonl": root / "index" / "days.jsonl",
            "staging_days": root / "staging" / "days",
        }

    @staticmethod
    def _ensure_dirs(root: Path) -> Dict[str, Path]:
        """
        Create required directories and initialize index files if missing.

        Args:
            root: Instance root.

        Returns:
            Same mapping as `_instance_dirs`, guaranteed to exist on disk.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Initializes 'day.json' (snapshot) and 'days.jsonl' (append-only log).

        d = DepthsLogger._instance_dirs(root)
        for k in ("configs", "index_dir", "staging_days"):
            d[k].mkdir(parents=True, exist_ok=True)
        if not d["index_json"].exists():
            d["index_json"].write_text("[]")
        if not d["index_jsonl"].exists():
            d["index_jsonl"].write_text("")
        return d

    @staticmethod
    def _local_day_path(root: Path, day: str) -> str:
        """
        Build the absolute local path to a UTC day’s root directory.

        Args:
            root: Instance root.
            day:  'YYYY-MM-DD' UTC day.

        Returns:
            Absolute string path to '<root>/staging/days/<day>'.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Used both by writers (current day) and shipper (past days).

        return str(DepthsLogger._instance_dirs(root)["staging_days"] / day)

    @staticmethod
    def _otel_table_path(day_root: str, sub: str) -> str:
        """
        Resolve a per-table Delta path under a UTC day root.

        Args:
            day_root: Absolute day directory.
            sub:      Table subpath ('spans', 'logs', ...).

        Returns:
            '<day_root>/otel/<sub>' path string.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Aligns with shipper’s discovery rules and S3 layout.

        return str(Path(day_root) / "otel" / sub)

    @staticmethod
    def _like_expr(col: str, pattern: str | None) -> pl.Expr | None:
        """
        Literal substring predicate builder for Polars.

        Args:
            col:     Column name.
            pattern: Substring to look for (no regex). None → no predicate.

        Returns:
            A Polars expression or None.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Uses `literal=True` so user inputs aren’t treated as regex.

        if not pattern:
            return None
        return pl.col(col).cast(pl.Utf8).str.contains(pattern, literal=True)  # literal contains, not regex

    @staticmethod
    def _eq_expr(col: str, value) -> pl.Expr | None:
        """
        Equality predicate builder for Polars.

        Args:
            col:   Column name.
            value: Value to compare; None → no predicate.

        Returns:
            Polars expression or None.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Keeps type as-is; upstream mapping/producer enforce dtypes.

        if value is None:
            return None
        return pl.col(col) == value

    @staticmethod
    def _ge_expr(col: str, value) -> pl.Expr | None:
        """
        Greater-or-equal predicate builder for Polars.

        Args:
            col:   Column name.
            value: Lower bound; None → no predicate.

        Returns:
            Polars expression or None.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Used for severity number & similar thresholds.

        if value is None:
            return None
        return pl.col(col) >= value

    @staticmethod
    def _time_range_expr(col: str, ms_from: int | None, ms_to: int | None) -> pl.Expr | None:
        """
        Build an inclusive time-window predicate (epoch ms).

        Args:
            col:     Column name holding epoch milliseconds.
            ms_from: Inclusive start in ms; None to omit.
            ms_to:   Inclusive end in ms; None to omit.

        Returns:
            Conjunction expression or None if both bounds are None.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Uses integer cast to ensure the pushdown matches column dtype.

        conds: list[pl.Expr] = []
        if ms_from is not None:
            conds.append(pl.col(col) >= int(ms_from))
        if ms_to is not None:
            conds.append(pl.col(col) <= int(ms_to))
        if not conds:
            return None
        expr = conds[0]
        for c in conds[1:]:
            expr = expr & c
        return expr

    def __init__(
        self,
        *,
        instance_id: str = "default",
        instance_dir: str = "./depths_logs",
        s3: S3Config | None = None,
        options: DepthsLoggerOptions | None = None,
    ) -> None:
        """
        Construct a DepthsLogger instance and wire all dependencies.

        Behavior (summary):
            1) Resolve instance root (optionally reusing on-disk instance.json).
            2) Ensure directory layout and bootstrap index files.
            3) Load/merge/persist options.json (disk ⟷ provided).
            4) Build six EventSchemas and per-table Producer+Aggregator.
            5) Prepare UTC-day staging root and mark current day.
            6) Install atexit/signal handlers as configured.
            7) Optionally autostart aggregators and shipper; enqueue past days.

        Args:
            instance_id:  Logical instance namespace.
            instance_dir: Base directory to contain the instance.
            s3:           S3Config for shipping & S3 readers (optional).
            options:      Overrides for DepthsLoggerOptions (merged with saved).
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - “Saved vs provided” merging treats None as “no override”.
        # - All six tables share the same Producer/Aggregator templates; only
        #   schema and table_path differ per table.

        self._opts = options or DepthsLoggerOptions()
        self._instance_id = instance_id
        self._instance_root = (Path(instance_dir).expanduser() / instance_id)

        inst_path = (self._instance_root / "configs" / "instance.json")
        on_disk_inst = _load_json(inst_path)
        if on_disk_inst:
            self._instance_id = on_disk_inst.get("instance_id", instance_id)
            root_str = on_disk_inst.get("instance_root") or str(self._instance_root)
            self._instance_root = Path(root_str)

        self._s3: S3Config | None = s3
        paths = self._ensure_dirs(self._instance_root)
        if not on_disk_inst:
            _save_json(
                paths["configs"] / "instance.json",
                {"instance_id": self._instance_id, "instance_root": str(self._instance_root), "version": 3},
            )

        saved_opts = DepthsLoggerOptions.load_from_dir(paths["configs"])
        if options is None:
            effective_opts = saved_opts or DepthsLoggerOptions()
        else:
            if saved_opts is None:
                effective_opts = options
            else:
                merged = saved_opts.to_dict()
                new = options.to_dict()
                for k, v in new.items():
                    if v is not None:
                        merged[k] = v
                effective_opts = DepthsLoggerOptions.from_dict(merged)
        effective_opts.save_to_dir(paths["configs"])
        self._opts = effective_opts

        self._schemas: Dict[str, EventSchema] = {
            "spans": SPAN_SCHEMA,
            "span_events": SPAN_EVENT_SCHEMA,
            "span_links": SPAN_LINK_SCHEMA,
            "logs": LOG_SCHEMA,
            "metrics_points": METRIC_POINT_SCHEMA,
            "metrics_hist": METRIC_HIST_SCHEMA,
        }

        today = _today_utc_str()
        self._current_day = today
        day_root = self._local_day_path(self._instance_root, today)
        Path(day_root).mkdir(parents=True, exist_ok=True)

        base_prod_cfg: LogProducerConfig = (self._opts.producer_config or LogProducerConfig())
        base_agg_cfg: LogAggregatorConfig = (self._opts.aggregator_config or LogAggregatorConfig())

        self._producers: Dict[str, LogProducer] = {}
        self._aggs: Dict[str, LogAggregator] = {}

        for name, schema in self._schemas.items():
            table_path = self._otel_table_path(day_root, name)

            prod_cfg = base_prod_cfg.override(schema=schema)
            agg_cfg = base_agg_cfg.override(schema=schema, table_path=table_path)

            prod = LogProducer(prod_cfg)
            agg = LogAggregator(prod, agg_cfg)

            self._producers[name] = prod
            self._aggs[name] = agg

        self._started = False
        self._lock = threading.RLock()

        self._mapper = OTLPMapper()

        self._shipper_stop = threading.Event()
        self._ship_queue: "queue.Queue[str]" = queue.Queue(maxsize=64)
        self._shipper_thread: Optional[threading.Thread] = None

        if self._opts.atexit_hook:
            atexit.register(self._atexit_stop)
        if self._opts.install_signal_handlers:
            self._install_signal_handlers()

        if self._opts.auto_start:
            self.start()
        if self._opts.shipper_enabled:
            self._start_shipper()
            self._enqueue_past_days()


    def start(self) -> None:
        """
        Start all table aggregators (idempotent).

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Writer is started inside each Aggregator.start().

        with self._lock:
            if self._started:
                return
            for agg in self._aggs.values():
                agg.start()
            self._started = True

    def stop(self, *, flush: Literal["auto","all","none"] = "auto") -> None:
        """
        Stop all aggregators and background shipper (idempotent).

        Args:
            flush: 'none' to stop immediately,
                'auto' to wait briefly for a final quiet/age flush,
                'all' behaves like 'auto' (v0.1 uses same path).

        Side effects:
            - Joins threads with small bounded waits; best-effort shutdown.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - 'auto' sleep uses min(max_age_s, quiet_flush_s, options cap).
        # - Always disarms shipper after aggregators stop.

        with self._lock:
            if not self._started:
                self._stop_shipper()
                return
            for agg in self._aggs.values():
                if flush == "none":
                    agg.stop(flush_remaining=False)
                else:
                    wait = max(agg._cfg.quiet_flush_s or 0.0, agg._cfg.max_age_s or 0.0)  # type: ignore
                    wait = min(self._opts.max_auto_flush_wait_s, wait)
                    if wait > 0:
                        time.sleep(wait)
                    agg.stop(flush_remaining=True)
            self._started = False
        self._stop_shipper()

    async def astop(self, *, flush: Literal["auto","all","none"] = "auto") -> None:
        """
        Async-friendly wrapper to stop via a worker thread.

        Args:
            flush: See `stop`.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Avoids mixing asyncio with the aggregator’s thread model.

        await asyncio.to_thread(self.stop, flush=flush)

    def _start_shipper(self) -> None:
        """
        Start the background shipper thread if not already running.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Thread name 'depths-shipper'; daemon for process exit safety.

        if self._shipper_thread and self._shipper_thread.is_alive():
            return
        self._shipper_stop.clear()
        t = threading.Thread(target=self._shipper_loop, name="depths-shipper", daemon=True)
        t.start()
        self._shipper_thread = t

    def _stop_shipper(self) -> None:
        """
        Stop the background shipper thread and join briefly.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Uses a sentinel on the queue to request exit.

        self._shipper_stop.set()
        try:
            self._ship_queue.put_nowait("__STOP__")
        except Exception:
            pass
        if self._shipper_thread and self._shipper_thread.is_alive():
            self._shipper_thread.join(timeout=10.0)

    def _enqueue_past_days(self) -> None:
        """
        Scan local staging/days and enqueue all past days for shipping.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Skips today (by design) and non-day directories.

        days_root = self._instance_dirs(self._instance_root)["staging_days"]
        if not days_root.exists():
            return
        today = _today_utc_str()
        for p in sorted(days_root.iterdir()):
            if p.is_dir() and p.name < today and _DAY_RE.match(p.name):
                self.enqueue_ship(p.name)

    def enqueue_ship(self, day: str) -> None:
        """
        Request shipping for a given UTC day (no-op if invalid/unavailable).

        Args:
            day: 'YYYY-MM-DD' string. Must not be the current day.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Drops silently if no S3 is configured or the queue is full.

        if not self._s3 or not _DAY_RE.match(day):
            return
        if day == self._current_day:
            return
        try:
            self._ship_queue.put_nowait(day)
        except queue.Full:
            pass

    def ship_now(self, day: str) -> Dict:
        """
        Synchronously ship a completed UTC day (seal → upload → verify → cleanup).

        Args:
            day: Past day in 'YYYY-MM-DD'. Current day is rejected.

        Returns:
            Compact result dict with local/remote rows and status.

        Raises:
            ValueError: If S3 isn’t configured, day invalid, or day is current.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Thin wrapper around `ship_day`; mostly validates and locates day path.

        if not self._s3:
            raise ValueError("No S3 configured")
        if not _DAY_RE.match(day):
            raise ValueError("Invalid day format")
        if day == self._current_day:
            raise ValueError("Won't ship the current day")
        local_day_path = self._local_day_path(self._instance_root, day)
        return ship_day(
            instance_id=self._instance_id,
            instance_root=str(self._instance_root),
            day=day,
            local_day_path=local_day_path,
            s3=self._s3,
            opts=ShipperOptions(
                verify_grace_s=self._opts.verify_grace_s,
                verify_timeout_s=self._opts.verify_timeout_s,
                upload_max_workers=self._opts.upload_max_workers,
            ),
        )

    def _shipper_loop(self) -> None:
        """
        Background shipper worker: drains day-requests and ships them.

        Behavior:
            - Waits for all writer queues to drain and no aggregator targeting
            the day’s path, then calls `ship_day(...)`.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Double-guards against accidentally shipping the current day.
        # - Uses small sleeps to re-check drain/retarget conditions.

        while not self._shipper_stop.is_set():
            try:
                day = self._ship_queue.get(timeout=0.5)
            except queue.Empty:
                continue
            if day == "__STOP__":
                break
            if not isinstance(day, str) or not _DAY_RE.match(day) or not self._s3:
                continue
            if day == self._current_day:
                continue

            local_day_path = self._local_day_path(self._instance_root, day)
            while True:
                all_empty = all(agg.metrics.writer_queue_size == 0 for agg in self._aggs.values())
                none_targeting = all(not _path_startswith(agg.table_path, local_day_path) for agg in self._aggs.values())
                if all_empty and none_targeting:
                    break
                if self._shipper_stop.is_set():
                    return
                time.sleep(0.25)

            try:
                ship_day(
                    instance_id=self._instance_id,
                    instance_root=str(self._instance_root),
                    day=day,
                    local_day_path=local_day_path,
                    s3=self._s3,
                    opts=ShipperOptions(
                        verify_grace_s=self._opts.verify_grace_s,
                        verify_timeout_s=self._opts.verify_timeout_s,
                        upload_max_workers=self._opts.upload_max_workers,
                    ),
                )
            finally:
                self._ship_queue.task_done()

    def _maybe_rollover_utc(self) -> None:
        """
        Detect UTC day change and retarget all aggregators to the new day.

        Behavior:
            - Creates the new day root, retargets each aggregator to its
            new '<new_day>/otel/<table>' path (with table init).
            - Optionally enqueues the previous day for shipping after a delay.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Aggregators capture target table path at schedule time to avoid
        #   misrouting batches already enqueued before rollover.

        today = _today_utc_str()
        if today == self._current_day:
            return
        old_day = self._current_day
        self._current_day = today
        new_day_root = self._local_day_path(self._instance_root, today)
        Path(new_day_root).mkdir(parents=True, exist_ok=True)
        
        for name, agg in self._aggs.items():
            new_path = self._otel_table_path(new_day_root, name)
            agg.retarget_table_path(new_path, initialize=True)
        
        if self._opts.shipper_enabled and _DAY_RE.match(old_day):
            delay = max(0.0, float(self._opts.ship_delay_after_rollover_s))
            threading.Timer(delay, lambda: self.enqueue_ship(old_day)).start()

    def _ensure_started_if_lazy(self) -> None:
        """
        Start aggregators on-demand if lazy-start is enabled.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Called at the head of each ingest_* method.

        if not self._started and self._opts.lazy_start_on_first_log:
            self.start()

    def ingest_span(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["spans"].ingest(row)

    def ingest_span_event(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["span_events"].ingest(row)

    def ingest_span_link(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["span_links"].ingest(row)

    def ingest_log(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["logs"].ingest(row)

    def ingest_metric_point(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["metrics_points"].ingest(row)

    def ingest_metric_hist(self, row: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
        self._ensure_started_if_lazy()
        self._maybe_rollover_utc()
        return self._producers["metrics_hist"].ingest(row)

    def ingest_otlp_traces_json(self, payload: Mapping[str, Any], *, project_id: Optional[str] = None) -> Dict[str, int]:
        """
        Decode already-parsed OTLP Traces payload into rows and ingest them.

        Overview:
            • Projects `payload` to three row lists via OTLPMapper.map_traces(...)
            → (spans, events, links).
            • For each row, calls the corresponding ingest_* method:
                ingest_span, ingest_span_event, ingest_span_link.
            • Aggregates total accepted vs rejected counts across all three categories.
            • Returns summary counts including the raw lengths of each list.

        Args:
            payload: A Mapping representing a decoded OTLP ExportTraceServiceRequest.
            project_id: Optional tenancy override passed through to the mapper.

        Returns:
            Dict with keys:
            {'accepted', 'rejected', 'spans', 'events', 'links'}

        Notes:
            - The per-row ingest_* methods internally handle lazy-start and UTC
            rollover checks; this method does not duplicate those concerns.
            - `dict(payload)` defensively copies to avoid caller-side aliasing.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Accepted/rejected are tallied across spans + events + links.
        # - The 'spans'/'events'/'links' counts in the return value are the number
        #   of mapped rows, not the number successfully persisted.

        spans, events, links = self._mapper.map_traces(dict(payload), project_id=project_id)
        ca = cr = 0
        for r in spans:
            ok, _ = self.ingest_span(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        for r in events:
            ok, _ = self.ingest_span_event(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        for r in links:
            ok, _ = self.ingest_span_link(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        return {"accepted": ca, "rejected": cr, "spans": len(spans), "events": len(events), "links": len(links)}

    def ingest_otlp_logs_json(self, payload: Mapping[str, Any], *, project_id: Optional[str] = None) -> Dict[str, int]:
        """
        Decode already-parsed OTLP Logs payload into rows and ingest them.

        Overview:
            • Uses OTLPMapper.map_logs(...) to produce a list of LOGS rows.
            • Enqueues each row via ingest_log(...), counting successes/failures.
            • Returns a compact summary with accepted/rejected and total rows mapped.

        Args:
            payload: A Mapping representing a decoded OTLP ExportLogsServiceRequest.
            project_id: Optional tenancy override passed through to the mapper.

        Returns:
            Dict with keys: {'accepted', 'rejected', 'logs'}

        Notes:
            - ingest_log(...) performs validation/normalization against LOG_SCHEMA,
            including AnyValue→string handling for `body`, then enqueues.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - `dict(payload)` defensively copies; the mapper may mutate during shaping.
        # - Counts are end-to-end admission outcomes, not just mapping sizes.

        rows = self._mapper.map_logs(dict(payload), project_id=project_id)
        ca = cr = 0
        for r in rows:
            ok, _ = self.ingest_log(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        return {"accepted": ca, "rejected": cr, "logs": len(rows)}

    def ingest_otlp_metrics_json(self, payload: Mapping[str, Any], *, project_id: Optional[str] = None) -> Dict[str, int]:
        """
        Decode already-parsed OTLP Metrics payload into rows and ingest them.

        Overview:
            • Uses OTLPMapper.map_metrics(...) to produce two lists:
                points (Gauge/Sum) and hists (Histogram/ExpHistogram/Summary).
            • Enqueues each via ingest_metric_point(...) and ingest_metric_hist(...).
            • Tracks total accepted vs rejected across both families and returns a summary.

        Args:
            payload: A Mapping representing a decoded OTLP ExportMetricsServiceRequest.
            project_id: Optional tenancy override passed through to the mapper.

        Returns:
            Dict with keys: {'accepted', 'rejected', 'points', 'hists'}

        Notes:
            - The hist family collapses multiple OTLP metric types into one table
            (METRIC_HIST) with normalized wide columns.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Mapping first, then validation/enqueue per row keeps responsibilities clear.
        # - Return 'points'/'hists' are mapped counts; admission may be lower if rows are rejected.

        points, hists = self._mapper.map_metrics(dict(payload), project_id=project_id)
        ca = cr = 0
        for r in points:
            ok, _ = self.ingest_metric_point(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        for r in hists:
            ok, _ = self.ingest_metric_hist(r); ca += 1 if ok else 0; cr += 0 if ok else 1
        return {"accepted": ca, "rejected": cr, "points": len(points), "hists": len(hists)}

    # ---- Self-observability (minimal v0.1) ----
    def internal_log(
        self,
        body: str,
        *,
        severity_text: str = "INFO",
        severity_number: int = 9,
        attrs: Optional[Dict[str, Any]] = None,
        project_id: str = "__depths__",
        service_name: str = "depths-internal",
    ) -> Tuple[bool, Optional[str]]:
        """
        Emit a minimal operational log into the OTel LOGS table.

        Overview:
            • Only active when DepthsLoggerOptions.internal_otel_logs is True.
            • Constructs a row with:
                project_id, schema_version=1, service_name,
                resource_attrs_json (from `attrs`), scope_* (name/version/attrs),
                time_unix_nano & observed_time_unix_nano (both = now in ns, UTC),
                severity_text, severity_number, body (string), log_attrs_json ({}),
                trace_id="", span_id="".
            • Submits via ingest_log(...) and returns its (ok, reason) tuple.

        Args:
            body: Message string.
            severity_text: OTel severity text (default 'INFO').
            severity_number: OTel severity number (default 9).
            attrs: Optional dict merged into resource_attrs_json.
            project_id: Logical tenant for this internal event.
            service_name: Service identity under which this event is recorded.

        Returns:
            Tuple[bool, Optional[str]] from ingest_log.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Uses current UTC time for both time_unix_nano and observed_time_unix_nano.
        # - Keeps log_attrs_json empty to reduce noise; attach custom keys via `attrs`
        #   under resource_attrs_json instead.

        if not self._opts.internal_otel_logs:
            return True, None
        now_ns = int(_dt.datetime.now(tz=_dt.UTC).timestamp() * 1_000_000_000)
        row = {
            "project_id": project_id,
            "schema_version": 1,
            "service_name": service_name,
            "resource_attrs_json": attrs or {},
            "scope_name": "depths",
            "scope_version": "v0.1",
            "scope_attrs_json": {},
            "time_unix_nano": now_ns,
            "observed_time_unix_nano": now_ns,
            "severity_text": severity_text,
            "severity_number": int(severity_number),
            "body": str(body),
            "log_attrs_json": {},
            "trace_id": "",
            "span_id": "",
        }
        return self.ingest_log(row)

    # ---- Readers (lazy) ----
    def _day_table_sources(
        self,
        start_day: str|None,
        end_day: str|None,
        sub: str,
        *,
        storage: Literal["auto","local","s3"] = "auto",
    ) -> list[tuple[str,str,dict|None]]:
        """Return (day, table_path, s3_opts) for requested range + table subpath."""
        def _iter_days(a: Optional[str], b: Optional[str]) -> list[str]:
            if not a and not b: return [_today_utc_str()]
            if not a: a = b
            if not b: b = a
            da = _dt.date.fromisoformat(a); db = _dt.date.fromisoformat(b)
            if da>db: da,db=db,da
            out=[]; cur=da
            while cur<=db: out.append(cur.isoformat()); cur=cur+_dt.timedelta(days=1)
            return out

        days = _iter_days(start_day, end_day)
        s3_opts = self._s3.to_delta_storage_options() if self._s3 else None
        out=[]
        for d in days:
            local_root = self._local_day_path(self._instance_root, d)
            local_tbl = self._otel_table_path(local_root, sub)
            if storage in {"auto","local"} and Path(local_tbl).exists():
                out.append((d, local_tbl, None))
                continue
            if storage in {"auto","s3"} and self._s3:
                out.append((d, self._s3.day_uri(self._instance_id, d).rstrip("/") + f"/otel/{sub}", s3_opts))
        return out

    def _lazy_for_table(
        self,
        sub: str,
        *,
        date_from: str|None = None,
        date_to: str|None = None,
        storage: Literal["auto","local","s3"] = "auto",
        where: pl.Expr | Iterable[pl.Expr] | None = None,
        select: Iterable[str] | None = None,
        limit: int | None = None,
    ) -> pl.LazyFrame:
        from depths.io.delta import read_delta
        lfs: list[pl.LazyFrame] = []
        for _, path, s3opts in self._day_table_sources(date_from, date_to, sub, storage=storage):
            try:
                lf_or_df = read_delta(path, storage_options=s3opts, return_lf=True)
                lf = lf_or_df if isinstance(lf_or_df, pl.LazyFrame) else lf_or_df.lazy()
                lfs.append(lf)
            except Exception:
                continue
        if not lfs:
            return pl.DataFrame(schema=self._schemas[sub].polars_schema()).lazy()
        lf = pl.concat(lfs, how="vertical")
        if where is not None:
            if isinstance(where, pl.Expr):
                lf = lf.filter(where)
            else:
                for w in where:
                    lf = lf.filter(w)
        if select:
            lf = lf.select(list(select))
        if limit is not None:
            lf = lf.limit(limit)
        return lf

    # Public readers
    def spans_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("spans", **kw)

    def span_events_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("span_events", **kw)

    def span_links_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("span_links", **kw)

    def logs_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("logs", **kw)

    def metrics_points_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("metrics_points", **kw)

    def metrics_hist_lazy(self, **kw) -> pl.LazyFrame:
        return self._lazy_for_table("metrics_hist", **kw)


    def _apply_select_limit(self, lf: pl.LazyFrame, *, select: list[str] | None, limit: int | None) -> pl.LazyFrame:
        if select:
            lf = lf.select([pl.col(c) for c in select])
        if limit is not None:
            lf = lf.limit(int(limit))
        return lf

    # ---------- named-predicate readers (return LazyFrame) ----------
    def read_spans(
        self,
        *,
        date_from: str | None = None,
        date_to: str | None = None,
        project_id: str | None = None,
        service_name: str | None = None,
        trace_id: str | None = None,
        span_id: str | None = None,
        name: str | None = None,
        name_like: str | None = None,
        status_code: str | None = None,
        kind: str | None = None,
        time_ms_from: int | None = None,
        time_ms_to: int | None = None,
        select: list[str] | None = None,
        limit: int | None = None,
        storage: Literal["auto","local","s3"] = "auto",
    ) -> pl.LazyFrame:
        """
        Build a pushdown-friendly LazyFrame for SPANS with common predicates.

        Overview:
            • Sources data across a date range from local and/or S3 via spans_lazy(...).
            • Applies equality filters on:
                project_id, service_name, trace_id, span_id, name, status_code, kind.
            • Applies a literal substring filter on `name` when name_like is set
            (no regex; uses .str.contains(..., literal=True)).
            • Applies an inclusive time window on `event_ts` (epoch ms) via _time_range_expr.
            • Optionally projects columns (`select`) and limits rows (`limit`).

        Args:
            date_from/date_to: Inclusive UTC day bounds ('YYYY-MM-DD').
            project_id/service_name/trace_id/span_id/name/name_like/status_code/kind: Optional filters.
            time_ms_from/time_ms_to: Inclusive event_ts bounds in epoch milliseconds.
            select: Optional list of column names to project.
            limit: Optional max rows to return (applied lazily).
            storage: 'auto' | 'local' | 's3' source selection.

        Returns:
            pl.LazyFrame with predicates composed and applied lazily for pushdown.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Predicates are AND-composed in one expression to maximize engine pushdown.
        # - Time predicates use integer comparisons matching the stored dtype.

        lf = self.spans_lazy(date_from=date_from, date_to=date_to, storage=storage)
        predicates: list[pl.Expr] = []
        for col, val in (
            ("project_id", project_id),
            ("service_name", service_name),
            ("trace_id", trace_id),
            ("span_id", span_id),
            ("name", name),
            ("status_code", status_code),
            ("kind", kind),
       ):
            e = self._eq_expr(col, val)
            if e is not None:
                predicates.append(e)
        
        e_like = self._like_expr("name", name_like)
        if e_like is not None:
            predicates.append(e_like)
        
        e_time = self._time_range_expr("event_ts", time_ms_from, time_ms_to)
        if e_time is not None:
            predicates.append(e_time)
        if predicates:
            expr = predicates[0]
            for p in predicates[1:]:
                expr = expr & p
            lf = lf.filter(expr) 
        return self._apply_select_limit(lf, select=select, limit=limit)

    def read_logs(
        self,
        *,
        date_from: str | None = None,
        date_to: str | None = None,
        project_id: str | None = None,
        service_name: str | None = None,
        severity_ge: int | None = None,
        body_like: str | None = None,
        trace_id: str | None = None,
        span_id: str | None = None,
        time_ms_from: int | None = None,
        time_ms_to: int | None = None,
        select: list[str] | None = None,
        limit: int | None = None,
        storage: Literal["auto","local","s3"] = "auto",
    ) -> pl.LazyFrame:
        """
        Build a pushdown-friendly LazyFrame for LOGS with common predicates.

        Overview:
            • Sources rows via logs_lazy(...).
            • Applies equality filters on: project_id, service_name, trace_id, span_id.
            • Applies a >= threshold on severity_number when severity_ge is set.
            • Applies a literal substring filter on `body` when body_like is set.
            • Applies an inclusive time window on `event_ts` (epoch ms).
            • Supports optional projection (`select`) and row limit (`limit`).

        Args:
            date_from/date_to: Inclusive UTC day bounds ('YYYY-MM-DD').
            project_id/service_name/trace_id/span_id: Optional equality filters.
            severity_ge: Minimum severity_number (inclusive).
            body_like: Literal substring to search in `body`.
            time_ms_from/time_ms_to: Inclusive event_ts bounds in epoch milliseconds.
            select: Optional list of column names.
            limit: Optional max rows to return.
            storage: 'auto' | 'local' | 's3' source selection.

        Returns:
            pl.LazyFrame with all applicable predicates applied lazily.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Substring filters use literal matching (no regex) for safety and pushdown.
        # - The filter expression is composed once to keep execution efficient.

        lf = self.logs_lazy(date_from=date_from, date_to=date_to, storage=storage)
        predicates: list[pl.Expr] = []
        for col, val in (
            ("project_id", project_id),
            ("service_name", service_name),
            ("trace_id", trace_id),
            ("span_id", span_id),
        ):
            e = self._eq_expr(col, val)
            if e is not None:
                predicates.append(e)
        if severity_ge is not None:
            e = self._ge_expr("severity_number", int(severity_ge))
            if e is not None:
                predicates.append(e)
        e_like = self._like_expr("body", body_like)
        if e_like is not None:
            predicates.append(e_like)
        e_time = self._time_range_expr("event_ts", time_ms_from, time_ms_to)
        if e_time is not None:
            predicates.append(e_time)
        if predicates:
            expr = predicates[0]
            for p in predicates[1:]:
                expr = expr & p
            lf = lf.filter(expr)
        return self._apply_select_limit(lf, select=select, limit=limit)

    def read_metrics_points(
        self,
        *,
        date_from: str | None = None,
        date_to: str | None = None,
        project_id: str | None = None,
        service_name: str | None = None,
        instrument_name: str | None = None,
        instrument_type: str | None = None,  # "Gauge" | "Sum"
        time_ms_from: int | None = None,
        time_ms_to: int | None = None,
        select: list[str] | None = None,
        limit: int | None = None,
        storage: Literal["auto","local","s3"] = "auto",
    ) -> pl.LazyFrame:
        """
        Build a pushdown-friendly LazyFrame for METRIC_POINT (Gauge/Sum).

        Overview:
            • Sources rows via metrics_points_lazy(...).
            • Applies equality filters on:
                project_id, service_name, instrument_name, instrument_type.
            • Applies an inclusive time window on `event_ts` (epoch ms).
            • Supports optional projection (`select`) and row limit (`limit`).

        Args:
            date_from/date_to: Inclusive UTC day bounds ('YYYY-MM-DD').
            project_id/service_name/instrument_name/instrument_type: Optional equality filters.
            time_ms_from/time_ms_to: Inclusive event_ts bounds in epoch milliseconds.
            select: Optional list of column names.
            limit: Optional max rows to return.
            storage: 'auto' | 'local' | 's3' source selection.

        Returns:
            pl.LazyFrame with composed lazy predicates.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Instrument metadata columns are preserved as-is for straightforward filtering.
        # - Time comparisons remain integer-based for predicate pushdown.

        lf = self.metrics_points_lazy(date_from=date_from, date_to=date_to, storage=storage)
        predicates: list[pl.Expr] = []
        for col, val in (
            ("project_id", project_id),
            ("service_name", service_name),
            ("instrument_name", instrument_name),
            ("instrument_type", instrument_type),
        ):
            e = self._eq_expr(col, val)
            if e is not None:
                predicates.append(e)
        e_time = self._time_range_expr("event_ts", time_ms_from, time_ms_to)
        if e_time is not None:
            predicates.append(e_time)
        if predicates:
            expr = predicates[0]
            for p in predicates[1:]:
                expr = expr & p
            lf = lf.filter(expr)
        return self._apply_select_limit(lf, select=select, limit=limit)

    def read_metrics_hist(
        self,
        *,
        date_from: str | None = None,
        date_to: str | None = None,
        project_id: str | None = None,
        service_name: str | None = None,
        instrument_name: str | None = None,
        instrument_type: str | None = None,  # "Histogram" | "ExpHistogram" | "Summary"
        time_ms_from: int | None = None,
        time_ms_to: int | None = None,
        select: list[str] | None = None,
        limit: int | None = None,
        storage: Literal["auto","local","s3"] = "auto",
    ) -> pl.LazyFrame:
        """
        Build a pushdown-friendly LazyFrame for METRIC_HIST (Histogram/ExpHistogram/Summary).

        Overview:
            • Sources rows via metrics_hist_lazy(...).
            • Applies equality filters on:
                project_id, service_name, instrument_name, instrument_type.
            • Applies an inclusive time window on `event_ts` (epoch ms).
            • Supports optional projection (`select`) and row limit (`limit`).

        Args:
            date_from/date_to: Inclusive UTC day bounds ('YYYY-MM-DD').
            project_id/service_name/instrument_name/instrument_type: Optional equality filters.
            time_ms_from/time_ms_to: Inclusive event_ts bounds in epoch milliseconds.
            select: Optional list of column names.
            limit: Optional max rows to return.
            storage: 'auto' | 'local' | 's3' source selection.

        Returns:
            pl.LazyFrame with composed lazy predicates across histogram-like families.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - The “hist” table unifies multiple OTLP metric types; filters work uniformly.
        # - Time windowing uses `event_ts` to align with spans/logs semantics.

        lf = self.metrics_hist_lazy(date_from=date_from, date_to=date_to, storage=storage)
        predicates: list[pl.Expr] = []
        for col, val in (
            ("project_id", project_id),
            ("service_name", service_name),
            ("instrument_name", instrument_name),
            ("instrument_type", instrument_type),
        ):
            e = self._eq_expr(col, val)
            if e is not None:
                predicates.append(e)
        e_time = self._time_range_expr("event_ts", time_ms_from, time_ms_to)
        if e_time is not None:
            predicates.append(e_time)
        if predicates:
            expr = predicates[0]
            for p in predicates[1:]:
                expr = expr & p
            lf = lf.filter(expr)
        return self._apply_select_limit(lf, select=select, limit=limit)

    def metrics(self) -> Dict[str, Any]:
        """
        Snapshot operational metrics for all aggregators.

        Returns:
            Dict with per-table counters:
            flushes, rows_scheduled_total, rows_persisted_total, rows_last_flush,
            last_commit_seconds, writer_queue_size, delta_init_ok,
            delta_last_error, and current table_path.
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Values are best-effort (no heavy synchronization).

        out = {"started": self._started, "current_day_utc": self._current_day, "aggregators": {}}
        for name, agg in self._aggs.items():
            m = agg.metrics
            out["aggregators"][name] = {
                "flushes": m.flushes, "rows_scheduled_total": m.rows_scheduled_total,
                "rows_persisted_total": m.rows_persisted_total, "writer_queue_size": m.writer_queue_size,
                "last_commit_seconds": m.last_commit_seconds, "delta_init_ok": m.delta_init_ok,
                "delta_last_error": m.delta_last_error, "table_path": agg.table_path
            }
        return out

    # ---- signals/atexit ----
    def _install_signal_handlers(self) -> None:
        """
        Register SIGINT/SIGTERM handlers to attempt a graceful stop.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Wrapped in try/except to tolerate restricted environments (e.g., Windows, notebooks).

        def _handler(signum, frame):
            try: self.stop(flush="auto")
            except Exception: pass
        try: signal.signal(signal.SIGINT, _handler)
        except Exception: pass
        try: signal.signal(signal.SIGTERM, _handler)
        except Exception: pass

    def _atexit_stop(self) -> None:
        """
        atexit hook to attempt a final flush and shutdown.

        Returns:
            None
        """
        # --- DEVELOPER NOTES -------------------------------------------------
        # - Swallows exceptions to avoid masking interpreter shutdown.

        try: self.stop(flush="auto")
        except Exception: pass
