# =============================================================================
# PPM: Predictive Push Memory — Phase 7: Observability & Production Polish
#
# Files:
#   observability/__init__.py
#   observability/metrics.py        ← Prometheus counters/histograms
#   observability/tracing.py        ← OpenTelemetry spans
#   observability/middleware.py     ← FastAPI request instrumentation
#   tests/bench/bench_momentum.py   ← latency benchmarks for math layer
#   tests/bench/bench_staging.py    ← end-to-end staging benchmarks
#   tests/bench/bench_store.py      ← store read/write benchmarks
#   docker/Dockerfile
#   docker/docker-compose.yml
#   docker/.env.example
#   pyproject.toml                  ← complete project config
#   README.md                       ← architecture + quickstart
#
# After this phase PPM is:
#   - Observable (metrics dashboard, distributed traces)
#   - Benchmarked (latency P50/P95/P99 per layer)
#   - Containerized (docker-compose up → running system)
#   - Documented (README with diagrams + quickstart)
# =============================================================================


### observability/__init__.py ###

# empty


### observability/metrics.py ###

"""
Prometheus metrics for PPM.

Metric naming convention: ppm_<layer>_<operation>_<unit>

Layers:
  surface   - observer + intent classifier
  nerve     - trajectory predictor
  staging   - prefetch cache + injector
  store     - vector/graph/meta reads and writes
  feedback  - hit/miss detection
  session   - end-to-end turn latency
  write     - memory distillation + storage

All histograms use standard latency buckets in seconds.
All counters are monotonically increasing (never reset).
"""

from prometheus_client import (
    Counter, Histogram, Gauge, Summary,
    CollectorRegistry, generate_latest, CONTENT_TYPE_LATEST,
)

# ── Registry ───────────────────────────────────────────────────────────────────
# Use default registry for simplicity — replace with custom registry
# for multi-process deployments.

REGISTRY = CollectorRegistry()

_LATENCY_BUCKETS = (
    0.001, 0.002, 0.005, 0.010, 0.025, 0.050,
    0.100, 0.250, 0.500, 1.000, 2.500, 5.000,
)

# ── Session metrics ────────────────────────────────────────────────────────────

session_turns_total = Counter(
    "ppm_session_turns_total",
    "Total conversation turns processed",
    ["intent"],
    registry=REGISTRY,
)

session_turn_latency = Histogram(
    "ppm_session_turn_latency_seconds",
    "End-to-end turn latency (hot path only, before post-response)",
    buckets=_LATENCY_BUCKETS,
    registry=REGISTRY,
)

session_active = Gauge(
    "ppm_session_active_total",
    "Number of currently active sessions",
    registry=REGISTRY,
)

session_context_tokens = Histogram(
    "ppm_session_context_tokens",
    "Tokens injected into LLM context per turn",
    buckets=(0, 50, 100, 200, 500, 1000, 2000, 4096),
    registry=REGISTRY,
)

session_memories_injected = Histogram(
    "ppm_session_memories_injected",
    "Number of distinct memories injected per turn",
    buckets=(0, 1, 2, 3, 5, 8, 13),
    registry=REGISTRY,
)

# ── Nerve / Predictor metrics ──────────────────────────────────────────────────

nerve_predictions_generated = Counter(
    "ppm_nerve_predictions_total",
    "Predictions generated by trajectory predictor",
    ["strategy", "intent"],
    registry=REGISTRY,
)

nerve_momentum_velocity = Histogram(
    "ppm_nerve_momentum_velocity",
    "Conversation momentum velocity per turn",
    buckets=(0, 0.02, 0.05, 0.10, 0.20, 0.40, 0.80, 1.60),
    registry=REGISTRY,
)

nerve_context_switches = Counter(
    "ppm_nerve_context_switches_total",
    "Context switch resets detected by observer",
    registry=REGISTRY,
)

nerve_observe_latency = Histogram(
    "ppm_nerve_observe_latency_seconds",
    "Observer.observe() latency (includes embedding call)",
    buckets=_LATENCY_BUCKETS,
    registry=REGISTRY,
)

# ── Staging metrics ────────────────────────────────────────────────────────────

staging_prefetch_latency = Histogram(
    "ppm_staging_prefetch_latency_seconds",
    "Per-slot prefetch latency (background, not on hot path)",
    ["strategy"],
    buckets=_LATENCY_BUCKETS,
    registry=REGISTRY,
)

staging_slot_hits = Counter(
    "ppm_staging_slot_hits_total",
    "Times a staged memory was successfully injected",
    ["tier"],
    registry=REGISTRY,
)

staging_slot_misses = Counter(
    "ppm_staging_slot_misses_total",
    "Times no staged memory was available for injection",
    registry=REGISTRY,
)

staging_evictions = Counter(
    "ppm_staging_evictions_total",
    "Staging slot evictions due to TTL expiry",
    registry=REGISTRY,
)

staging_slots_ready = Gauge(
    "ppm_staging_slots_ready",
    "Number of filled, non-expired staging slots",
    registry=REGISTRY,
)

# ── Store metrics ──────────────────────────────────────────────────────────────

store_vector_search_latency = Histogram(
    "ppm_store_vector_search_latency_seconds",
    "Qdrant vector search latency",
    buckets=_LATENCY_BUCKETS,
    registry=REGISTRY,
)

store_memory_writes_total = Counter(
    "ppm_store_memory_writes_total",
    "Memory write operations",
    ["action"],  # written, duplicate, skipped, error
    registry=REGISTRY,
)

store_chunks_written_total = Counter(
    "ppm_store_chunks_written_total",
    "Total chunks written to vector store",
    registry=REGISTRY,
)

store_outbox_pending = Gauge(
    "ppm_store_outbox_pending",
    "Number of pending outbox entries awaiting propagation",
    registry=REGISTRY,
)

store_outbox_dead_letters = Counter(
    "ppm_store_outbox_dead_letters_total",
    "Outbox entries that exhausted retries and went to dead letter queue",
    registry=REGISTRY,
)

# ── Feedback metrics ───────────────────────────────────────────────────────────

feedback_hit_rate = Histogram(
    "ppm_feedback_hit_rate",
    "Per-turn prediction hit rate",
    buckets=(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0),
    registry=REGISTRY,
)

feedback_hits_total = Counter(
    "ppm_feedback_hits_total",
    "Total prediction hits",
    ["signal"],  # overlap, semantic, prevented
    registry=REGISTRY,
)

feedback_misses_total = Counter(
    "ppm_feedback_misses_total",
    "Total prediction misses",
    ["strategy"],
    registry=REGISTRY,
)

feedback_bandit_confidence = Histogram(
    "ppm_feedback_bandit_confidence",
    "Bandit confidence scores at prediction time",
    ["strategy", "intent"],
    buckets=(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0),
    registry=REGISTRY,
)

trajectory_samples_total = Counter(
    "ppm_trajectory_samples_total",
    "Total trajectory samples recorded for fine-tuning",
    registry=REGISTRY,
)

# ── Write pipeline metrics ─────────────────────────────────────────────────────

write_distill_latency = Histogram(
    "ppm_write_distill_latency_seconds",
    "MemoryDistiller LLM call latency",
    buckets=_LATENCY_BUCKETS,
    registry=REGISTRY,
)

write_candidates_per_turn = Histogram(
    "ppm_write_candidates_per_turn",
    "Memory candidates extracted per turn by distiller",
    buckets=(0, 1, 2, 3, 5),
    registry=REGISTRY,
)


# ── Helpers ────────────────────────────────────────────────────────────────────

def record_turn(intent: str, latency_s: float, tokens: int, memories: int) -> None:
    """Convenience function — record all session-level metrics for one turn."""
    session_turns_total.labels(intent=intent).inc()
    session_turn_latency.observe(latency_s)
    session_context_tokens.observe(tokens)
    session_memories_injected.observe(memories)


def record_feedback(
    hit_rate: float,
    hits: list[dict],   # [{"signal": str}]
    misses: list[dict], # [{"strategy": str}]
) -> None:
    """Record all feedback-layer metrics for one turn."""
    feedback_hit_rate.observe(hit_rate)
    for h in hits:
        feedback_hits_total.labels(signal=h.get("signal", "unknown")).inc()
    for m in misses:
        feedback_misses_total.labels(strategy=m.get("strategy", "unknown")).inc()


def metrics_response() -> tuple[bytes, str]:
    """Return (body, content_type) for /metrics endpoint."""
    return generate_latest(REGISTRY), CONTENT_TYPE_LATEST


### observability/tracing.py ###

"""
OpenTelemetry distributed tracing for PPM.

Traces every significant operation with spans:
  ppm.session.turn          - full turn (parent span)
  ppm.observer.observe      - embedding + intent extraction
  ppm.predictor.predict     - trajectory prediction
  ppm.staging.prefetch      - background prefetch per slot
  ppm.staging.inject        - knapsack injection planning
  ppm.store.vector_search   - Qdrant search
  ppm.feedback.evaluate     - hit/miss detection
  ppm.write.distill         - memory distillation
  ppm.write.store           - memory persistence

Usage:
  Spans are automatically exported to any OTel-compatible backend
  (Jaeger, Honeycomb, Datadog, etc.) via OTLP exporter.
  Set PPM_OTEL_ENDPOINT env var to enable (disabled by default).
"""

import os
import functools
import time
from contextlib import contextmanager, asynccontextmanager
from typing import Any, Callable

_OTEL_AVAILABLE = False
_tracer = None

try:
    from opentelemetry import trace
    from opentelemetry.sdk.trace import TracerProvider
    from opentelemetry.sdk.trace.export import BatchSpanProcessor
    from opentelemetry.sdk.resources import Resource
    _OTEL_AVAILABLE = True
except ImportError:
    pass


def init_tracing(
    service_name: str = "ppm",
    endpoint: str | None = None,
) -> None:
    """
    Initialize OpenTelemetry tracing.
    No-op if opentelemetry-sdk is not installed or endpoint not set.
    """
    global _tracer

    otel_endpoint = endpoint or os.getenv("PPM_OTEL_ENDPOINT")
    if not _OTEL_AVAILABLE or not otel_endpoint:
        _tracer = _NoOpTracer()
        return

    try:
        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
        resource = Resource.create({"service.name": service_name, "service.version": "0.1.0"})
        provider = TracerProvider(resource=resource)
        exporter = OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
        provider.add_span_processor(BatchSpanProcessor(exporter))
        trace.set_tracer_provider(provider)
        _tracer = trace.get_tracer("ppm")
    except Exception as e:
        _tracer = _NoOpTracer()


def get_tracer():
    global _tracer
    if _tracer is None:
        _tracer = _NoOpTracer()
    return _tracer


@contextmanager
def span(name: str, attributes: dict | None = None):
    """Sync context manager for a trace span."""
    tracer = get_tracer()
    with tracer.start_as_current_span(name) as s:
        if attributes and hasattr(s, "set_attribute"):
            for k, v in attributes.items():
                try:
                    s.set_attribute(k, v)
                except Exception:
                    pass
        yield s


@asynccontextmanager
async def async_span(name: str, attributes: dict | None = None):
    """Async context manager for a trace span."""
    tracer = get_tracer()
    with tracer.start_as_current_span(name) as s:
        if attributes and hasattr(s, "set_attribute"):
            for k, v in (attributes or {}).items():
                try:
                    s.set_attribute(k, v)
                except Exception:
                    pass
        yield s


def traced(span_name: str):
    """Decorator for async functions — wraps in a trace span."""
    def decorator(fn: Callable):
        @functools.wraps(fn)
        async def wrapper(*args, **kwargs):
            async with async_span(span_name):
                return await fn(*args, **kwargs)
        return wrapper
    return decorator


class _NoOpTracer:
    """Null tracer — used when OTel is not configured."""

    def start_as_current_span(self, name: str):
        return _NoOpSpan()


class _NoOpSpan:
    def __enter__(self): return self
    def __exit__(self, *_): pass
    def set_attribute(self, *_): pass
    def record_exception(self, *_): pass
    def set_status(self, *_): pass


### observability/middleware.py ###

"""
FastAPI middleware for automatic request instrumentation.

Adds to every request:
  - Prometheus request counter + latency histogram
  - OpenTelemetry span with HTTP attributes
  - Request ID header (X-Request-ID)
  - Structured access log
"""

import time
import uuid
import structlog
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.types import ASGIApp

from observability.metrics import REGISTRY
from prometheus_client import Counter, Histogram

log = structlog.get_logger("ppm.http")

http_requests_total = Counter(
    "ppm_http_requests_total",
    "Total HTTP requests",
    ["method", "path", "status"],
    registry=REGISTRY,
)

http_request_latency = Histogram(
    "ppm_http_request_latency_seconds",
    "HTTP request latency",
    ["method", "path"],
    buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5),
    registry=REGISTRY,
)


class ObservabilityMiddleware(BaseHTTPMiddleware):
    """
    Instruments every HTTP request with metrics, tracing, and logging.
    Adds X-Request-ID header to every response.
    """

    async def dispatch(self, request: Request, call_next) -> Response:
        request_id = str(uuid.uuid4())[:8]
        t0 = time.monotonic()

        # Normalize path for metrics (replace path params with {param})
        path = self._normalize_path(request.url.path)

        try:
            response = await call_next(request)
            status = response.status_code
        except Exception as e:
            status = 500
            log.error(
                "http.request_error",
                request_id=request_id,
                method=request.method,
                path=path,
                error=str(e),
            )
            raise
        finally:
            latency = time.monotonic() - t0
            http_requests_total.labels(
                method=request.method,
                path=path,
                status=str(status),
            ).inc()
            http_request_latency.labels(
                method=request.method,
                path=path,
            ).observe(latency)

        response.headers["X-Request-ID"] = request_id
        response.headers["X-Latency-Ms"] = f"{latency * 1000:.1f}"

        log.info(
            "http.request",
            request_id=request_id,
            method=request.method,
            path=path,
            status=status,
            latency_ms=round(latency * 1000, 1),
        )

        return response

    @staticmethod
    def _normalize_path(path: str) -> str:
        """Replace UUIDs and IDs in paths with {id} placeholder."""
        import re
        # Replace UUID segments
        path = re.sub(
            r'/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
            '/{id}', path
        )
        return path


def add_metrics_endpoint(app) -> None:
    """Add /metrics endpoint to FastAPI app for Prometheus scraping."""
    from fastapi import Response as FastAPIResponse
    from observability.metrics import metrics_response

    @app.get("/metrics", include_in_schema=False)
    async def prometheus_metrics():
        body, content_type = metrics_response()
        return FastAPIResponse(content=body, media_type=content_type)


### tests/bench/bench_momentum.py ###

"""
Benchmark suite for math_core/momentum.py

Measures latency for all math operations on the hot path.
Target: all operations < 2ms on a modern laptop CPU.

Run:
  python tests/bench/bench_momentum.py

Output:
  operation                    P50      P95      P99     mean
  l2_normalize (d=1536)       0.01ms   0.02ms   0.03ms  0.01ms
  conversation_state (N=6)    0.05ms   0.07ms   0.09ms  0.05ms
  momentum_tangent            0.02ms   0.03ms   0.04ms  0.02ms
  predict_future_state        0.01ms   0.02ms   0.02ms  0.01ms
"""

import time
import statistics
import numpy as np

# Fix import path for running directly
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))

from math_core.momentum import (
    l2_normalize, conversation_state,
    momentum_tangent, predict_future_state,
)

N_WARMUP = 100
N_ITERS  = 10_000
DIM      = 1536   # production embedding dimension


def bench(name: str, fn, n: int = N_ITERS) -> dict:
    """Run fn n times, return latency statistics in milliseconds."""
    # Warmup
    for _ in range(N_WARMUP):
        fn()

    times = []
    for _ in range(n):
        t0 = time.perf_counter()
        fn()
        times.append((time.perf_counter() - t0) * 1000)

    return {
        "name": name,
        "p50":  round(statistics.median(times), 3),
        "p95":  round(statistics.quantiles(times, n=20)[18], 3),
        "p99":  round(statistics.quantiles(times, n=100)[98], 3),
        "mean": round(statistics.mean(times), 3),
        "n":    n,
    }


def run_benchmarks():
    print(f"\nPPM Math Benchmarks (d={DIM}, n={N_ITERS:,})")
    print("=" * 65)
    print(f"{'Operation':<35} {'P50':>7} {'P95':>7} {'P99':>7} {'mean':>7}")
    print("-" * 65)

    results = []

    # l2_normalize
    v = np.random.randn(DIM).astype(np.float32)
    r = bench("l2_normalize", lambda: l2_normalize(v))
    results.append(r)

    # conversation_state (N=6 turns, production window)
    turns = [l2_normalize(np.random.randn(DIM).astype(np.float32)) for _ in range(6)]
    r = bench("conversation_state (N=6)", lambda: conversation_state(turns, 0.85))
    results.append(r)

    # momentum_tangent
    C_t    = l2_normalize(np.random.randn(DIM).astype(np.float32))
    C_prev = l2_normalize(np.random.randn(DIM).astype(np.float32))
    M_prev = np.random.randn(DIM).astype(np.float32) * 0.1
    r = bench("momentum_tangent", lambda: momentum_tangent(C_t, C_prev, M_prev))
    results.append(r)

    # predict_future_state
    M_hat = l2_normalize(np.random.randn(DIM).astype(np.float32))
    vel   = 0.15
    r = bench("predict_future_state (k=2)", lambda: predict_future_state(C_t, M_hat, vel, k=2))
    results.append(r)

    # Full pipeline: state → momentum → predict
    def full_pipeline():
        C = conversation_state(turns, 0.85)
        M, v = momentum_tangent(C, C_prev, M_prev)
        return predict_future_state(C, M, v, k=1)

    r = bench("full_pipeline (state+momentum+predict)", full_pipeline, n=N_ITERS // 2)
    results.append(r)

    for r in results:
        print(f"  {r['name']:<33} {r['p50']:>6}ms {r['p95']:>6}ms {r['p99']:>6}ms {r['mean']:>6}ms")

    print("-" * 65)

    # Check targets
    print("\nTarget validation (all ops < 2ms):")
    all_pass = True
    for r in results:
        status = "✓" if r["p99"] < 2.0 else "✗"
        if r["p99"] >= 2.0:
            all_pass = False
        print(f"  {status} {r['name']}: P99={r['p99']}ms")

    print(f"\n{'All targets met ✓' if all_pass else 'Some targets missed ✗'}")
    return results


if __name__ == "__main__":
    run_benchmarks()


### tests/bench/bench_staging.py ###

"""
Benchmark: Staging layer operations.

Measures:
  - Knapsack selection (N chunks, B budget)
  - Injector.plan() end-to-end
  - Reranker.rerank() for N staged memories
  - StagingCache slot operations

Target:
  - knapsack_01 (N=50, B=4096): < 1ms
  - injector.plan (10 memories): < 2ms
  - reranker.rerank (10 memories): < 1ms
"""

import time
import statistics
import numpy as np
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))

from math_core.momentum import l2_normalize
from math_core.knapsack import knapsack_01
from core.staging.injector import Injector
from core.staging.reranker import Reranker
from core.staging.models import StagedChunk, StagedMemory
from core.nerve.models import Prediction, PrefetchStrategy, IntentSignal

N_WARMUP = 50
N_ITERS  = 5_000
DIM      = 1536


def rand_unit(d=DIM):
    return l2_normalize(np.random.randn(d).astype(np.float32))


def make_chunks(n: int, budget: int) -> list[dict]:
    return [
        {
            "id": f"chunk-{i}",
            "parent_id": f"mem-{i // 3}",
            "chunk_index": i % 3,
            "content": f"Content of chunk {i} with some realistic text here.",
            "tokens": np.random.randint(20, 150),
            "score": np.random.random(),
            "source_type": "prose",
            "source": "",
        }
        for i in range(n)
    ]


def make_staged_memories(n: int) -> list[StagedMemory]:
    mems = []
    for i in range(n):
        pred = Prediction(
            query_vector=rand_unit(), query_text="test",
            graph_seeds=[], annotation_tags=[],
            confidence=np.random.random(),
            strategy=PrefetchStrategy.SEMANTIC,
            intent=IntentSignal.EXPLORE, k_steps=1,
        )
        chunks = [
            StagedChunk(
                chunk_id=f"c-{i}-{j}", parent_id=f"m-{i}",
                chunk_index=j, content=f"chunk content {i}-{j}",
                tokens=50, score=np.random.random(), source_type="prose",
            )
            for j in range(3)
        ]
        mems.append(StagedMemory(
            prediction=pred, chunks=chunks,
            raw_confidence=np.random.random(), rerank_score=np.random.random(),
        ))
    return mems


def bench(name, fn, n=N_ITERS):
    for _ in range(N_WARMUP):
        fn()
    times = []
    for _ in range(n):
        t0 = time.perf_counter()
        fn()
        times.append((time.perf_counter() - t0) * 1000)
    return {
        "name": name,
        "p50":  round(statistics.median(times), 3),
        "p95":  round(statistics.quantiles(times, n=20)[18], 3),
        "p99":  round(statistics.quantiles(times, n=100)[98], 3),
    }


def run_benchmarks():
    print(f"\nPPM Staging Benchmarks (n={N_ITERS:,})")
    print("=" * 60)
    print(f"{'Operation':<35} {'P50':>7} {'P95':>7} {'P99':>7}")
    print("-" * 60)

    results = []

    # Knapsack at various sizes
    for n_chunks, budget in [(10, 1000), (50, 4096), (100, 4096)]:
        chunks = make_chunks(n_chunks, budget)
        r = bench(
            f"knapsack_01 (N={n_chunks}, B={budget})",
            lambda c=chunks, b=budget: knapsack_01(c, b),
            n=N_ITERS // (n_chunks // 10),
        )
        results.append(r)

    # Injector.plan
    injector = Injector()
    staged_10 = make_staged_memories(10)
    r = bench(
        "injector.plan (10 memories)",
        lambda: injector.plan(staged_10, token_budget=4096),
        n=N_ITERS // 5,
    )
    results.append(r)

    # Reranker
    reranker = Reranker()
    staged_10 = make_staged_memories(10)
    query_emb = rand_unit()
    chunk_embs = {
        f"c-{i}-{j}": rand_unit()
        for i in range(10) for j in range(3)
    }
    r = bench(
        "reranker.rerank (10 memories)",
        lambda: reranker.rerank(staged_10, query_emb, chunk_embs),
        n=N_ITERS // 5,
    )
    results.append(r)

    for r in results:
        print(f"  {r['name']:<33} {r['p50']:>6}ms {r['p95']:>6}ms {r['p99']:>6}ms")

    targets = {"knapsack": 1.0, "injector": 2.0, "reranker": 1.0}
    print("\nTarget validation:")
    for r in results:
        key = next((k for k in targets if k in r["name"].lower()), None)
        if key:
            target = targets[key]
            ok = r["p99"] < target
            print(f"  {'✓' if ok else '✗'} {r['name']}: P99={r['p99']}ms (target <{target}ms)")

    return results


if __name__ == "__main__":
    run_benchmarks()


### tests/bench/bench_store.py ###

"""
Benchmark: Store layer operations.

Measures SQLite read/write performance for common MetaStore queries.
Qdrant benchmarks require a running Qdrant instance — skipped if unavailable.

Targets:
  - SQLite annotation search: < 5ms P99
  - SQLite memory insert:     < 10ms P99
  - Outbox enqueue:           < 2ms P99
"""

import asyncio
import time
import statistics
import uuid
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))

from core.store.meta import MetaStore
from config.settings import settings

N_ITERS = 1_000


async def bench_async(name: str, coro_fn, n: int = N_ITERS) -> dict:
    """Benchmark an async function."""
    times = []
    for _ in range(n):
        t0 = time.perf_counter()
        await coro_fn()
        times.append((time.perf_counter() - t0) * 1000)
    return {
        "name": name,
        "p50":  round(statistics.median(times), 3),
        "p95":  round(statistics.quantiles(times, n=20)[18], 3),
        "p99":  round(statistics.quantiles(times, n=100)[98], 3),
    }


async def run_benchmarks():
    print(f"\nPPM Store Benchmarks (n={N_ITERS:,})")
    print("=" * 60)
    print(f"{'Operation':<35} {'P50':>7} {'P95':>7} {'P99':>7}")
    print("-" * 60)

    # Use in-memory SQLite for benchmarking (no disk I/O noise)
    meta = MetaStore(":memory:")
    await meta.connect()

    results = []

    # Seed some data for reads
    for i in range(100):
        mid = str(uuid.uuid4())
        await meta._db.execute(
            "INSERT INTO memories (id, content, content_hash, source_type, token_count, created_at) VALUES (?,?,?,?,?,?)",
            (mid, f"Content {i}", f"hash{i}", "prose", 100, int(time.time()))
        )
        await meta._db.execute(
            "INSERT INTO forward_annotations (memory_id, context_tag, created_at) VALUES (?,?,?)",
            (mid, f"intent:DEBUG", int(time.time()))
        )
    await meta._db.commit()

    # Annotation search
    r = await bench_async(
        "annotation search (1 tag)",
        lambda: meta.search_by_annotation(["intent:DEBUG"], limit=10),
    )
    results.append(r)

    # Memory insert
    async def insert_memory():
        mid = str(uuid.uuid4())
        await meta.insert_memory({
            "id": mid,
            "content": "Test memory content for benchmarking purposes.",
            "source": "bench.py",
            "source_type": "prose",
            "token_count": 50,
            "chunks": [],
            "forward_contexts": ["intent:EXPLORE"],
            "graph_edges": [],
        })

    r = await bench_async("memory insert (no chunks)", insert_memory, n=N_ITERS // 2)
    results.append(r)

    # Recently written query (read-your-writes)
    r = await bench_async(
        "recently_written (5s window)",
        lambda: meta.get_recently_written(within_seconds=5.0),
    )
    results.append(r)

    await meta.close()

    for r in results:
        print(f"  {r['name']:<33} {r['p50']:>6}ms {r['p95']:>6}ms {r['p99']:>6}ms")

    print("\nTarget validation:")
    targets = {
        "annotation": 5.0,
        "insert": 10.0,
        "recently": 5.0,
    }
    for r in results:
        key = next((k for k in targets if k in r["name"].lower()), None)
        if key:
            ok = r["p99"] < targets[key]
            print(f"  {'✓' if ok else '✗'} {r['name']}: P99={r['p99']}ms")


if __name__ == "__main__":
    asyncio.run(run_benchmarks())


### docker/Dockerfile ###
# NOTE: This is a Dockerfile — Claude Code should save it as docker/Dockerfile (no .py extension)

DOCKERFILE_CONTENT = """
FROM python:3.12-slim

WORKDIR /app

# System deps for numpy, Qdrant client
RUN apt-get update && apt-get install -y --no-install-recommends \
    gcc g++ curl \
    && rm -rf /var/lib/apt/lists/*

# Install Python dependencies first (layer caching)
COPY pyproject.toml .
RUN pip install --no-cache-dir -e ".[all]"

# Copy source
COPY . .

# Create data directory
RUN mkdir -p /data/ppm

# Default environment
ENV PPM_SQLITE_PATH=/data/ppm/ppm.db
ENV PPM_QDRANT_PATH=/data/ppm/qdrant
ENV PPM_KUZU_PATH=/data/ppm/kuzu
ENV PPM_LOG_LEVEL=INFO

# Initialize stores on first run
RUN python -m cli.ppm init || true

EXPOSE 8000 9090

HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
    CMD curl -f http://localhost:8000/v1/health || exit 1

CMD ["python", "-m", "cli.ppm", "serve", "--port", "8000"]
"""


### docker/docker-compose.yml ###
# NOTE: Save as docker/docker-compose.yml (no .py extension)

DOCKER_COMPOSE_CONTENT = """
version: '3.9'

services:
  ppm:
    build:
      context: ..
      dockerfile: docker/Dockerfile
    ports:
      - "8000:8000"    # REST API
      - "9090:9090"    # Prometheus metrics
    volumes:
      - ppm_data:/data/ppm
    env_file:
      - .env
    environment:
      - PPM_SQLITE_PATH=/data/ppm/ppm.db
      - PPM_QDRANT_PATH=/data/ppm/qdrant
      - PPM_LOG_LEVEL=INFO
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/v1/health"]
      interval: 30s
      timeout: 5s
      retries: 3

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9091:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=7d'
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=ppm_admin
      - GF_USERS_ALLOW_SIGN_UP=false
    restart: unless-stopped

volumes:
  ppm_data:
  prometheus_data:
  grafana_data:
"""


### docker/.env.example ###
# NOTE: Save as docker/.env.example (no .py extension)

ENV_EXAMPLE_CONTENT = """
# LLM Backend
PPM_LLM_BACKEND=anthropic           # anthropic | openai | ollama
PPM_LLM_MODEL=claude-sonnet-4-20250514

# API Keys
ANTHROPIC_API_KEY=sk-ant-...
OPENAI_API_KEY=sk-...               # required if embedder_backend=openai

# Embedder
PPM_EMBEDDER_BACKEND=openai         # openai | nomic | bge
PPM_EMBEDDER_MODEL=text-embedding-3-small
PPM_EMBEDDER_DIM=1536

# Storage paths (override for docker volumes)
PPM_SQLITE_PATH=./ppm_data/ppm.db
PPM_QDRANT_PATH=./ppm_data/qdrant
PPM_KUZU_PATH=./ppm_data/kuzu

# Momentum tuning
PPM_DECAY_LAMBDA_BASE=0.85
PPM_CONTEXT_SWITCH_THRESHOLD=0.40
PPM_SLERP_STEP_SIZE=0.30

# Staging
PPM_SLOT_COUNT=10
PPM_AUTO_INJECT_THRESHOLD=0.80
PPM_HOT_THRESHOLD=0.50
PPM_MAX_INJECT_TOKENS=4096
PPM_SLOT_TTL_SECONDS=120

# Observability
PPM_LOG_LEVEL=INFO
PPM_METRICS_PORT=9090
PPM_OTEL_ENDPOINT=                  # set to enable tracing (e.g. http://jaeger:4317)
"""


### pyproject.toml ###
# NOTE: Save as pyproject.toml in project root (no .py extension)

PYPROJECT_CONTENT = """
[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.backends.legacy:build"

[project]
name = "ppm"
version = "0.1.0"
description = "Predictive Push Memory — anticipatory memory brain for LLMs"
requires-python = ">=3.12"
readme = "README.md"
license = { text = "MIT" }

dependencies = [
    # Core async
    "fastapi>=0.115.0",
    "uvicorn[standard]>=0.30.0",
    "websockets>=12.0",
    "aiosqlite>=0.20.0",
    "aiofiles>=23.0.0",
    # Storage
    "qdrant-client==1.16.2",
    "kuzu>=0.3.0",
    # Math
    "numpy>=1.26.0",
    "scipy>=1.12.0",
    "scikit-learn>=1.4.0",
    # Settings
    "pydantic>=2.6.0",
    "pydantic-settings>=2.2.0",
    # Observability
    "structlog>=24.0.0",
    "prometheus-client>=0.20.0",
    # HTTP client (CLI ingest)
    "httpx>=0.27.0",
]

[project.optional-dependencies]
anthropic = ["anthropic>=0.25.0"]
openai    = ["openai>=1.30.0"]
local     = ["sentence-transformers>=2.7.0"]
otel      = [
    "opentelemetry-api>=1.24.0",
    "opentelemetry-sdk>=1.24.0",
    "opentelemetry-exporter-otlp-proto-grpc>=1.24.0",
]
all = ["ppm[anthropic,openai,local,otel]"]
dev = [
    "pytest>=8.0.0",
    "pytest-asyncio>=0.23.0",
    "hypothesis>=6.100.0",
    "httpx>=0.27.0",   # for TestClient
]

[project.scripts]
ppm = "cli.ppm:main"

[tool.setuptools.packages.find]
where = ["."]
exclude = ["tests*", "docker*"]

[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
filterwarnings = [
    "ignore::DeprecationWarning",
    "ignore::UserWarning",
]

[tool.ruff]
line-length = 100
target-version = "py312"
select = ["E", "F", "I", "N", "UP", "RUF"]
ignore = ["E501", "N803", "N806"]

[tool.mypy]
python_version = "3.12"
ignore_missing_imports = true
strict = false
"""


### README.md ###
# NOTE: Save as README.md in project root (no .py extension)

README_CONTENT = """
# PPM — Predictive Push Memory

> An anticipatory memory brain for LLMs. Retrieves before you ask.

PPM is a novel memory architecture that **pushes** relevant context to the LLM
proactively — before the user's next message arrives — rather than reactively
retrieving it after the fact.

## Architecture

```
User Message
    │
    ├──[HOT PATH]────────────────────────────────────────────────────┐
    │  Observer: embed + intent + signals (< 2ms)                    │
    │  Staging Cache: grab pre-fetched AUTO slots (< 1ms)            │
    │  Reranker: refine against actual message (< 1ms)               │
    │  Knapsack Injector: fit into context budget (< 1ms)            │
    │  LLM: call with enriched context                               │
    └──[BACKGROUND]──────────────────────────────────────────────────┘
       Predictor: geodesic extrapolation → next predictions
       Prefetcher: fetch from vector + graph + annotation stores
       Feedback: hit/miss → bandit update → dataset log
       Write Pipeline: distill + chunk + store new memories
```

## The Core Innovation

**Kinematic trajectory in embedding space:**
Every conversation has a position (C_t), velocity (M_t), and acceleration (A_t)
in high-dimensional semantic space. PPM predicts where the conversation will be
in k steps and prefetches memories from that predicted location — before the
user asks.

**Self-improving via Bayesian bandits:**
Every prediction outcome updates a Beta-Bernoulli bandit.
The system knows its own confidence and gets smarter every turn.

## Quickstart

### Local Development

```bash
# 1. Install
pip install -e ".[all]"
cp docker/.env.example .env
# Edit .env with your API keys

# 2. Initialize stores
ppm init

# 3. Ingest your codebase
ppm ingest ./your_project/

# 4. Start the API
ppm serve

# 5. Chat
ppm chat
```

### Docker

```bash
cd docker
cp .env.example .env
# Edit .env with your API keys
docker-compose up -d

# View metrics dashboard
open http://localhost:3000  # Grafana (admin/ppm_admin)
open http://localhost:9091  # Prometheus
open http://localhost:8000/docs  # API docs
```

### REST API

```bash
# Create session
curl -X POST http://localhost:8000/v1/session

# Submit turn
curl -X POST http://localhost:8000/v1/turn \\
  -H "Content-Type: application/json" \\
  -d '{"session_id": "...", "message": "explain verify_token"}'

# Ingest a file
curl -X POST http://localhost:8000/v1/ingest \\
  -H "Content-Type: application/json" \\
  -d '{"content": "...", "source": "auth.py", "source_type": "code"}'

# Search memories
curl "http://localhost:8000/v1/memory/search?query=authentication&top_k=5"

# Health check
curl http://localhost:8000/v1/health
```

## Key Parameters

| Setting | Default | Description |
|---|---|---|
| `PPM_DECAY_LAMBDA_BASE` | 0.85 | Exponential decay for conversation state |
| `PPM_CONTEXT_SWITCH_THRESHOLD` | 0.40 | Cosine distance triggering momentum reset |
| `PPM_SLERP_STEP_SIZE` | 0.30 | Arc length per step in geodesic extrapolation |
| `PPM_AUTO_INJECT_THRESHOLD` | 0.80 | Bandit confidence for automatic injection |
| `PPM_MAX_INJECT_TOKENS` | 4096 | Token budget for context injection |
| `PPM_SLOT_TTL_SECONDS` | 120 | How long staged memories stay warm |

## Running Benchmarks

```bash
python tests/bench/bench_momentum.py   # math layer (target: all < 2ms P99)
python tests/bench/bench_staging.py    # staging layer
python tests/bench/bench_store.py      # store layer
```

## Project Structure

```
ppm/
├── math_core/          # Momentum, SLERP, knapsack, PPR, bandits
├── core/
│   ├── surface/        # ConversationObserver + IntentClassifier
│   ├── nerve/          # TrajectoryPredictor + ConversationStateManager
│   ├── staging/        # StagingCache + Prefetcher + Injector + Reranker
│   ├── write/          # MemoryDistiller + ConflictResolver + WritePipeline
│   ├── feedback/       # HitMissDetector + FeedbackLoop + TrajectoryDataset
│   └── session/        # SessionManager + SessionFactory
├── adapters/           # Embedder + LLM adapters (OpenAI, Anthropic, local)
├── core/store/         # VectorStore + MetaStore + OutboxWorker
├── api/                # FastAPI REST + WebSocket
├── observability/      # Prometheus metrics + OpenTelemetry tracing
├── cli/                # ppm CLI
├── docker/             # Dockerfile + docker-compose
└── tests/
    ├── unit/           # Per-module unit tests
    ├── integration/    # Cross-module integration tests
    └── bench/          # Latency benchmarks
```

## License

MIT
"""


### Final integration: wire observability into server ###
# Add this to api/server.py after the existing imports

OBSERVABILITY_ADDITIONS = """
# Add to api/server.py — import section:
from observability.middleware import ObservabilityMiddleware, add_metrics_endpoint
from observability.metrics import (
    record_turn, record_feedback,
    session_active, staging_evictions,
)
from observability.tracing import init_tracing

# Add after app = FastAPI(...):
app.add_middleware(ObservabilityMiddleware)
add_metrics_endpoint(app)

# Add to lifespan startup:
init_tracing(endpoint=os.getenv("PPM_OTEL_ENDPOINT"))

# Add to /v1/turn endpoint, after result = await session.turn(...):
record_turn(
    intent=result.intent.value,
    latency_s=result.latency_ms / 1000,
    tokens=result.tokens_injected,
    memories=result.memories_injected,
)

# Add to create_session:
session_active.inc()

# Add to close_session:
session_active.dec()
"""
