#!/usr/bin/env python3
"""
CDC Decode - Vault-anchored text generation from compressed seeds
Integrates with FlowTorch for on-device CDC inference
Uses TRUE hierarchical Helix encoding + FibPi3D vault tiles + SAT/morph guidance
"""
import os
import sys
import json
import hashlib
import platform
import struct
import numpy as np
from pathlib import Path
from typing import List, Tuple, Optional, Dict, Any

# Vault-anchored CDC imports
from .fibpi_adapter import compute_anchor
from .vault_client import ensure_vault_client
from .morph_table import MorphTable
from .glyphscope_code import ids_to_code, CodeSAT, token_class, is_valid_code, AST_POLICY_SHA256
from .glyphscope_json import JsonSAT, ids_to_json, json_token_class, JSON_VOCAB_SHA256

# Hierarchical Helix encoding (111=h, hhh=#, ###=:)
BIT_TO_LETTER = {
    '000': 'a', '001': 'b', '010': 'c', '011': 'd',
    '100': 'e', '101': 'f', '110': 'g', '111': 'h'
}

# Build letter->symbol map (aaa→A, bbb→B, ... hhh→H)
LETTER_TO_SYMBOL = {
    'aaa': 'A', 'bbb': 'B', 'ccc': 'C', 'ddd': 'D',
    'eee': 'E', 'fff': 'F', 'ggg': 'G', 'hhh': 'H'
}

# Symbol->glyph (AAA→@, BBB→#, etc)
SYMBOL_TO_GLYPH = {
    'AAA': '@', 'BBB': '#', 'CCC': '$', 'DDD': '%',
    'EEE': '^', 'FFF': '&', 'GGG': '*', 'HHH': '!'
}

HELIX64_AVAILABLE = True  # Built-in encoding

# Helix64 = Base64 alphabet (64 symbols)
HELIX64_ALPHABET = (
    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"  # 0-25
    "abcdefghijklmnopqrstuvwxyz"  # 26-51
    "0123456789"                   # 52-61
    "+/"                           # 62-63
)

def id_to_symbol(i: int) -> str:
    """Convert 0-63 ID to Helix64 base64 symbol."""
    return HELIX64_ALPHABET[i % 64]

def symbol_to_id(sym: str) -> int:
    """Convert Helix64 symbol to 0-63 ID."""
    return HELIX64_ALPHABET.index(sym) if sym in HELIX64_ALPHABET else 0

# Note: Code grammar SAT now handled by CodeSAT class from glyphscope_code.py
# which provides stateful constraints based on Python syntax

# Receipt mode: lite (hashes only) or full (include explicit allowed_ids for deep audits)
RECEIPT_MODE = os.environ.get("GLYPHSCOPE_RECEIPT_MODE", "lite")  # lite|full

def _env_fingerprint() -> Dict[str, str]:
    """
    Capture environment fingerprint for time/OS invariance checks.
    Returns minimal platform info to detect cross-platform determinism.
    """
    import time
    fp = {
        "python_version": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
        "platform": platform.system(),
        "platform_release": platform.release(),
        "machine": platform.machine(),
        "timestamp_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
    }
    # CPU flags (x86 only)
    try:
        if platform.machine() in ("x86_64", "AMD64", "i386", "i686"):
            import cpuinfo
            cpu = cpuinfo.get_cpu_info()
            fp["cpu_flags"] = ",".join(sorted(cpu.get("flags", []))[:10])  # First 10 flags
    except:
        pass
    return fp

# Import CDR coherence for quality measurement
try:
    from helixcdc.metrics.cdr_coherence import cdr_anchor_coherence
    CDR_AVAILABLE = True
except ImportError:
    CDR_AVAILABLE = False

# Initialize morph table and SAT guidance
try:
    MORPH = MorphTable()
    SAT_EN = (os.environ.get("HELIX_SAT", "1") == "1")
except Exception:
    MORPH = None
    SAT_EN = False


class CDCDecoder:
    """
    Lightweight CDC decoder for text generation.

    Uses:
    - Helix64 tokenizer (6-bit, vocab_size=68)
    - Anchor-based coherence (CDR)
    - Deterministic seed-based generation
    - Receipt generation for provenance
    """

    def __init__(self, seed_path: Optional[str] = None, device: str = "cpu", vault_path: Optional[str] = None):
        """
        Initialize CDC decoder with vault support.

        Args:
            seed_path: Path to .hxz seed or superglyph manifest
            device: "cpu" or "cuda"
            vault_path: Path to Crystal Vault directory
        """
        self.device = device
        self.seed_path = seed_path
        self.vault_path = vault_path or os.environ.get("HELIX_VAULT_PATH")

        # Use built-in hierarchical encoding
        self.bit_to_letter = BIT_TO_LETTER
        self.letter_to_symbol = LETTER_TO_SYMBOL
        self.symbol_to_glyph = SYMBOL_TO_GLYPH

        # Load seed if provided
        self.seed_data = None
        self.seed_hash = None
        self.seed_sha256 = None
        if seed_path and Path(seed_path).exists():
            self.load_seed(seed_path)

        # Initialize vault client
        self.vault = ensure_vault_client(vault_path=self.vault_path, seed_path=seed_path)

        # Initialize stateful SAT for code and JSON generation
        self.code_sat = CodeSAT()
        self.json_sat = JsonSAT()

        # Track current task for mode detection
        self._current_task = None

    def load_seed(self, seed_path: str):
        """Load .hxz seed or superglyph manifest."""
        path = Path(seed_path)

        if path.suffix == '.hxz':
            # Load .hxz capsule
            self.seed_data = path.read_bytes()
            self.seed_hash = hashlib.sha256(self.seed_data).hexdigest()
        elif path.suffix == '.json':
            # Load superglyph manifest
            manifest = json.loads(path.read_text())
            self.seed_data = json.dumps(manifest, sort_keys=True).encode()
            self.seed_hash = hashlib.sha3_512(self.seed_data).hexdigest()

    def generate_from_anchor(
        self,
        prompt: str,
        max_new_tokens: int = 64,
        temperature: float = 0.7,
        seed: Optional[int] = None,
        overlay_schedule: Optional[Dict[str, Any]] = None,
        phase_omega: float = 0.0,
        grammar_file: Optional[str] = None,
        task: Optional[str] = None
    ) -> Tuple[str, Dict[str, Any]]:
        """
        Generate text using CDC anchor-based method.

        This is a lightweight generator that uses:
        1. Helix64 tokenization
        2. Anchor coherence for quality
        3. Deterministic seeding
        4. Optional grammar-constrained decoding (via JsonSAT or CodeSAT)

        Args:
            prompt: Input prompt
            max_new_tokens: Tokens to generate
            temperature: Sampling temperature
            seed: Random seed for determinism
            overlay_schedule: Optional overlay schedule for phase-modulated alpha
            phase_omega: Phase modulation frequency
            grammar_file: Optional GBNF grammar file for constrained decoding
            task: Task type ("json", "code", "math") - determines vocab and SAT

        Returns:
            (generated_text, trace_dict)
        """
        # Use basin_seed for anchor computation if provided
        effective_seed_hash = self.seed_hash or "0"*64
        if seed is not None:
            np.random.seed(seed)
            # Hash the basin_seed to create unique anchor space per basin
            effective_seed_hash = hashlib.sha256(f"basin_{seed}".encode()).hexdigest()

        # For code generation with scaffold, start from empty prompt
        # The scaffold will provide the exact function header
        # (Prompt is used for seeding only, not for token initialization)
        # BASIN CONTROL: Seed initial IDs with basin-specific hash for diversity
        if seed is not None:
            # Use first few bytes of effective_seed_hash as initial token IDs
            seed_bytes = bytes.fromhex(effective_seed_hash[:16])  # 8 bytes
            prompt_ids = [b % 64 for b in seed_bytes]  # Map to Helix64 range
        else:
            prompt_ids = []  # Empty - scaffold drives generation from START

        # Parse overlay schedule
        # Format: {"alpha0": 0.0, "alpha1": 1.0, "phase_deg": 0.0}
        # α(t) = alpha0 + (alpha1 - alpha0) * sin((phase_deg + omega*t) * π/180)
        schedule = overlay_schedule or {}
        alpha0 = schedule.get("alpha0", 0.0)
        alpha1 = schedule.get("alpha1", 0.0)
        phase_base_deg = schedule.get("phase_deg", 0.0)

        # Initialize trace
        trace = {
            "method": "cdc_anchor",
            "prompt_tokens": len(prompt_ids),
            "seed_hash": self.seed_hash[:16] if self.seed_hash else None,
        }

        # Track current task for mode switching
        self._current_task = task

        # Track grammar if provided (actual enforcement via FlowTorch backend)
        # FlowTorch supports grammar-constrained decoding through SAT/morph guidance
        grammar_active = False
        pruned_steps = 0  # Track how many steps actually applied constraints
        considered_total = 0  # Total candidate tokens considered
        pruned_total = 0  # Total tokens pruned

        # Explicit JSON mode detection (prefer task param over filename inference)
        json_mode = (task == "json")

        if grammar_file and os.path.exists(grammar_file):
            grammar_active = True
            grammar_sha = hashlib.sha256(Path(grammar_file).read_bytes()).hexdigest()
            trace["grammar_requested"] = True
            trace["grammar_file"] = os.path.basename(grammar_file)
            trace["grammar_sha256"] = grammar_sha
            trace["grammar_backend"] = "flowtorch_sat"  # Uses FlowTorch + SAT constraints

            # Validate grammar file matches task
            if json_mode:
                trace["grammar_mode"] = "json_constrained"
            elif task == "code":
                trace["grammar_mode"] = "code_constrained"

        # Generate tokens using vault-anchored CDC with stateful SAT
        generated_ids = prompt_ids.copy()
        symbols_out = ''.join(id_to_symbol(i) for i in prompt_ids)  # Track Helix64 symbols
        tokens_out = []  # Track code tokens for SAT state
        coherence_scores = []
        alpha_per_step = []  # Track α(t) for depth analysis
        trace_steps = []  # Tile receipts per step

        # Track early stop and macro usage
        early_stop_reason = None
        macro_pos = None  # Token position where macro fired
        tiles_before_macro = None  # Number of tiles before macro
        macro_uses = 0  # Count of macro uses in this generation

        # Reset SAT state for fresh generation
        self.code_sat.reset()
        if json_mode:
            self.json_sat.reset()

        for step in range(max_new_tokens):
            # Compute phase-modulated overlay alpha for this step
            # α(t) = alpha0 + (alpha1 - alpha0) * sin((phase_base + omega*t) * π/180)
            phase_t_deg = phase_base_deg + phase_omega * step
            phase_t_rad = phase_t_deg * np.pi / 180.0
            alpha_t = alpha0 + (alpha1 - alpha0) * np.sin(phase_t_rad)
            alpha_per_step.append(alpha_t)

            # Create anchor from current context
            ctx_bytes = bytes(generated_ids[-32:])  # Last 32 tokens as context

            # Measure coherence if CDR available
            if CDR_AVAILABLE:
                coherence = cdr_anchor_coherence(ctx_bytes)
                coherence_scores.append(coherence)
            else:
                coherence = 0.5

            # VAULT-ANCHORED SAMPLING: FibPi3D anchor → vault tile → SAT/morph-guided selection
            # Compute FibPi3D anchor for this generation step
            anchor = compute_anchor(ctx_bytes, step, effective_seed_hash)

            # Fetch tile from vault using anchor
            tile = self.vault.get_segment(seed_sha256=effective_seed_hash, anchor_key=anchor.key)
            tile_bytes = tile["bytes"]
            tile_sha = tile["sha256"]

            # Get SAT-allowed next IDs using task-specific SAT
            # Use JsonSAT for JSON mode, CodeSAT for code mode
            tail_tokens = tokens_out[-8:] if len(tokens_out) >= 8 else tokens_out

            # Track whether this step used widen fallback
            widened_step = False

            if json_mode and grammar_active:
                # Use JsonSAT for grammar-constrained JSON generation
                allowed_ids = self.json_sat.allowed_ids(tail_tokens)

                # Empty set guard: widen if FSM returned empty
                if not allowed_ids:
                    allowed_ids = self.json_sat.widen(tail_tokens)
                    widened_step = True
                    # Mark this step as widened (FSM edge case)
                    if not hasattr(self, '_json_widen_count'):
                        self._json_widen_count = 0
                    self._json_widen_count += 1

                # Endgame clamp: if budget can't cover minimal close path, prefer closers
                remaining = max_new_tokens - step
                try:
                    min_close = self.json_sat.min_close_steps()
                except Exception:
                    min_close = 0
                if remaining <= (min_close + 1):
                    closers = set(self.json_sat.closing_ids())
                    if closers:
                        clamped = [i for i in allowed_ids if i in closers]
                        if clamped:
                            allowed_ids = clamped
            else:
                # Use CodeSAT for code generation
                allowed_ids = self.code_sat.allowed_ids(tail_tokens)

            # If morph table available with allowed_ids method, intersect with it
            if SAT_EN and MORPH and hasattr(MORPH, 'allowed_ids') and not json_mode:
                try:
                    # Morph table works on Helix64 symbols
                    tail_symbols = symbols_out[-8:] if len(symbols_out) >= 8 else symbols_out
                    morph_allowed = MORPH.allowed_ids(tail_symbols)
                    if morph_allowed:
                        # Intersect CodeSAT with morph table
                        allowed_ids = [i for i in allowed_ids if i in morph_allowed]
                        if not allowed_ids:  # fallback if intersection is empty
                            allowed_ids = self.code_sat.allowed_ids(tail_tokens)
                except Exception:
                    pass  # Keep CodeSAT allowlist

            # Track JSON grammar constraint effectiveness
            # JsonSAT already applied constraints above at line 289
            if json_mode and grammar_active:
                # Measure constraint impact by comparing with unconstrained baseline
                # (For honest receipts: enforced=true only if JsonSAT actually pruned tokens)
                unconstrained_count = 64  # Full Helix64 vocab size
                constrained_count = len(allowed_ids)
                considered_total += unconstrained_count

                # Token-space proof (first step only) - show JsonSAT is working
                if step == 0 and constrained_count > 0:
                    sample_ids = allowed_ids[:min(8, constrained_count)]
                    from .glyphscope_json import json_id_to_token
                    sample_tokens = [json_id_to_token(i) for i in sample_ids]
                    trace["json_sat_debug"] = {
                        "sample_ids": sample_ids,
                        "sample_tokens": sample_tokens,
                        "vocab": "JSON_VOCAB",
                        "state": self.json_sat.state,
                        "morph_intersection": False,  # Morph disabled for JSON mode
                        "note": "JsonSAT constraining token IDs to valid JSON tokens"
                    }

                # Count pruning (constraint effectiveness)
                # Don't count widened steps - those aren't real constraints
                if unconstrained_count > constrained_count and not widened_step:
                    pruned_steps += 1
                    pruned_total += (unconstrained_count - constrained_count)

            # Score each allowed candidate deterministically using vault tile
            # Use tile_sha256 digest + token_class + SAT state for scoring stability
            def score(cand: int) -> int:
                m = hashlib.shake_256()
                m.update(b"HELIX|cdc_sample|v1|")
                m.update(bytes.fromhex(self.seed_hash or "0"*64))
                m.update(anchor.key.encode())
                m.update(hashlib.sha256(ctx_bytes).digest())
                m.update(bytes.fromhex(tile_sha))  # Use tile hash digest
                # Add token class for scoring stability (KW vs DIGIT vs NAME, etc)
                # Use task-appropriate token class function
                if json_mode:
                    tok_class = json_token_class(cand)
                else:
                    tok_class = token_class(cand)
                m.update(b"class:" + tok_class.encode())
                # Bake SAT state into salt (prevents drift if class collisions happen)
                sat_state_for_score = self.json_sat.state if json_mode else self.code_sat.state
                m.update(b"state:" + sat_state_for_score.encode())
                # Add depth for structural context
                depth = self.json_sat.depth if json_mode else getattr(self.code_sat, 'depth', 0)
                m.update(b"depth:" + str(depth).encode())
                # Use appropriate SAT schema hash
                if json_mode:
                    m.update(b"sat:json_vocab_" + JSON_VOCAB_SHA256.encode())
                else:
                    from .glyphscope_code import SAT_SCHEMA_SHA256
                    m.update(b"sat:" + SAT_SCHEMA_SHA256.encode())
                # Prefer tokens that reduce depth when we are near endgame (deterministic bonus)
                if json_mode:
                    try:
                        is_closer = cand in set(self.json_sat.closing_ids())
                        if is_closer:
                            m.update(b"closer:+1")
                    except Exception:
                        pass
                # Use 16-bit packing for future-proofing (matches allowed-set fingerprint)
                m.update(struct.pack(">H", cand))
                return int.from_bytes(m.digest(8), "little")

            # Deterministic tie-breaks: sort by (score, id) for stable ordering
            ranked = sorted(allowed_ids, key=lambda i: (score(i), i))

            # Compute allowed-set fingerprint (hash of sorted allowed IDs)
            # Use 16-bit packing for future-proofing (supports IDs up to 65535)
            buf = b"".join(struct.pack(">H", i) for i in sorted(allowed_ids))
            sat_allowed_sha256 = hashlib.sha256(buf).hexdigest()

            # Determinism under SAT/scaffold:
            # Force top-1 during scaffold, but allow choice at body start (for macro selection)
            in_scaffold = (self.code_sat.scaffold_pos > 0)
            in_strict_sat = (self.code_sat.state in ("START_OR_TOP", "AFTER_DEF", "IN_PARAMS",
                                                       "AFTER_CLOSE_PAREN", "AFTER_COLON", "AFTER_NEWLINE"))
            det_env = (os.environ.get("GLYPHSCOPE_DETERMINISTIC", "1") == "1")

            # Determine selection zone and k
            if in_scaffold:
                selection_zone = "scaffold"
                k_selected = 1
                next_id = ranked[0]
            elif in_strict_sat and det_env:
                selection_zone = "strict_sat"
                k_selected = 1
                next_id = ranked[0]
            elif self.code_sat.state == "IN_BODY_START":
                selection_zone = "body_start"
                if det_env:
                    k_selected = 1
                    next_id = ranked[0]
                else:
                    temp = float(temperature) * (1.0 + alpha_t * 0.2)
                    k = max(1, int((1.0 + temp) * 4))
                    k_selected = k
                    choice_idx = min(k - 1, len(ranked) - 1) if k > 1 else 0
                    next_id = ranked[choice_idx]
            else:  # IN_BODY_EXPR or fallback
                selection_zone = "body_expr"
                temp = float(temperature) * (1.0 + alpha_t * 0.2)
                k = max(1, int((1.0 + temp) * 4))
                k_selected = k
                choice_idx = min(k - 1, len(ranked) - 1) if k > 1 else 0
                next_id = ranked[choice_idx]

            next_sym = id_to_symbol(next_id)

            # Convert ID to actual token via appropriate vocab
            if json_mode:
                from .glyphscope_json import json_id_to_token
                next_token = json_id_to_token(next_id)
            else:
                from .glyphscope_code import id_to_token
                next_token = id_to_token(next_id)

            generated_ids.append(next_id)
            symbols_out += next_sym
            tokens_out.append(next_token)  # Track for SAT state

            # Keep JsonSAT state in sync and stop immediately if complete
            if json_mode and grammar_active:
                try:
                    self.json_sat.consume(next_token)
                    if self.json_sat.is_terminal():
                        early_stop_reason = "terminal"
                        # Build final step_rec before breaking
                        sat_state = self.json_sat.state
                        step_rec = {
                            "step": step,
                            "anchor": {"kind": anchor.kind, "key": anchor.key},
                            "tile_sha256": tile_sha,
                            "tile_len": len(tile_bytes),
                            "sat_state": sat_state,
                            "sat_allowed": len(allowed_ids),
                            "sat_allowed_sha256": sat_allowed_sha256[:16],
                            "selection_zone": selection_zone,
                            "k_selected": k_selected,
                            "chosen_in_allowed": (next_id in allowed_ids),
                            "choice": next_id,
                            "symbol": next_sym,
                        }
                        trace_steps.append(step_rec)
                        break
                except Exception:
                    pass

            # Validate choice was in allowed set (integrity check)
            chosen_in_allowed = next_id in allowed_ids

            # Log tile receipt for this step with state provenance
            sat_state = self.json_sat.state if json_mode else self.code_sat.state
            step_rec = {
                "step": step,
                "anchor": {"kind": anchor.kind, "key": anchor.key},
                "tile_sha256": tile_sha,
                "tile_len": len(tile_bytes),
                "sat_state": sat_state,  # Current FSM state (JsonSAT or CodeSAT)
                "sat_allowed": len(allowed_ids),
                "sat_allowed_sha256": sat_allowed_sha256[:16],  # Fingerprint of allowed set
                "selection_zone": selection_zone,  # scaffold | strict_sat | body_start | body_expr
                "k_selected": k_selected,  # The k value used for selection
                "chosen_in_allowed": chosen_in_allowed,  # Integrity gate
                "choice": next_id,
                "symbol": next_sym,
            }
            # Full receipt mode: include explicit allowed_ids for deep audits
            if RECEIPT_MODE == "full":
                step_rec["allowed_ids"] = sorted(allowed_ids)
            trace_steps.append(step_rec)

            # Early stop if @LEV_DP macro used (completes function body)
            from .glyphscope_code import id_of, BODY_HINT
            if next_token == "@LEV_DP":
                # Macro completes the function - stop generation
                early_stop_reason = "macro"
                macro_pos = step
                tiles_before_macro = len(trace_steps)
                macro_uses += 1
                self.code_sat.macro_uses += 1  # Increment SAT counter
                break

            # Early stop if body_hint completed (baseline function)
            if (self.code_sat.use_body_hint and
                self.code_sat.body_scaffold_pos >= len(BODY_HINT)):
                # Body hint completes the function - stop generation
                early_stop_reason = "body_hint"
                break

        # If we didn't early stop, reason is max_tokens
        if early_stop_reason is None:
            early_stop_reason = "max_tokens"

        # Decode IDs to text using appropriate glyphscope
        # This converts Helix64 IDs (transport) to task-specific tokens (semantics)
        # Handle expansion cap gracefully
        try:
            if json_mode:
                generated_text = ids_to_json(generated_ids)
            else:
                generated_text = ids_to_code(generated_ids)
        except RuntimeError as e:
            if "exceeds expansion cap" in str(e):
                early_stop_reason = "expansion_cap"
                # Generate partial text up to the macro
                if json_mode:
                    generated_text = ids_to_json(generated_ids[:-1]) if len(generated_ids) > 1 else ""
                else:
                    generated_text = ids_to_code(generated_ids[:-1]) if len(generated_ids) > 1 else ""
            else:
                raise

        # Validate decoded text (no NULs, mostly printable)
        text_valid = is_valid_code(generated_text)

        # Compile check: does it parse as valid Python?
        compiled = False
        compile_error = None
        try:
            compile(generated_text, "<cdc>", "exec")
            compiled = True
        except Exception as e:
            compile_error = str(e).splitlines()[0] if str(e) else "Unknown error"

        # AST safety gate: validate only safe nodes (no imports, exec, etc.)
        from .glyphscope_code import validate_ast_safe
        ast_ok, ast_violation = validate_ast_safe(generated_text)

        # Compute depth metrics
        metrics = {}

        if coherence_scores:
            # Legacy metrics
            metrics["resonance_index"] = float(np.std(coherence_scores))

            # Stability S: inverse of coherence variance (higher = more stable)
            coh_var = np.var(coherence_scores)
            metrics["stability"] = float(1.0 / (coh_var + 1e-6))

            # Spectral resonance at drive frequency (R_s) and phase lag (φ_l)
            # ω here is the per-token phase advance in radians
            omega_rad = (phase_omega or 0.0) * np.pi / 180.0
            if omega_rad != 0.0:
                t = np.arange(len(coherence_scores), dtype=np.float64)
                coh = np.asarray(coherence_scores, dtype=np.float64)
                # Complex projection of coherence onto e^{-i ω t}
                proj = np.sum(coh * np.exp(-1j * omega_rad * t)) / max(1, len(coh))
                R_s = np.abs(proj)
                # Phase lag of coherence relative to drive (degrees)
                phi_l_deg = float((-np.angle(proj)) * 180.0 / np.pi)  # negative = lag
                # Wrap to [0, 360)
                phi_l_deg = (phi_l_deg % 360.0 + 360.0) % 360.0
                metrics["R_spectral"] = float(R_s)
                metrics["phase_lag_deg"] = float(phi_l_deg)

        if alpha_per_step:
            # Sensitivity σΞ: variance of overlay alpha over steps
            # (measures how much phase drive modulates the overlay)
            metrics["sigma_overlay"] = float(np.std(alpha_per_step))
            metrics["alpha_mean"] = float(np.mean(alpha_per_step))
            metrics["alpha_range"] = [float(min(alpha_per_step)), float(max(alpha_per_step))]

        # Scaffold and macro metadata
        from .glyphscope_code import (
            SCAFFOLD, BODY_HINT, CODE_VOCAB, VOCAB_META, LEV_DP_SHA256,
            GLYPHSCOPE_VERSION, SAT_SCHEMA_SHA256, TOKEN_CLASS_FROZEN_SHA256, VOCAB_META_FROZEN_SHA256,
            NEWLINE_STYLE
        )
        vocab_sha256 = hashlib.sha256("".join(CODE_VOCAB).encode()).hexdigest()
        scaffold_used = self.code_sat.scaffold_pos if hasattr(self.code_sat, 'scaffold_pos') else 0
        body_scaffold_used = self.code_sat.body_scaffold_pos if hasattr(self.code_sat, 'body_scaffold_pos') else 0

        # Check if @LEV_DP macro was used (check tokens before expansion)
        macro_used = "@LEV_DP" in tokens_out
        macro_sha256 = LEV_DP_SHA256 if macro_used else None

        # Compute scaffold hash (actual scaffold used, may differ from SCAFFOLD if overridden)
        scaffold_actual = getattr(self.code_sat, "scaffold", SCAFFOLD)
        scaffold_sha256 = hashlib.sha256("".join(scaffold_actual).encode()).hexdigest()

        # Macro selection mode
        macro_mode = getattr(self.code_sat, "macro_mode", "auto")
        if macro_mode == "force":
            macro_selection = "forced"
        elif macro_used:
            macro_selection = "chosen"
        else:
            macro_selection = "n/a"

        # Early-stop invariants
        assert early_stop_reason in {"macro", "body_hint", "max_tokens", "expansion_cap", "terminal"}, f"Invalid stop reason: {early_stop_reason}"
        assert all(s["chosen_in_allowed"] for s in trace_steps), "Integrity violation: choice not in allowed set"

        # Selection-zone summary rollup (for quick eyeballing)
        from collections import Counter
        zone_counts = Counter(s["selection_zone"] for s in trace_steps)

        # Macro catalog (declared macros and what was expanded)
        from .glyphscope_code import LEV_DP_BODY, MAX_EXPANSION_BYTES
        macro_catalog = {
            "macros_declared": ["@LEV_DP"],
            "macros_enabled": self.code_sat.allow_macros,
            "max_expansion_bytes": MAX_EXPANSION_BYTES,
            "macros_expanded": []
        }
        if macro_used:
            macro_catalog["macros_expanded"].append({
                "name": "@LEV_DP",
                "sha256": LEV_DP_SHA256[:16],
                "bytes": len(LEV_DP_BODY.encode())
            })

        # Update trace with tile stream receipts + glyphscope metadata
        trace.update({
            "token_budget_requested": max_new_tokens,
            "token_budget_used": len(trace_steps),
            "temperature_requested": temperature,
            "top_p_requested": None,  # top-p not yet implemented
            "generated_tokens": len(generated_ids) - len(prompt_ids),
            "mean_coherence": float(np.mean(coherence_scores)) if coherence_scores else None,
            "output_length": len(generated_text),
            "text_valid": text_valid,
            "compiled": compiled,
            "compile_error": compile_error if not compiled else None,
            "ast_ok": ast_ok,
            "ast_violation": ast_violation,
            "raw_symbols": symbols_out,  # Helix64 base64 symbols for receipts
            "scaffold_applied": scaffold_used > 0,
            "scaffold_tokens_used": f"{scaffold_used}/{len(scaffold_actual)}" if scaffold_used > 0 else None,
            "scaffold_sha256": scaffold_sha256[:16],
            "scaffold_len": len(scaffold_actual),
            "body_scaffold_applied": body_scaffold_used > 0,
            "body_scaffold_tokens_used": f"{body_scaffold_used}/{len(BODY_HINT)}" if body_scaffold_used > 0 else None,
            "body_hint_enabled": self.code_sat.use_body_hint,
            "macro_used": "@LEV_DP" if macro_used else None,
            "macro_sha256": macro_sha256[:16] if macro_sha256 else None,
            "macro_selection": macro_selection,
            "macro_pos": macro_pos,
            "tiles_before_macro": tiles_before_macro,
            "macro_quota": self.code_sat.macro_quota,
            "macro_uses": macro_uses,
            "macro_mode": macro_mode,
            "early_stop_reason": early_stop_reason,
            "top_p_effective": False,  # top-p not implemented (only temperature affects selection)
            "glyphscope_version": GLYPHSCOPE_VERSION,
            "glyphscope_vocab_sha256": vocab_sha256[:16],
            "token_class_sha256": TOKEN_CLASS_FROZEN_SHA256[:16],
            "vocab_meta_sha256": VOCAB_META_FROZEN_SHA256[:16],
            "sat_schema_sha256": SAT_SCHEMA_SHA256[:16],
            "ast_policy_sha256": AST_POLICY_SHA256[:16],
            "scaffold_override_rejected": getattr(self.code_sat, "scaffold_override_rejected", False),
            "macros_allowed": self.code_sat.allow_macros,
            "macro_catalog": macro_catalog,
            "selection_zone_counts": dict(zone_counts),
            "newline_style": NEWLINE_STYLE,
            "receipt_mode": RECEIPT_MODE,
            "env_fingerprint": _env_fingerprint(),
            "metrics": metrics,
            "overlay_schedule": {
                "alpha0": alpha0,
                "alpha1": alpha1,
                "phase_base_deg": phase_base_deg,
                "phase_omega": phase_omega,
            } if (alpha0 != 0 or alpha1 != 0) else None,
            "tile_stream": trace_steps,  # Per-step vault tile receipts
            "vault_anchored": True,
        })

        # Add grammar enforcement metrics (honest reporting)
        if grammar_active and json_mode:
            import math

            enforced = pruned_steps > 0  # Only true if constraints actually pruned
            prune_rate = (pruned_total / considered_total) if considered_total > 0 else 0.0

            # Compute coverage and allowed_avg for credibility checks
            allowed_avg = (sum(s.get("sat_allowed", 64) for s in trace_steps) / len(trace_steps)) if trace_steps else 64
            coverage = (sum(1 for s in trace_steps if s.get("sat_allowed", 64) < 64) / len(trace_steps)) if trace_steps else 0.0

            # Branch complexity: average bits needed to choose next token
            # log2(sat_allowed) - expect << 6 when constraints work
            branch_bits_avg = 6.0  # default if no steps
            if trace_steps:
                total_bits = sum(math.log2(s.get("sat_allowed", 64)) for s in trace_steps)
                branch_bits_avg = total_bits / len(trace_steps)

            # Terminal state check: is output complete?
            terminal = self.json_sat.is_terminal()

            # Widen count (should be 0 in normal operation)
            widen_count = getattr(self, '_json_widen_count', 0)

            # Credibility: enforced AND credible constraint strength AND complete AND no fallbacks
            credible = (coverage >= 0.2) and (allowed_avg <= 48) and terminal and (widen_count == 0)

            trace["grammar_enforcement"] = {
                "enforced": enforced,
                "pruned_steps": pruned_steps,
                "total_steps": len(trace_steps),
                "prune_rate": round(prune_rate, 4),
                "allowed_avg": round(allowed_avg, 2),
                "coverage": round(coverage, 3),
                "branch_bits_avg": round(branch_bits_avg, 3),  # Branching complexity
                "terminal": terminal,  # Is output complete?
                "credible": credible,  # Is enforcement strong, complete, and clean?
                "widen_count": widen_count,  # Empty set fallbacks
                "vocab": "JSON_VOCAB",
                "vocab_sha256": JSON_VOCAB_SHA256[:16],
                "note": "JsonSAT constraints applied - credible=true only if strong+complete+no-fallbacks"
            }

        return generated_text, trace

    def generate_simple(
        self,
        prompt: str,
        max_new_tokens: int = 64,
        temperature: float = 0.7
    ) -> str:
        """
        Simple generation interface (returns only text).

        Args:
            prompt: Input prompt
            max_new_tokens: Tokens to generate
            temperature: Sampling temperature

        Returns:
            Generated text
        """
        text, _ = self.generate_from_anchor(prompt, max_new_tokens, temperature)
        return text


def cdc_decode_batch(
    prompts: List[str],
    max_new_tokens: int = 64,
    temperature: float = 0.7,
    seed_path: Optional[str] = None,
    device: str = "cpu",
    extras: Optional[Dict[str, Any]] = None
) -> Tuple[List[str], Dict[str, Any]]:
    """
    Batch CDC decode for multiple prompts.

    Args:
        prompts: List of input prompts
        max_new_tokens: Tokens to generate per prompt
        temperature: Sampling temperature
        seed_path: Path to .hxz seed (optional)
        device: "cpu" or "cuda"
        extras: Optional dict with geometry/quantum params

    Returns:
        (generated_texts, trace_dict)
    """
    extras = extras or {}
    overlay_alpha = float(extras.get("overlay_alpha", 0.0))
    fib_scale = extras.get("fib_scale", 1.0)
    fib_shift_deg = extras.get("fib_shift_deg", 0.0)
    temp_override = extras.get("temperature", temperature)

    # Overlay schedule params (per-step phase modulation)
    overlay_schedule = extras.get("overlay_schedule")
    phase_omega = extras.get("phase_omega", None)  # deg/token or None

    # New: optional frequency-based drive
    freq_hz = extras.get("freq_hz", None)  # float Hz
    dt_ms = extras.get("dt_ms_per_token", None)  # float ms/token

    # If caller supplied F and Δt but not ω, infer ω = 360 * F * (Δt/1000)
    if phase_omega is None and (freq_hz is not None and dt_ms is not None):
        phase_omega = 360.0 * float(freq_hz) * (float(dt_ms) / 1000.0)
    if phase_omega is None:
        phase_omega = 0.0

    decoder = CDCDecoder(seed_path=seed_path, device=device)

    outputs = []
    traces = []

    for i, prompt in enumerate(prompts):
        # Basin control override: use basin_seed if provided
        if "basin_seed" in extras:
            seed_base = extras["basin_seed"]
        else:
            # Use prompt index as seed for determinism
            # Apply geometry params to seed calculation
            seed_base = 42 + i
            if fib_scale != 1.0:
                seed_base = int(seed_base * fib_scale) % 1000000
            if fib_shift_deg != 0.0:
                seed_base = (seed_base + int(fib_shift_deg * 100)) % 1000000

        text, trace = decoder.generate_from_anchor(
            prompt,
            max_new_tokens=max_new_tokens,
            temperature=temp_override * (1.0 + overlay_alpha * 0.1),  # Overlay affects temp
            seed=seed_base,
            overlay_schedule=overlay_schedule,
            phase_omega=phase_omega
        )
        outputs.append(text)
        traces.append(trace)

    # Aggregate trace
    agg_trace = {
        "method": "cdc_batch",
        "batch_size": len(prompts),
        "per_prompt_traces": traces,
        "mean_coherence": float(np.mean([t.get("mean_coherence", 0) for t in traces if t.get("mean_coherence")])) if traces else None,
        "overlay_alpha": overlay_alpha,
        "fib_scale": fib_scale,
        "fib_shift_deg": fib_shift_deg,
        "seed_used": bool(seed_path),
        "vault_anchored": any(t.get("vault_anchored", False) for t in traces),
        "tile_stream": traces[0].get("tile_stream", []) if traces else [],  # First prompt's tiles
    }

    # Echo drive provenance clearly (separates chosen drive F from measured cadence Δt from computed ω)
    agg_trace.setdefault("overlay", {})
    agg_trace["overlay"].update({
        "freq_hz": freq_hz,
        "dt_ms_per_token": dt_ms,
        "phase_omega_deg_per_token": phase_omega,
    })

    # Aggregate depth metrics across batch
    all_metrics = [t.get("metrics", {}) for t in traces if t.get("metrics")]
    if all_metrics:
        agg_metrics = {}
        for key in ["resonance_index", "stability", "sigma_overlay", "alpha_mean", "R_spectral", "phase_lag_deg"]:
            values = [m.get(key) for m in all_metrics if m.get(key) is not None]
            if values:
                agg_metrics[f"mean_{key}"] = float(np.mean(values))
                agg_metrics[f"std_{key}"] = float(np.std(values))
        agg_trace["metrics"] = agg_metrics

    return outputs, agg_trace


if __name__ == "__main__":
    # Quick test
    decoder = CDCDecoder()

    prompt = "Explain CDC in 3 sentences"
    text, trace = decoder.generate_from_anchor(prompt, max_new_tokens=32)

    print("=" * 60)
    print("CDC DECODE TEST")
    print("=" * 60)
    print(f"\nPrompt: {prompt}")
    print(f"\nGenerated: {text[:200]}")
    print(f"\nTrace: {json.dumps(trace, indent=2)}")
