"""
BrainCache — Read/write operations for the .sigmanticaicache/ directory.

This module provides:
1. BrainCache class for structured access to cache files
2. populate_cache_from_codebase() — scan HDL files and extract ports/symbols
3. write_cache_from_pipeline_state() — persist pipeline state to cache
4. persist_compression_to_cache() — save compressed context + decisions to disk
5. KnowledgeStore — agent-readable/writable knowledge store (lessons + session)

The cache is designed as a QUICK INDEX — a table of contents for the project.
The agent should ALWAYS still explore files with read_file/grep when it needs
exact details. The cache helps the agent know WHAT exists and WHERE, but the
source of truth is always the actual source files.
"""

from __future__ import annotations

import hashlib
import json
import os
import re
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple


# ============================================================================
# Constants
# ============================================================================

CACHE_DIR_NAME = ".sigmanticaicache"

CACHE_VERSION = "2.0"

# HDL file extensions to scan (covers Verilog, SystemVerilog, VHDL, headers)
HDL_EXTENSIONS = {"*.sv", "*.v", "*.svh", "*.vh", "*.vhd", "*.vhdl"}

# Files within the cache directory
META_FILE = "meta.json"
PORTS_FILE = "ports.json"
FILE_MAP_FILE = "file_map.json"          # 100% reliable file inventory
ANALYSIS_FILE = "analysis.json"          # Static analysis findings (design cards)
ARCHITECTURE_FILE = "architecture.json"
DECISIONS_FILE = "decisions.json"
SYMBOLS_FILE = "symbols.json"
COVERAGE_STATE_FILE = "coverage_state.json"
CONTEXT_SUMMARY_FILE = "context_summary.md"
LESSONS_LEARNED_FILE = "lessons_learned.jsonl"
SESSION_KNOWLEDGE_FILE = "session_knowledge.jsonl"


# ============================================================================
# BrainCache — Core read/write interface
# ============================================================================

def _get_session_id() -> Optional[str]:
    """Get session ID from environment (set on cloud workers)."""
    return os.environ.get("SESSION_ID")


class BrainCache:
    """
    Structured interface to the .sigmanticaicache/ directory.

    This class handles reading and writing individual cache files.
    The agent accesses these files through its normal tools (read_file, etc.),
    but pipeline code and the compression hook use this class directly.

    Session isolation:
        When session_id is provided (or auto-detected from SESSION_ID env var),
        session-specific files (architecture, coverage_state, context_summary,
        decisions) are stored in a subdirectory: .sigmanticaicache/sessions/<id>/
        Shared files (file_map, ports, symbols, analysis, meta) stay in the root
        since they are derived from source files and are safe to share.
    """

    # Files that are session-specific (derived from LLM pipeline state)
    _SESSION_FILES = {
        ARCHITECTURE_FILE, COVERAGE_STATE_FILE,
        CONTEXT_SUMMARY_FILE, DECISIONS_FILE,
    }

    def __init__(self, project_dir: Optional[Path] = None, session_id: Optional[str] = None):
        """
        Initialize BrainCache.

        Args:
            project_dir: Root directory of the project. Defaults to cwd.
            session_id: Optional session ID for isolation. If not provided,
                        auto-detects from SESSION_ID env var. When set,
                        session-specific files go in sessions/<id>/ subdir.
        """
        self.project_dir = Path(project_dir) if project_dir else Path.cwd()
        self.cache_dir = self.project_dir / CACHE_DIR_NAME
        self.session_id = session_id or _get_session_id()
        if self.session_id:
            self._session_dir = self.cache_dir / "sessions" / self.session_id
        else:
            self._session_dir = None

    @property
    def exists(self) -> bool:
        """Check if the cache directory exists."""
        return self.cache_dir.is_dir()

    def ensure_dir(self) -> Path:
        """Create the cache directory if it doesn't exist."""
        self.cache_dir.mkdir(parents=True, exist_ok=True)
        if self._session_dir:
            self._session_dir.mkdir(parents=True, exist_ok=True)
        return self.cache_dir

    def _resolve_path(self, filename: str) -> Path:
        """Resolve file path, routing session-specific files to the session subdir."""
        if self._session_dir and filename in self._SESSION_FILES:
            return self._session_dir / filename
        return self.cache_dir / filename

    # ── Individual file read/write ──────────────────────────────────────

    def read_json(self, filename: str) -> Optional[Dict[str, Any]]:
        """Read a JSON cache file. Returns None if missing or corrupt."""
        filepath = self._resolve_path(filename)
        if not filepath.exists():
            # Fallback: check root dir for session files (backwards compat)
            if self._session_dir and filename in self._SESSION_FILES:
                filepath = self.cache_dir / filename
                if not filepath.exists():
                    return None
            else:
                return None
        try:
            return json.loads(filepath.read_text())
        except (json.JSONDecodeError, OSError):
            return None

    def write_json(self, filename: str, data: Dict[str, Any]) -> None:
        """Write a JSON cache file."""
        self.ensure_dir()
        filepath = self._resolve_path(filename)
        filepath.write_text(json.dumps(data, indent=2) + "\n")

    def read_text(self, filename: str) -> Optional[str]:
        """Read a text cache file. Returns None if missing."""
        filepath = self._resolve_path(filename)
        if not filepath.exists():
            if self._session_dir and filename in self._SESSION_FILES:
                filepath = self.cache_dir / filename
                if not filepath.exists():
                    return None
            else:
                return None
        try:
            return filepath.read_text()
        except OSError:
            return None

    def write_text(self, filename: str, content: str) -> None:
        """Write a text cache file."""
        self.ensure_dir()
        filepath = self._resolve_path(filename)
        filepath.write_text(content)

    # ── Typed accessors ─────────────────────────────────────────────────

    def read_meta(self) -> Optional[Dict[str, Any]]:
        return self.read_json(META_FILE)

    def read_ports(self) -> Optional[Dict[str, Any]]:
        return self.read_json(PORTS_FILE)

    def read_architecture(self) -> Optional[Dict[str, Any]]:
        return self.read_json(ARCHITECTURE_FILE)

    def read_decisions(self) -> Optional[List[Dict[str, Any]]]:
        return self.read_json(DECISIONS_FILE)

    def read_symbols(self) -> Optional[Dict[str, Any]]:
        return self.read_json(SYMBOLS_FILE)

    def read_file_map(self) -> Optional[Dict[str, Any]]:
        return self.read_json(FILE_MAP_FILE)

    def read_analysis(self) -> Optional[Dict[str, Any]]:
        return self.read_json(ANALYSIS_FILE)

    def read_coverage_state(self) -> Optional[Dict[str, Any]]:
        return self.read_json(COVERAGE_STATE_FILE)

    def read_context_summary(self) -> Optional[str]:
        return self.read_text(CONTEXT_SUMMARY_FILE)

    # ── Staleness detection ─────────────────────────────────────────────

    def is_stale(self) -> bool:
        """
        Check if the cache is stale (source files changed since last update).

        Compares the hash of all HDL files against the stored hash in meta.json.
        """
        meta = self.read_meta()
        if not meta:
            return True  # No meta = needs population

        stored_hash = meta.get("source_files_hash", "")
        current_hash = _compute_hdl_hash(self.project_dir)
        return stored_hash != current_hash

    def get_cache_summary(self) -> str:
        """
        Return a one-line summary of cache state for display.

        Used by the agent's welcome message to show if a cached project exists.
        """
        if not self.exists:
            return "No project cache found"

        meta = self.read_meta()
        if not meta:
            return "Cache exists but meta.json is missing"

        design = "unknown"
        arch = self.read_architecture()
        if arch:
            design = arch.get("design_name", "unknown")

        updated = meta.get("last_updated", "unknown")
        hdl_count = len(meta.get("hdl_files_scanned", meta.get("sv_files_scanned", [])))
        stale = " (stale)" if self.is_stale() else ""

        return (
            f"Project: {design} | "
            f"{hdl_count} source files | "
            f"Updated: {updated}{stale}"
        )


# ============================================================================
# KnowledgeStore — Agent-readable/writable knowledge for brain_read/brain_write
# ============================================================================

class KnowledgeStore:
    """
    Two-tier knowledge store that agents can read from and write to at runtime.

    Long-term (lessons_learned.jsonl):
        Persists across runs. Contains verified findings that should inform
        all future verification sessions. Users can also seed this manually.
        Agents write here when severity="critical".

    Short-term (session_knowledge.jsonl):
        Per-session scratchpad. Cleared at the start of each new session.
        Agents write findings, observations, and hypotheses during execution.
        Other agents can read these to coordinate without hardcoded prompts.

    Each entry is a JSON line:
        {
            "timestamp": "2026-03-16T01:23:45",
            "category": "dead_signal",
            "finding": "slave_tx_shift in tb_top.sv is declared but never assigned",
            "source": "coverage_manager",
            "severity": "high"
        }
    """

    def __init__(self, project_dir: Optional[Path] = None, session_id: Optional[str] = None):
        self.project_dir = Path(project_dir) if project_dir else Path.cwd()
        self.cache_dir = self.project_dir / CACHE_DIR_NAME
        self.session_id = session_id or _get_session_id()
        if self.session_id:
            self._session_dir = self.cache_dir / "sessions" / self.session_id
        else:
            self._session_dir = self.cache_dir

    @property
    def lessons_path(self) -> Path:
        return self.cache_dir / LESSONS_LEARNED_FILE

    @property
    def session_path(self) -> Path:
        if self._session_dir:
            return self._session_dir / SESSION_KNOWLEDGE_FILE
        return self.cache_dir / SESSION_KNOWLEDGE_FILE

    def _ensure_dirs(self):
        self.cache_dir.mkdir(parents=True, exist_ok=True)
        if self._session_dir and self._session_dir != self.cache_dir:
            self._session_dir.mkdir(parents=True, exist_ok=True)

    def write(self, category: str, finding: str, source: str = "agent",
              severity: str = "medium") -> str:
        """
        Append a knowledge entry. Returns a confirmation string.

        If severity is "critical", the entry goes to BOTH lessons_learned
        (long-term) and session_knowledge (short-term). Otherwise only
        session_knowledge.
        """
        self._ensure_dirs()
        entry = {
            "timestamp": datetime.now().isoformat(),
            "category": category,
            "finding": finding,
            "source": source,
            "severity": severity,
        }
        line = json.dumps(entry) + "\n"

        with open(self.session_path, "a") as f:
            f.write(line)

        if severity == "critical":
            with open(self.lessons_path, "a") as f:
                f.write(line)
            return f"Written to long-term AND session knowledge: [{category}] {finding[:80]}"

        return f"Written to session knowledge: [{category}] {finding[:80]}"

    def read(self, query: str = "", category: str = "", limit: int = 30) -> str:
        """
        Read knowledge entries matching a query or category.

        Searches both long-term lessons and session knowledge.
        Returns formatted text suitable for LLM consumption.
        """
        results = []

        for label, path in [("LESSON", self.lessons_path),
                            ("SESSION", self.session_path)]:
            if not path.exists():
                continue
            try:
                for line in path.read_text().strip().split("\n"):
                    if not line.strip():
                        continue
                    try:
                        entry = json.loads(line)
                    except json.JSONDecodeError:
                        continue

                    if category and entry.get("category", "") != category:
                        continue

                    if query:
                        text = f"{entry.get('category', '')} {entry.get('finding', '')}"
                        if query.lower() not in text.lower():
                            continue

                    results.append((label, entry))
            except OSError:
                continue

        if not results:
            if query or category:
                return f"No knowledge entries found for query='{query}' category='{category}'"
            return "Knowledge store is empty."

        results = results[-limit:]

        lines = []
        for label, entry in results:
            sev = entry.get("severity", "medium")
            cat = entry.get("category", "general")
            src = entry.get("source", "unknown")
            finding = entry.get("finding", "")
            lines.append(f"[{label}|{sev}|{cat}|{src}] {finding}")

        return "\n".join(lines)

    def clear_session(self):
        """Clear session knowledge (called at the start of a new run)."""
        if self.session_path.exists():
            self.session_path.unlink()

    def get_all_lessons(self) -> List[Dict[str, Any]]:
        """Return all long-term lessons as a list of dicts."""
        if not self.lessons_path.exists():
            return []
        entries = []
        for line in self.lessons_path.read_text().strip().split("\n"):
            if not line.strip():
                continue
            try:
                entries.append(json.loads(line))
            except json.JSONDecodeError:
                continue
        return entries


# ============================================================================
# Population: Auto-parse existing codebase
# ============================================================================

def populate_cache_from_codebase(
    project_dir: Optional[Path] = None,
    force: bool = False,
    session_id: Optional[str] = None,
) -> BrainCache:
    """
    Scan ALL HDL files (.sv, .v, .svh, .vh, .vhd, .vhdl) in the project
    and populate the cache.

    Produces:
    - file_map.json:  100% reliable inventory of every HDL file (path, size,
                      type, line count). This can NEVER miss anything.
    - ports.json:     Best-effort regex extraction of module ports/params.
    - symbols.json:   Best-effort regex extraction of classes/packages/instances.
    - meta.json:      File hashes for staleness detection.

    Args:
        project_dir: Root directory to scan. Defaults to cwd.
        force: If True, repopulate even if cache exists and is fresh.

    Returns:
        BrainCache instance pointing to the populated cache.
    """
    cache = BrainCache(project_dir, session_id=session_id)

    # Skip if cache is fresh (unless forced)
    if cache.exists and not cache.is_stale() and not force:
        return cache

    project_dir = cache.project_dir

    # ── Discover ALL HDL files (100% reliable) ──────────────────────
    hdl_files = _find_hdl_files(project_dir)

    if not hdl_files:
        # Still write an empty file_map so the LLM knows the scan ran
        cache.write_json(FILE_MAP_FILE, {
            "total_files": 0,
            "files": [],
            "by_directory": {},
            "by_extension": {},
        })
        cache.write_json(META_FILE, {
            "cache_version": CACHE_VERSION,
            "created": datetime.now().isoformat(),
            "last_updated": datetime.now().isoformat(),
            "source_files_hash": _compute_hdl_hash(project_dir),
            "hdl_files_scanned": [],
            "agent_version": _get_agent_version(),
            "parser_coverage": "No HDL files found",
        })
        return cache

    # ── Build file_map.json (ALWAYS accurate) ───────────────────────
    file_map_entries = []
    by_directory: Dict[str, List[str]] = {}
    by_extension: Dict[str, int] = {}

    for hdl_file in hdl_files:
        rel_path = str(hdl_file.relative_to(project_dir))
        try:
            size = hdl_file.stat().st_size
            line_count = hdl_file.read_text(errors="replace").count("\n") + 1
        except OSError:
            size = 0
            line_count = 0

        ext = hdl_file.suffix.lower()
        dir_name = str(hdl_file.parent.relative_to(project_dir))
        if dir_name == ".":
            dir_name = "/"

        file_map_entries.append({
            "path": rel_path,
            "size_bytes": size,
            "lines": line_count,
            "extension": ext,
            "directory": dir_name,
        })

        by_directory.setdefault(dir_name, []).append(rel_path)
        by_extension[ext] = by_extension.get(ext, 0) + 1

    cache.write_json(FILE_MAP_FILE, {
        "total_files": len(file_map_entries),
        "files": file_map_entries,
        "by_directory": {d: sorted(files) for d, files in sorted(by_directory.items())},
        "by_extension": by_extension,
    })

    # ── Best-effort regex extraction (ports, symbols) ───────────────
    modules: Dict[str, Dict[str, Any]] = {}
    interfaces: Dict[str, Dict[str, Any]] = {}
    classes: Dict[str, Optional[str]] = {}
    packages: List[str] = []
    instances: Dict[str, str] = {}
    rtl_instances: Dict[str, str] = {}  # instance_name → module_type
    parse_warnings: List[str] = []

    for hdl_file in hdl_files:
        ext = hdl_file.suffix.lower()
        # Skip VHDL for regex extraction (our regex is SV/Verilog-focused)
        if ext in (".vhd", ".vhdl"):
            continue

        try:
            content = hdl_file.read_text(errors="replace")
        except OSError:
            continue

        rel_path = str(hdl_file.relative_to(project_dir))

        # Strip single-line and multi-line comments to avoid false matches
        content_stripped = _strip_comments(content)

        # Extract modules with ports and parameters
        _extract_modules(content_stripped, rel_path, modules)

        # Extract interfaces with signals and modports
        _extract_interfaces(content_stripped, rel_path, interfaces)

        # Extract classes (UVM components)
        _extract_classes(content_stripped, classes)

        # Extract packages
        _extract_packages(content_stripped, packages)

        # Extract UVM factory instances
        _extract_instances(content_stripped, instances)

        # Extract RTL instantiations (module_type instance_name)
        _extract_rtl_instances(content_stripped, rtl_instances)

    # ── Count what we found vs what we might have missed ────────────
    total_sv_verilog = sum(1 for f in hdl_files if f.suffix.lower() in (".sv", ".v", ".svh", ".vh"))
    parsed_modules = len(modules)

    if total_sv_verilog > 0 and parsed_modules == 0:
        parse_warnings.append(
            f"WARNING: Found {total_sv_verilog} SV/Verilog files but extracted 0 modules. "
            f"The codebase may use non-standard formatting. Use read_file to inspect files directly."
        )

    # ── Write cache files ───────────────────────────────────────────

    # ports.json — best-effort, may be incomplete
    cache.write_json(PORTS_FILE, {
        "modules": modules,
        "interfaces": interfaces,
        "_parser_note": (
            f"Regex-extracted from {total_sv_verilog} SV/Verilog files. "
            f"Found {len(modules)} modules, {len(interfaces)} interfaces. "
            f"This is BEST-EFFORT — always verify with read_file for critical details."
        ),
    })

    # symbols.json — best-effort, may be incomplete
    cache.write_json(SYMBOLS_FILE, {
        "classes": classes,
        "packages": packages,
        "instances": instances,
        "rtl_instances": rtl_instances,
        "_parser_note": (
            f"Found {len(classes)} classes, {len(packages)} packages, "
            f"{len(instances)} UVM instances, {len(rtl_instances)} RTL instantiations. "
            f"Best-effort extraction — use grep_search for definitive lookups."
        ),
    })

    # ── analysis.json — static analysis (design cards) ──────────────
    try:
        from sigmanticai.brain.analysis import analyze_codebase
        analysis_report = analyze_codebase(hdl_files, project_dir)
        cache.write_json(ANALYSIS_FILE, analysis_report)
        analysis_summary = analysis_report.get("summary", {})
    except Exception:
        analysis_summary = {"error": "analysis pass failed"}
        cache.write_json(ANALYSIS_FILE, {
            "summary": analysis_summary,
            "modules": {},
            "findings": [],
            "_note": "Analysis pass failed — explore codebase manually.",
        })

    # meta.json
    cache.write_json(META_FILE, {
        "cache_version": CACHE_VERSION,
        "created": datetime.now().isoformat(),
        "last_updated": datetime.now().isoformat(),
        "source_files_hash": _compute_hdl_hash(project_dir),
        "hdl_files_scanned": [str(f.relative_to(project_dir)) for f in hdl_files],
        "agent_version": _get_agent_version(),
        "parser_coverage": (
            f"{len(modules)} modules, {len(interfaces)} interfaces, "
            f"{len(classes)} classes from {total_sv_verilog} SV/Verilog files. "
            f"File map covers all {len(hdl_files)} HDL files (100% accurate)."
        ),
        "analysis_coverage": (
            f"{analysis_summary.get('total_findings', 0)} findings, "
            f"{analysis_summary.get('high_confidence_count', 0)} high-confidence, "
            f"across {analysis_summary.get('modules_analyzed', 0)} modules"
        ),
        "parse_warnings": parse_warnings,
    })

    return cache


# ============================================================================
# Population: From pipeline state (after generation)
# ============================================================================

def write_cache_from_pipeline_state(
    state: Dict[str, Any],
    output_dir: Optional[Path] = None,
    session_id: Optional[str] = None,
) -> BrainCache:
    """
    Write cache files from a pipeline's VerificationState.

    Called after the v2 LangGraph pipeline completes (or after key stages).
    Extracts design_spec, intents, gaps, symbols, and coverage from the
    pipeline state and writes them to the cache.

    Args:
        state: VerificationState dict from the pipeline.
        output_dir: Output directory. Defaults to state["output_dir"] or cwd.

    Returns:
        BrainCache instance.
    """
    import os

    if output_dir is None:
        output_dir = Path(state.get("output_dir", os.getcwd()))
    cache = BrainCache(output_dir, session_id=session_id)

    design_name = state.get("design_name", "design")
    design_spec = state.get("design_spec", {})
    if not isinstance(design_spec, dict):
        if isinstance(design_spec, str):
            try:
                import json as _json
                design_spec = _json.loads(design_spec)
            except Exception:
                design_spec = {}
        else:
            design_spec = {}
        if not isinstance(design_spec, dict):
            design_spec = {}

    # ── architecture.json ───────────────────────────────────────────

    # Extract module hierarchy from design_spec
    module_hierarchy = {}
    modules = design_spec.get("modules", [])
    if modules:
        # The first module is typically the top; sub-modules are the rest
        top_name = modules[0].get("name", design_name) if modules else design_name
        sub_names = [m.get("name", "") for m in modules[1:] if m.get("name")]
        if sub_names:
            module_hierarchy[top_name] = sub_names

    # Extract FSMs
    fsms = {}
    for mod in modules:
        for fsm in mod.get("fsms", []):
            fsm_name = fsm.get("name", f"fsm_{len(fsms)}")
            fsms[fsm_name] = {
                "module": mod.get("name", "unknown"),
                "states": fsm.get("states", []),
                "transitions": fsm.get("transitions", []),
                "description": fsm.get("description", ""),
            }

    # Extract interface types
    interfaces_used = [
        iface.get("type", iface.get("name", ""))
        for iface in design_spec.get("interfaces", [])
    ]

    # Extract requirements summary
    requirements_summary = [
        f"{req.get('id', 'REQ')}: {req.get('description', '')}"
        for req in design_spec.get("requirements", [])
    ]

    cache.write_json(ARCHITECTURE_FILE, {
        "design_name": design_name,
        "description": design_spec.get("description", ""),
        "module_hierarchy": module_hierarchy,
        "modules": [
            {
                "name": m.get("name", ""),
                "description": m.get("description", ""),
                "port_count": len(m.get("ports", [])),
            }
            for m in modules
        ],
        "fsms": fsms,
        "interfaces_used": interfaces_used,
        "requirements_summary": requirements_summary,
        "corner_cases": [
            cc.get("description", "") for cc in design_spec.get("corner_cases", [])
        ],
    })

    # ── ports.json ──────────────────────────────────────────────────

    port_modules = {}
    for mod in modules:
        mod_name = mod.get("name", "")
        if not mod_name:
            continue
        ports = {}
        for port in mod.get("ports", []):
            port_name = port.get("name", "")
            if port_name:
                ports[port_name] = {
                    "direction": port.get("direction", "input"),
                    "width": port.get("width", 1),
                    "description": port.get("description", ""),
                }
        params = {}
        for param in mod.get("parameters", []):
            pname = param.get("name", "")
            if pname:
                params[pname] = param.get("default", "")
        port_modules[mod_name] = {
            "ports": ports,
            "params": params,
        }

    port_interfaces = {}
    for iface in design_spec.get("interfaces", []):
        iname = iface.get("name", "")
        if iname:
            port_interfaces[iname] = {
                "type": iface.get("type", "custom"),
                "signals": iface.get("signals", []),
            }

    cache.write_json(PORTS_FILE, {
        "modules": port_modules,
        "interfaces": port_interfaces,
    })

    # ── coverage_state.json ─────────────────────────────────────────

    cache.write_json(COVERAGE_STATE_FILE, {
        "coverage_score": state.get("coverage_score", 0.0),
        "gaps": state.get("gaps", []),
        "intents": state.get("intents", []),
        "intent_graph": state.get("intent_graph", {}),
        "iteration": state.get("iteration", 0),
        "max_iterations": state.get("max_iterations", 3),
    })

    # ── symbols.json ────────────────────────────────────────────────

    symbol_table = state.get("symbol_table", {})
    naming = state.get("naming_conventions", {})
    if symbol_table or naming:
        cache.write_json(SYMBOLS_FILE, {
            "classes": symbol_table.get("classes", {}),
            "packages": symbol_table.get("packages", []),
            "modules": symbol_table.get("modules", []),
            "interfaces": symbol_table.get("interfaces", []),
            "instances": symbol_table.get("instances", {}),
            "naming_conventions": naming,
        })

    # ── file_map.json ────────────────────────────────────────────────
    # Also build a reliable file map from the output directory
    hdl_files = _find_hdl_files(output_dir)
    if hdl_files:
        file_map_entries = []
        by_directory: Dict[str, List[str]] = {}
        by_extension: Dict[str, int] = {}
        for hdl_file in hdl_files:
            rel_path = str(hdl_file.relative_to(output_dir))
            try:
                size = hdl_file.stat().st_size
                line_count = hdl_file.read_text(errors="replace").count("\n") + 1
            except OSError:
                size = 0
                line_count = 0
            ext = hdl_file.suffix.lower()
            dir_name = str(hdl_file.parent.relative_to(output_dir))
            if dir_name == ".":
                dir_name = "/"
            file_map_entries.append({
                "path": rel_path,
                "size_bytes": size,
                "lines": line_count,
                "extension": ext,
                "directory": dir_name,
            })
            by_directory.setdefault(dir_name, []).append(rel_path)
            by_extension[ext] = by_extension.get(ext, 0) + 1

        cache.write_json(FILE_MAP_FILE, {
            "total_files": len(file_map_entries),
            "files": file_map_entries,
            "by_directory": {d: sorted(files) for d, files in sorted(by_directory.items())},
            "by_extension": by_extension,
        })

    # ── meta.json ───────────────────────────────────────────────────

    existing_meta = cache.read_meta() or {}
    cache.write_json(META_FILE, {
        "cache_version": CACHE_VERSION,
        "created": existing_meta.get("created", datetime.now().isoformat()),
        "last_updated": datetime.now().isoformat(),
        "source_files_hash": _compute_hdl_hash(output_dir),
        "hdl_files_scanned": [str(f.relative_to(output_dir)) for f in hdl_files] if hdl_files else existing_meta.get("hdl_files_scanned", []),
        "agent_version": _get_agent_version(),
        "pipeline_stage": state.get("stage", 0),
        "design_name": design_name,
    })

    return cache


# ============================================================================
# Population: From context compression
# ============================================================================

def persist_compression_to_cache(
    summary: str,
    key_state: Dict[str, Any],
    project_dir: Optional[Path] = None,
    session_id: Optional[str] = None,
) -> None:
    """
    Persist context compression results to the cache.

    Called from SigmanticAgent._compress_history() to externalize
    the compressed summary and key state to disk so they survive
    across sessions.

    Args:
        summary: The LLM-generated conversation summary.
        key_state: Extracted key state (output dirs, design names, etc.).
        project_dir: Project directory. Defaults to cwd.
    """
    cache = BrainCache(project_dir, session_id=session_id)
    cache.ensure_dir()

    # ── context_summary.md ──────────────────────────────────────────

    cache.write_text(CONTEXT_SUMMARY_FILE, summary)

    # ── Append to decisions.json ────────────────────────────────────

    # Extract decision-like entries from key_state
    decisions = cache.read_decisions() or []

    new_decision = {
        "timestamp": datetime.now().isoformat(),
        "type": "context_compression",
        "key_state": key_state,
        "summary_length": len(summary),
    }
    decisions.append(new_decision)

    cache.write_json(DECISIONS_FILE, decisions)

    # ── Update meta.json timestamp ──────────────────────────────────

    meta = cache.read_meta() or {
        "cache_version": CACHE_VERSION,
        "created": datetime.now().isoformat(),
    }
    meta["last_updated"] = datetime.now().isoformat()
    cache.write_json(META_FILE, meta)


# ============================================================================
# Helpers: File discovery
# ============================================================================

def _find_hdl_files(project_dir: Path) -> List[Path]:
    """
    Find ALL HDL files in the project directory.

    Scans for .sv, .v, .svh, .vh, .vhd, .vhdl — this is the 100% reliable
    part of the brain. Even if regex extraction misses details, the file map
    will always list every single file.
    """
    all_files: List[Path] = []
    for ext_glob in HDL_EXTENSIONS:
        all_files.extend(project_dir.rglob(ext_glob))

    # Exclude files inside the cache directory
    all_files = [f for f in all_files if CACHE_DIR_NAME not in f.parts]
    # Deduplicate and sort
    all_files = sorted(set(all_files))
    return all_files


def _strip_comments(content: str) -> str:
    """
    Remove single-line (//) and multi-line (/* */) comments from SV/Verilog.

    This prevents false regex matches on commented-out code.
    """
    # Remove multi-line comments first
    content = re.sub(r'/\*.*?\*/', '', content, flags=re.DOTALL)
    # Remove single-line comments
    content = re.sub(r'//[^\n]*', '', content)
    return content


def _find_balanced_parens(text: str, start: int) -> Tuple[int, str]:
    """
    Find the matching closing paren for an opening paren at position `start`.

    Handles nested parens like: parameter int W = $clog2(DEPTH)

    Returns (end_position, content_between_parens).
    Returns (-1, "") if no match found.
    """
    if start >= len(text) or text[start] != '(':
        return (-1, "")

    depth = 1
    pos = start + 1
    while pos < len(text) and depth > 0:
        if text[pos] == '(':
            depth += 1
        elif text[pos] == ')':
            depth -= 1
        pos += 1

    if depth == 0:
        return (pos - 1, text[start + 1: pos - 1])
    return (-1, "")


# ============================================================================
# Helpers: SV/Verilog extraction (regex-based, no LLM)
# ============================================================================

def _extract_modules(
    content: str,
    file_path: str,
    modules: Dict[str, Dict[str, Any]],
) -> None:
    """
    Extract module definitions with ports and parameters.

    Handles:
    - module name #(params) (ports);           (standard ANSI)
    - module name (ports);                      (no params)
    - module name;                              (no ports — portless modules)
    - module name import pkg::*; #(...) (...);  (import between name and ports)
    - Nested parens in parameters: $clog2(N)
    - Custom types: input my_pkg::my_type_t data
    - signed keyword: input logic signed [7:0] data
    """
    # Strategy: find "module <name>" then use balanced-paren matching
    # This handles nested parens that break simple regex

    # Phase 1: Find all "module <name>" declarations
    module_start_pattern = re.compile(
        r'\bmodule\s+(\w+)',
        re.MULTILINE,
    )

    for m_start in module_start_pattern.finditer(content):
        module_name = m_start.group(1)
        rest = content[m_start.end():]

        # Skip past optional import statements
        rest = re.sub(r'^\s*import\s+[^;]+;\s*', '', rest)

        params_str = ""
        ports_str = ""

        # Check for #(params)
        rest_stripped = rest.lstrip()
        if rest_stripped.startswith('#'):
            # Find the opening paren after #
            hash_paren = rest_stripped.find('(', 1)
            if hash_paren >= 0:
                abs_pos = rest_stripped.index('(', 1)
                end_pos, params_str = _find_balanced_parens(rest_stripped, abs_pos)
                if end_pos >= 0:
                    rest_stripped = rest_stripped[end_pos + 1:].lstrip()

        # Check for (ports)
        if rest_stripped.startswith('('):
            _, ports_str = _find_balanced_parens(rest_stripped, 0)

        # Extract port details with expanded patterns
        ports = {}
        # Pattern covers: input/output/inout [wire|reg|logic] [signed] [type] [width] name
        port_pattern = re.compile(
            r'(input|output|inout)\s+'
            r'(?:(wire|reg|logic|var)\s+)?'     # optional net/var type
            r'(?:signed\s+)?'                    # optional signed
            r'(?:(\w+::)?\w+\s+)?'              # optional package::type
            r'(?:(\[.*?\])\s+)?'                 # optional [width]
            r'(\w+)',                            # port name
        )
        for pm in port_pattern.finditer(ports_str):
            direction = pm.group(1)
            width_str = pm.group(4) or ""
            port_name = pm.group(5)
            # Try to parse width from [N:0] or [N-1:0] format
            width = 1
            width_match = re.search(r'\[(\d+):(\d+)\]', width_str)
            if width_match:
                width = abs(int(width_match.group(1)) - int(width_match.group(2))) + 1
            ports[port_name] = {
                "direction": direction,
                "width": width,
            }

        # If port_pattern found nothing, try a simpler fallback for non-ANSI
        # or unusual formatting — just get the names
        if not ports and ports_str.strip():
            # Fallback: just grab identifiers that follow direction keywords
            simple_pattern = re.compile(r'(input|output|inout)\s+.*?(\w+)\s*(?:,|$)', re.DOTALL)
            for pm in simple_pattern.finditer(ports_str):
                port_name = pm.group(2)
                ports[port_name] = {
                    "direction": pm.group(1),
                    "width": 1,
                }

        # Extract parameters with balanced-paren-aware matching
        params = {}
        param_pattern = re.compile(
            r'parameter\s+(?:(?:int|integer|logic|bit|real|string|type)\s+)?'
            r'(?:\[.*?\]\s+)?'
            r'(\w+)\s*=\s*([^,\)]*(?:\([^)]*\)[^,\)]*)*)',
        )
        for pm in param_pattern.finditer(params_str):
            params[pm.group(1)] = pm.group(2).strip()

        # Also catch localparam in the params block
        lparam_pattern = re.compile(
            r'localparam\s+(?:\w+\s+)?(\w+)\s*=\s*([^,\)]*(?:\([^)]*\)[^,\)]*)*)',
        )
        for pm in lparam_pattern.finditer(params_str):
            params[pm.group(1)] = pm.group(2).strip()

        modules[module_name] = {
            "file": file_path,
            "ports": ports,
            "params": params,
        }


def _extract_interfaces(
    content: str,
    file_path: str,
    interfaces: Dict[str, Dict[str, Any]],
) -> None:
    """
    Extract interface definitions with signals and modports.

    Handles:
    - logic, wire, reg, bit signals inside interfaces
    - modport declarations (master, slave directionality)
    """
    pattern = re.compile(
        r'\binterface\s+(\w+)\s*(?:\(.*?\))?\s*;(.*?)endinterface',
        re.DOTALL,
    )
    for match in pattern.finditer(content):
        iface_name = match.group(1)
        body = match.group(2)

        # Extract signals of all types (logic, wire, reg, bit)
        signals = re.findall(
            r'(?:logic|wire|reg|bit)\s+(?:\[.*?\]\s+)?(\w+)\s*;',
            body,
        )

        # Extract modport declarations: modport master (input clk, output data, ...)
        modports = {}
        modport_pattern = re.compile(
            r'modport\s+(\w+)\s*\((.*?)\)\s*;',
            re.DOTALL,
        )
        for mp_match in modport_pattern.finditer(body):
            mp_name = mp_match.group(1)
            mp_body = mp_match.group(2)
            mp_signals = re.findall(r'(input|output|inout)\s+(\w+)', mp_body)
            modports[mp_name] = [
                {"direction": d, "name": n} for d, n in mp_signals
            ]

        interfaces[iface_name] = {
            "file": file_path,
            "signals": signals,
            "modports": modports,
        }


def _extract_classes(content: str, classes: Dict[str, Optional[str]]) -> None:
    """
    Extract class definitions (name → parent_class).

    Handles:
    - class name extends parent;
    - class name extends parent #(type);
    - class name;
    - virtual class name extends parent;
    """
    # class <name> extends <parent> [#(...)];
    pattern = re.compile(
        r'(?:virtual\s+)?class\s+(\w+)\s+extends\s+(\w+)',
        re.MULTILINE,
    )
    for match in pattern.finditer(content):
        classes[match.group(1)] = match.group(2)

    # class <name>; (no extends) — but skip if already found via extends
    pattern2 = re.compile(r'(?:virtual\s+)?class\s+(\w+)\s*(?:#\s*\(.*?\)\s*)?;', re.MULTILINE)
    for match in pattern2.finditer(content):
        name = match.group(1)
        if name not in classes:
            classes[name] = None


def _extract_packages(content: str, packages: List[str]) -> None:
    """Extract package names."""
    pattern = re.compile(r'\bpackage\s+(\w+)\s*;', re.MULTILINE)
    for match in pattern.finditer(content):
        pkg = match.group(1)
        if pkg not in packages:
            packages.append(pkg)


def _extract_instances(content: str, instances: Dict[str, str]) -> None:
    """Extract UVM factory create instances: var = Type::type_id::create(...)"""
    pattern = re.compile(r'(\w+)\s*=\s*(\w+)::type_id::create\s*\(', re.MULTILINE)
    for match in pattern.finditer(content):
        instances[match.group(1)] = match.group(2)


def _extract_rtl_instances(content: str, rtl_instances: Dict[str, str]) -> None:
    """
    Extract RTL module instantiations.

    Handles:
    - module_type instance_name (...);
    - module_type #(...) instance_name (...);
    - module_type #(.PARAM(val)) instance_name (...);

    This captures the module hierarchy — which module instantiates which.
    """
    # Pattern: <module_type> [#(...)] <instance_name> (
    # We need to avoid matching keywords like "module", "if", "for", "always", etc.
    keywords = {
        "module", "endmodule", "interface", "endinterface", "class", "endclass",
        "function", "endfunction", "task", "endtask", "package", "endpackage",
        "if", "else", "for", "while", "do", "begin", "end", "case", "endcase",
        "always", "always_ff", "always_comb", "always_latch", "assign",
        "initial", "final", "generate", "endgenerate", "input", "output",
        "inout", "wire", "reg", "logic", "integer", "real", "string",
        "return", "import", "export", "typedef", "enum", "struct", "union",
    }

    # Match: identifier [whitespace] #(...) [whitespace] identifier [whitespace] (
    # OR:    identifier [whitespace] identifier [whitespace] (
    pattern = re.compile(
        r'\b(\w+)\s+'                  # module type
        r'(?:#\s*\(.*?\)\s+)?'         # optional #(params) — non-greedy
        r'(\w+)\s*\(',                 # instance name followed by (
        re.DOTALL,
    )

    for match in pattern.finditer(content):
        module_type = match.group(1)
        instance_name = match.group(2)

        # Skip if module_type is a keyword
        if module_type.lower() in keywords:
            continue
        # Skip if instance_name is a keyword
        if instance_name.lower() in keywords:
            continue
        # Skip obvious function calls (single-word followed by parens in expressions)
        # Heuristic: RTL instances usually have u_ or i_ prefixes or are PascalCase
        # But we keep everything and let the LLM decide
        rtl_instances[instance_name] = module_type


# ============================================================================
# Helpers: Hashing and version
# ============================================================================

def _compute_hdl_hash(project_dir: Path) -> str:
    """Compute a hash of all HDL files for staleness detection."""
    hasher = hashlib.sha256()
    hdl_files = _find_hdl_files(project_dir)

    for hdl_file in hdl_files:
        try:
            hasher.update(str(hdl_file.relative_to(project_dir)).encode())
            hasher.update(hdl_file.read_bytes())
        except OSError:
            continue

    return hasher.hexdigest()[:16]


def _get_agent_version() -> str:
    """Get the current agent version string."""
    try:
        from sigmanticai import __version__
        return __version__
    except ImportError:
        return "unknown"

