"""
ShelfAI Session Management Agent.

Pipeline:
    1. INGEST  — Parse transcript into structured messages
    2. EXTRACT — LLM call to pull learnings from the conversation
    3. DEDUP   — Compare extractions against existing shelf memories
    4. MERGE   — Apply updates to memory files and index.md (locked + backed up)
    5. REPORT  — Return a structured report of what changed

Note: Steps 1-3 read shelf state without locking. In concurrent scenarios,
the extraction may be based on slightly stale data. The dedup step and
memory limit check (both under lock in step 4) mitigate this.
"""

import json
import time
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Optional
import re

from shelfai.core.shelf import Shelf
from shelfai.core.transcript import TranscriptParser, Transcript
from shelfai.core.safety import (
    validate_path,
    shelf_lock,
    backup_files,
    generate_session_id,
    count_memory_entries,
    logger,
)
from shelfai.providers import LLMProvider
from shelfai.prompts import (
    EXTRACTION_SYSTEM_PROMPT,
    EXTRACTION_USER_PROMPT,
    DEDUP_SYSTEM_PROMPT,
    DEDUP_USER_PROMPT,
)

__all__ = ["SessionManager", "SessionReport", "Extraction"]


# ──────────────────────────────────────────────
# Data Classes
# ──────────────────────────────────────────────


@dataclass
class Extraction:
    """Structured extraction from a session."""

    user_preferences: list = field(default_factory=list)
    agent_lessons: list = field(default_factory=list)
    task_patterns: list = field(default_factory=list)
    knowledge_corrections: list = field(default_factory=list)
    new_context: list = field(default_factory=list)
    abstract_updates: list = field(default_factory=list)

    @property
    def total_items(self) -> int:
        return (
            len(self.user_preferences)
            + len(self.agent_lessons)
            + len(self.task_patterns)
            + len(self.knowledge_corrections)
            + len(self.new_context)
            + len(self.abstract_updates)
        )

    @property
    def is_empty(self) -> bool:
        return self.total_items == 0


@dataclass
class SessionReport:
    """Report of what the session manager did (or would do in dry-run mode)."""

    extraction: Extraction
    session_id: str = ""
    applied_changes: list = field(default_factory=list)
    skipped_changes: list = field(default_factory=list)
    errors: list = field(default_factory=list)
    dry_run: bool = False
    backup_dir: Optional[str] = None


# ──────────────────────────────────────────────
# Extraction Schema Validation
# ──────────────────────────────────────────────

EXTRACTION_SCHEMAS = {
    "user_preferences": {"content", "confidence"},
    "agent_lessons": {"content", "confidence"},
    "task_patterns": {"content", "confidence"},
    "knowledge_corrections": {"file", "issue", "correction"},
    "new_context": {"suggested_path", "suggested_abstract", "content_summary"},
    "abstract_updates": {"file", "suggested_abstract"},
}


EXTRACTION_TYPE_FLAGS = {
    "user_preferences": "extract_user_preferences",
    "agent_lessons": "extract_agent_lessons",
    "task_patterns": "extract_task_patterns",
    "knowledge_corrections": "extract_knowledge_corrections",
    "new_context": "extract_new_context",
    "abstract_updates": "extract_abstract_updates",
}


CONFIDENCE_GATED_CATEGORIES = {"user_preferences", "agent_lessons", "task_patterns"}


def _normalize_confidence(value: str) -> str:
    """Normalize confidence values for policy enforcement."""
    return str(value or "").strip().lower()


def _apply_extraction_policy(data: dict, config) -> dict:
    """Filter extraction categories and entries based on runtime config."""
    filtered = {key: list(value) for key, value in data.items()}

    for category, flag_name in EXTRACTION_TYPE_FLAGS.items():
        if not getattr(config.session, flag_name, True):
            filtered[category] = []

    if config.session.require_high_confidence:
        for category in CONFIDENCE_GATED_CATEGORIES:
            filtered[category] = [
                item
                for item in filtered.get(category, [])
                if _normalize_confidence(item.get("confidence")) == "high"
            ]

    return filtered


def _deterministic_deduplicate_items(items: list, existing_text: str) -> list:
    """Cheap exact-ish dedup pass before LLM dedup."""
    if not items:
        return []

    def normalize(value: str) -> str:
        value = str(value or "").lower()
        value = re.sub(r"\[confidence:\s*(high|medium|low)\]", "", value)
        value = re.sub(r"[^a-z0-9\s]", " ", value)
        value = re.sub(r"\s+", " ", value).strip()
        return value

    existing_norm = normalize(existing_text)
    seen = set()
    result = []
    for item in items:
        norm = normalize(item.get("content", ""))
        if not norm:
            continue
        if norm in seen:
            continue
        if norm and norm in existing_norm:
            continue
        seen.add(norm)
        result.append(item)
    return result


def _validate_new_context_path(path: str, allowed_categories: list[str]) -> None:
    """Enforce protocol-safe constraints for session-created files."""
    path_obj = Path(path)
    parts = path_obj.parts
    if len(parts) < 2:
        raise ValueError("New context file must include a category directory")
    if parts[0] not in set(allowed_categories):
        raise ValueError(f"Category '{parts[0]}' is not allowed for new context files")
    if path_obj.suffix.lower() != ".md":
        raise ValueError("New context files must use the .md extension")


def _sanitize_abstract(text: str, max_length: int) -> str:
    """Normalize and bound abstracts so downstream retrieval gets clean routing metadata."""
    value = str(text or "").strip().replace("\n", " ")
    value = re.sub(r"\s+", " ", value)
    if len(value) > max_length:
        value = value[: max_length - 3].rstrip() + "..."
    return value


def _lint_abstract(text: str, min_length: int, max_length: int) -> list[str]:
    """Return protocol-oriented abstract lint issues for routing quality."""
    issues = []
    value = _sanitize_abstract(text, max_length)
    lowered = value.lower()
    if not value:
        return ["abstract is empty"]
    if len(value) < min_length:
        issues.append(f"abstract is too short (< {min_length} chars)")
    generic_patterns = [
        r"^general notes?\b",
        r"^useful information\b",
        r"^reference material\b",
        r"^misc(?:ellaneous)?\b",
        r"^notes\b",
    ]
    if any(re.search(pattern, lowered) for pattern in generic_patterns):
        issues.append("abstract is too generic for reliable routing")
    word_count = len(re.findall(r"[a-z0-9][a-z0-9_-]*", lowered))
    if word_count < 4:
        issues.append("abstract needs more discriminative terms")
    if not re.search(r"\b(use|when|for|covers?|explains?|tracks?|contains?|reference|playbook|workflow)\b", lowered):
        issues.append("abstract should state when or how the doc is used")
    return issues


def _validate_extraction(data: dict, categories: Optional[set] = None) -> dict:
    """
    Validate and sanitize extracted data against the expected schema.

    Args:
        data: Raw parsed JSON from the LLM
        categories: If provided, only validate these categories.
                    If None, validates all categories in EXTRACTION_SCHEMAS.

    Returns:
        Sanitized dict with only valid entries
    """
    schemas = EXTRACTION_SCHEMAS
    if categories:
        schemas = {k: v for k, v in schemas.items() if k in categories}

    sanitized = {}
    for category, required_keys in schemas.items():
        items = data.get(category, [])
        if not isinstance(items, list):
            logger.warning(f"Category '{category}' is not a list, skipping")
            sanitized[category] = []
            continue

        valid = []
        for i, item in enumerate(items):
            if not isinstance(item, dict):
                logger.warning(f"Item {i} in '{category}' is not a dict, skipping")
                continue
            missing = required_keys - set(item.keys())
            if missing:
                logger.warning(
                    f"Item {i} in '{category}' missing keys {missing}, skipping"
                )
                continue
            valid.append(item)

        sanitized[category] = valid

    return sanitized


# ──────────────────────────────────────────────
# Robust JSON Parsing (brace-balanced)
# ──────────────────────────────────────────────


def _extract_json_block(text: str) -> Optional[str]:
    """
    Extract the first top-level JSON object or array from text using brace balancing.

    Supports both objects ({}) and arrays ([]) at top level.
    Only treats backslash as escape inside quoted strings (not outside).
    """
    obj_start = text.find("{")
    arr_start = text.find("[")

    if obj_start == -1 and arr_start == -1:
        return None

    if obj_start == -1:
        start = arr_start
        open_char, close_char = "[", "]"
    elif arr_start == -1:
        start = obj_start
        open_char, close_char = "{", "}"
    elif obj_start <= arr_start:
        start = obj_start
        open_char, close_char = "{", "}"
    else:
        start = arr_start
        open_char, close_char = "[", "]"

    depth = 0
    in_string = False
    escape = False

    for i in range(start, len(text)):
        c = text[i]

        # Fix #9: Escape handling ONLY inside strings
        if in_string:
            if escape:
                escape = False
                continue
            if c == "\\":
                escape = True
                continue
            if c == '"':
                in_string = False
            continue

        # Outside strings
        if c == '"':
            in_string = True
            continue
        if c == open_char:
            depth += 1
        elif c == close_char:
            depth -= 1
            if depth == 0:
                return text[start : i + 1]

    return None


def _parse_json_response(response: str) -> Optional[dict]:
    """
    Parse a JSON response from the LLM with multiple fallback strategies.

    Strategies (in order):
        1. Direct parse (model returned clean JSON)
        2. Strip markdown code fences and parse
        3. Brace-balanced extraction of first JSON object or array
    """
    # Fix #10: Guard against None input
    if not response:
        return None

    text = response.strip()

    # Strategy 1: Direct parse
    try:
        return json.loads(text)
    except json.JSONDecodeError:
        pass

    # Strategy 2: Strip code fences
    stripped = text
    if stripped.startswith("```"):
        stripped = stripped.split("\n", 1)[1] if "\n" in stripped else stripped[3:]
    if stripped.endswith("```"):
        stripped = stripped.rsplit("```", 1)[0]
    stripped = stripped.strip()

    try:
        return json.loads(stripped)
    except json.JSONDecodeError:
        pass

    # Strategy 3: Brace-balanced extraction
    json_block = _extract_json_block(text)
    if json_block:
        try:
            return json.loads(json_block)
        except json.JSONDecodeError:
            pass

    return None


# Fix #11: Only retry transient/network errors, not auth failures or bad requests
_RETRYABLE_ERRORS = (ConnectionError, TimeoutError, OSError)


def _llm_call_with_retry(
    provider: LLMProvider,
    system: str,
    user: str,
    max_tokens: int = 4096,
    max_retries: int = 2,
    retry_delay: float = 1.0,
) -> Optional[str]:
    """LLM call with exponential backoff retry on transient errors only.

    Retries ConnectionError, TimeoutError, OSError by default.
    Providers can declare additional retryable errors by setting a
    `retryable_errors` attribute (tuple of exception classes).
    """
    # Fix #33: Merge provider-specific retryable errors
    retryable = _RETRYABLE_ERRORS
    if hasattr(provider, "retryable_errors"):
        retryable = retryable + tuple(provider.retryable_errors)
    last_error = None
    for attempt in range(max_retries + 1):
        try:
            return provider.complete(system=system, user=user, max_tokens=max_tokens)
        except Exception as e:
            if isinstance(e, retryable):
                last_error = e
                logger.warning(f"LLM call attempt {attempt + 1} failed (retryable): {e}")
                if attempt < max_retries:
                    time.sleep(retry_delay * (2 ** attempt))
                    continue
            else:
                # Non-retryable error (auth failure, invalid key, etc.)
                logger.error(f"LLM call failed (non-retryable): {e}")
                return None

    logger.error(f"LLM call failed after {max_retries + 1} attempts: {last_error}")
    return None


# ──────────────────────────────────────────────
# Session Manager
# ──────────────────────────────────────────────


class SessionManager:
    """
    Post-conversation agent that extracts learnings and updates the shelf.

    Safety features:
        - File locking prevents concurrent corruption
        - Backups before any writes allow rollback
        - Path validation prevents traversal attacks
        - Schema validation catches malformed LLM output
        - Retry logic handles transient LLM failures
        - Memory entry limits prevent unbounded growth
    """

    def __init__(self, shelf: Shelf, provider: LLMProvider):
        self.shelf = shelf
        self.provider = provider
        self.parser = TranscriptParser()

    def process_file(self, path: str, dry_run: bool = False) -> SessionReport:
        """Process a transcript file through the full pipeline."""
        with open(path, "r", encoding="utf-8") as f:
            content = f.read()
        return self.process_text(content, dry_run=dry_run)

    def process_text(self, content: str, dry_run: bool = False) -> SessionReport:
        """Process a transcript string through the full pipeline."""
        session_id = generate_session_id()
        logger.info(f"Session {session_id}: starting processing")

        # Step 1: Parse
        transcript = self.parser.parse(content)
        logger.info(
            f"Session {session_id}: parsed {transcript.message_count} messages "
            f"({transcript.source_format})"
        )

        if transcript.message_count == 0:
            return SessionReport(
                extraction=Extraction(),
                session_id=session_id,
                applied_changes=[
                    {"type": "info", "message": "Empty transcript, nothing to process."}
                ],
                dry_run=dry_run,
            )

        # Step 2: Extract
        extraction = self._extract(transcript, session_id)

        if extraction.is_empty:
            logger.info(f"Session {session_id}: no learnings extracted")
            return SessionReport(
                extraction=extraction,
                session_id=session_id,
                applied_changes=[
                    {"type": "info", "message": "No notable learnings found in this session."}
                ],
                dry_run=dry_run,
            )

        # Step 3: Deduplicate
        extraction = self._deduplicate(extraction, session_id)

        if extraction.is_empty:
            logger.info(f"Session {session_id}: all extractions deduplicated")
            return SessionReport(
                extraction=extraction,
                session_id=session_id,
                applied_changes=[
                    {"type": "info", "message": "All extractions already exist in shelf."}
                ],
                dry_run=dry_run,
            )

        # Step 4: Preview or apply
        if dry_run:
            report = SessionReport(
                extraction=extraction, session_id=session_id, dry_run=True
            )
            report.applied_changes = self._preview_changes(extraction)
            return report
        else:
            return self._apply_changes(extraction, session_id)

    # ──────────────────────────────────────────────
    # Step 2: EXTRACT
    # ──────────────────────────────────────────────

    def _extract(self, transcript: Transcript, session_id: str) -> Extraction:
        """Use the LLM to extract learnings from the transcript."""

        index_contents = self.shelf.read_index()
        user_memories = self.shelf.read_file(
            "memory/user/preferences.md", default="(empty)"
        )
        agent_memories = self.shelf.read_file(
            "memory/agent/lessons.md", default="(empty)"
        )

        # Fix #13: Truncate at message boundaries, not mid-message
        max_transcript_chars = 60000
        transcript_text = transcript.text
        if len(transcript_text) > max_transcript_chars:
            messages = transcript.messages
            keep = max(2, len(messages) // 4)
            head = messages[:keep]
            tail = messages[-keep:]

            head_text = "\n".join(
                f"## {m.role.title()}\n{m.content}\n" for m in head
            )
            tail_text = "\n".join(
                f"## {m.role.title()}\n{m.content}\n" for m in tail
            )

            omitted = len(messages) - (2 * keep)
            transcript_text = (
                head_text
                + f"\n\n... ({omitted} messages omitted for token budget) ...\n\n"
                + tail_text
            )
            logger.warning(
                f"Session {session_id}: transcript truncated from "
                f"{len(messages)} to {2 * keep} messages"
            )

        user_prompt = EXTRACTION_USER_PROMPT.format(
            index_contents=index_contents,
            user_memories=user_memories,
            agent_memories=agent_memories,
            transcript=transcript_text,
        )

        response = _llm_call_with_retry(
            self.provider, EXTRACTION_SYSTEM_PROMPT, user_prompt
        )

        if response is None:
            logger.error(f"Session {session_id}: extraction LLM call failed")
            return Extraction()

        data = _parse_json_response(response)
        if data is None:
            logger.error(
                f"Session {session_id}: failed to parse extraction response. "
                f"Raw response (first 500 chars): {response[:500]}"
            )
            return Extraction()

        # Validate all categories and enforce runtime extraction policy
        data = _validate_extraction(data)
        data = _apply_extraction_policy(data, self.shelf.load_config())

        logger.info(
            f"Session {session_id}: extracted "
            f"{sum(len(v) for v in data.values())} items across "
            f"{sum(1 for v in data.values() if v)} categories after policy filtering"
        )

        return Extraction(
            user_preferences=data.get("user_preferences", []),
            agent_lessons=data.get("agent_lessons", []),
            task_patterns=data.get("task_patterns", []),
            knowledge_corrections=data.get("knowledge_corrections", []),
            new_context=data.get("new_context", []),
            abstract_updates=data.get("abstract_updates", []),
        )

    # ──────────────────────────────────────────────
    # Step 3: DEDUPLICATE
    # ──────────────────────────────────────────────

    def _deduplicate(self, extraction: Extraction, session_id: str) -> Extraction:
        """Remove extractions that already exist in the shelf."""

        if (
            not extraction.user_preferences
            and not extraction.agent_lessons
            and not extraction.task_patterns
        ):
            return extraction

        existing_user_prefs = self.shelf.read_file(
            "memory/user/preferences.md", default=""
        )
        existing_agent_lessons = self.shelf.read_file(
            "memory/agent/lessons.md", default=""
        )
        existing_patterns = self.shelf.read_file(
            "memory/agent/patterns.md", default=""
        )

        extraction.user_preferences = _deterministic_deduplicate_items(
            extraction.user_preferences, existing_user_prefs
        )
        extraction.agent_lessons = _deterministic_deduplicate_items(
            extraction.agent_lessons, existing_agent_lessons
        )
        extraction.task_patterns = _deterministic_deduplicate_items(
            extraction.task_patterns, existing_patterns
        )

        if (
            not extraction.user_preferences
            and not extraction.agent_lessons
            and not extraction.task_patterns
        ):
            logger.info(f"Session {session_id}: deterministic dedup removed all memory entries")
            return extraction

        new_extractions_json = json.dumps(
            {
                "user_preferences": extraction.user_preferences,
                "agent_lessons": extraction.agent_lessons,
                "task_patterns": extraction.task_patterns,
            },
            indent=2,
        )

        user_prompt = DEDUP_USER_PROMPT.format(
            existing_user_prefs=existing_user_prefs,
            existing_agent_lessons=existing_agent_lessons,
            existing_patterns=existing_patterns,
            new_extractions=new_extractions_json,
        )

        response = _llm_call_with_retry(
            self.provider, DEDUP_SYSTEM_PROMPT, user_prompt
        )

        if response is None:
            logger.warning(
                f"Session {session_id}: dedup LLM call failed, keeping all extractions"
            )
            return extraction

        data = _parse_json_response(response)
        if data is not None:
            # Fix #14: Validate dedup response with same schema as extraction
            dedup_categories = {"user_preferences", "agent_lessons", "task_patterns"}
            data = _validate_extraction(data, categories=dedup_categories)

            extraction.user_preferences = data.get("user_preferences", [])
            extraction.agent_lessons = data.get("agent_lessons", [])
            extraction.task_patterns = data.get("task_patterns", [])
            logger.info(f"Session {session_id}: dedup complete")
        else:
            logger.warning(
                f"Session {session_id}: dedup parse failed, keeping all extractions. "
                f"Raw (first 500): {response[:500] if response else '(empty)'}"
            )

        return extraction

    # ──────────────────────────────────────────────
    # Step 4: APPLY CHANGES (locked + backed up)
    # ──────────────────────────────────────────────

    def _apply_changes(self, extraction: Extraction, session_id: str) -> SessionReport:
        """Apply extracted changes to the shelf under file lock with backup."""

        report = SessionReport(extraction=extraction, session_id=session_id)
        today = datetime.now().strftime("%Y-%m-%d")
        config = self.shelf.load_config()
        max_entries = config.session.max_entries_per_file

        files_to_backup = ["index.md"]
        if extraction.user_preferences:
            files_to_backup.append("memory/user/preferences.md")
        if extraction.agent_lessons:
            files_to_backup.append("memory/agent/lessons.md")
        if extraction.task_patterns:
            files_to_backup.append("memory/agent/patterns.md")
        if extraction.knowledge_corrections:
            files_to_backup.append("memory/agent/corrections_pending.md")

        try:
            with shelf_lock(self.shelf.path):
                logger.info(f"Session {session_id}: lock acquired, backing up files")

                backup_dir = backup_files(
                    self.shelf.path, files_to_backup, session_id
                )
                report.backup_dir = str(backup_dir)

                # Reload index under lock to prevent stale-write races
                self.shelf.index.reload()

                self._apply_memory_entries(
                    report, extraction.user_preferences,
                    "memory/user/preferences.md", "user_preference",
                    today, max_entries, session_id,
                )
                self._apply_memory_entries(
                    report, extraction.agent_lessons,
                    "memory/agent/lessons.md", "agent_lesson",
                    today, max_entries, session_id,
                )
                self._apply_memory_entries(
                    report, extraction.task_patterns,
                    "memory/agent/patterns.md", "task_pattern",
                    today, max_entries, session_id,
                )

                if extraction.knowledge_corrections:
                    if config.session.auto_apply_corrections:
                        report.skipped_changes.append({
                            "type": "knowledge_correction",
                            "reason": (
                                "auto_apply_corrections is enabled, but direct correction "
                                "writes are not supported yet; staged for review instead"
                            ),
                        })
                    self._apply_corrections(
                        report, extraction.knowledge_corrections, today
                    )

                if extraction.new_context:
                    self._apply_new_files(
                        report, extraction.new_context, today
                    )

                if extraction.abstract_updates:
                    self._apply_abstract_updates(report, extraction.abstract_updates)

        except TimeoutError as e:
            logger.error(f"Session {session_id}: {e}")
            report.errors.append({"type": "lock", "error": str(e)})
        # Fix #15: Catch filesystem errors during lock acquisition
        except OSError as e:
            logger.error(f"Session {session_id}: lock filesystem error: {e}")
            report.errors.append(
                {"type": "lock", "error": f"Lock filesystem error: {e}"}
            )

        logger.info(
            f"Session {session_id}: complete. "
            f"{len(report.applied_changes)} applied, {len(report.errors)} errors"
        )

        return report

    def _apply_memory_entries(
        self, report, entries, file_path, entry_type,
        date_str, max_entries, session_id="",
    ):
        """Append extracted entries to a memory file with entry limit check."""
        if not entries:
            return

        try:
            full_path = self.shelf.path / file_path
            current_count = count_memory_entries(full_path)

            # Fix #16: Account for batch size, not just current count
            available = max_entries - current_count

            if available <= 0:
                logger.warning(
                    f"Memory file '{file_path}' has {current_count} entries "
                    f"(limit: {max_entries}). Run `shelfai compact --write` to consolidate."
                )
                report.skipped_changes.append({
                    "type": entry_type,
                    "file": file_path,
                    "reason": (
                        f"Entry limit reached ({current_count}/{max_entries}). "
                        f"Run `shelfai compact --write`."
                    ),
                    "entries": [e.get("content", "") for e in entries],
                })
                return

            # Truncate to available slots if batch would overshoot
            if len(entries) > available:
                skipped_entries = entries[available:]
                entries = entries[:available]
                logger.warning(
                    f"Session {session_id}: truncating {len(entries) + len(skipped_entries)} "
                    f"entries to {len(entries)} (limit: {max_entries}, "
                    f"current: {current_count})"
                )
                report.skipped_changes.append({
                    "type": entry_type,
                    "file": file_path,
                    "reason": (
                        f"Partial write: {len(skipped_entries)} entries skipped "
                        f"(limit: {max_entries})"
                    ),
                    "entries": [e.get("content", "") for e in skipped_entries],
                })

            lines = []
            for entry in entries:
                confidence = entry.get("confidence", "medium")
                content = entry.get("content", "(no content)")
                lines.append(f"- {content} [confidence: {confidence}]")

            self.shelf.append_to_file(
                file_path,
                f"\n## {date_str}\n" + "\n".join(lines) + "\n",
            )
            report.applied_changes.append({
                "type": entry_type,
                "file": file_path,
                "count": len(entries),
                "entries": [e.get("content", "") for e in entries],
            })
        except Exception as e:
            logger.error(f"Failed to append {entry_type} to {file_path}: {e}")
            report.errors.append({"type": entry_type, "error": str(e)})

    def _apply_corrections(self, report, corrections, date_str):
        """Write knowledge corrections to pending review file."""
        try:
            entries = []
            for correction in corrections:
                entries.append(
                    f"- **{correction.get('file', '?')}**: "
                    f"{correction.get('issue', '?')} → "
                    f"{correction.get('correction', '?')}"
                )
            self.shelf.append_to_file(
                "memory/agent/corrections_pending.md",
                f"\n## {date_str}\n" + "\n".join(entries) + "\n",
            )
            report.applied_changes.append({
                "type": "knowledge_correction",
                "file": "memory/agent/corrections_pending.md",
                "count": len(corrections),
                "note": "Saved for manual review (not auto-applied)",
                "entries": [c.get("issue", "") for c in corrections],
            })
        except Exception as e:
            logger.error(f"Failed to write corrections: {e}")
            report.errors.append({"type": "knowledge_correction", "error": str(e)})

    def _apply_new_files(self, report, new_context, date_str):
        """Stage or create new shelf files suggested by the session agent."""
        config = self.shelf.load_config()
        allowed_categories = config.session.new_context_allowed_categories or ["memory"]
        max_abstract_length = config.index.abstract_max_length
        min_abstract_length = config.index.abstract_min_length
        mode = (config.session.new_context_mode or "stage").strip().lower()
        stage_dir = config.session.new_context_stage_dir

        for idx, ctx in enumerate(new_context, 1):
            try:
                path = ctx.get("suggested_path", "")
                validate_path(self.shelf.path, path)
                _validate_new_context_path(path, allowed_categories)

                full_path = self.shelf.path / path
                if full_path.exists():
                    raise ValueError(f"Refusing to overwrite existing file: {path}")

                abstract = _sanitize_abstract(
                    ctx.get("suggested_abstract", "(no abstract)"),
                    max_length=max_abstract_length,
                )
                lint_issues = _lint_abstract(
                    abstract,
                    min_length=min_abstract_length,
                    max_length=max_abstract_length,
                )
                if lint_issues:
                    raise ValueError("; ".join(lint_issues))

                category = Path(path).parts[0]
                if mode == "apply":
                    if category != "memory":
                        raise ValueError(
                            f"Apply mode may only auto-create files in memory/; stage proposals for {category}/ instead"
                        )
                    content = self._generate_new_file_content(ctx, date_str)
                    self.shelf.write_file(path, content)
                    self.shelf.add_index_entry(file_path=path, abstract=abstract)
                    report.applied_changes.append({
                        "type": "new_file",
                        "file": path,
                        "abstract": abstract,
                    })
                else:
                    review_slug = re.sub(r"[^a-zA-Z0-9_-]", "-", Path(path).stem).strip("-") or "proposal"
                    session_slug = re.sub(r"[^a-zA-Z0-9_-]", "", report.session_id)[:8] or "session"
                    stage_path = f"{stage_dir}/{date_str}-{idx:02d}-{review_slug}-{session_slug}.md"
                    stage_content = (
                        f"---\n"
                        f"type: staged_new_context\n"
                        f"created_by: session_manager\n"
                        f"created_at: {date_str}\n"
                        f"proposed_path: {path}\n"
                        f"proposed_abstract: {abstract}\n"
                        f"---\n\n"
                        f"# Proposed New Context\n\n"
                        f"## Proposed Path\n`{path}`\n\n"
                        f"## Proposed Abstract\n{abstract}\n\n"
                        f"## Content Summary\n{ctx.get('content_summary', '')}\n"
                    )
                    self.shelf.write_file(stage_path, stage_content)
                    report.applied_changes.append({
                        "type": "staged_new_file",
                        "file": stage_path,
                        "content": f"Staged proposed file {path} for review",
                    })
            except ValueError as e:
                logger.error(f"Path validation failed for new file: {e}")
                report.errors.append({"type": "new_file", "error": str(e)})
            except Exception as e:
                logger.error(f"Failed to create new file: {e}")
                report.errors.append({"type": "new_file", "error": str(e)})

    def _apply_abstract_updates(self, report, updates):
        """Update abstracts in the index."""
        for update in updates:
            # Fix #19: All .get() access, no bracket access
            file_path = update.get("file", "")
            new_abstract = update.get("suggested_abstract", "")

            if not file_path or not new_abstract:
                logger.warning(
                    f"Skipping abstract update with missing file or abstract: {update}"
                )
                continue

            # Fix #18: Check file exists in index before updating
            existing = self.shelf.index.get(file_path)
            if existing is None:
                logger.warning(
                    f"Abstract update skipped: '{file_path}' not in index "
                    f"(possible LLM hallucination)"
                )
                report.skipped_changes.append({
                    "type": "abstract_update",
                    "file": file_path,
                    "reason": "File not found in index",
                })
                continue

            try:
                clean_abstract = _sanitize_abstract(
                    new_abstract,
                    max_length=self.shelf.load_config().index.max_abstract_length,
                )
                lint_issues = _lint_abstract(
                    clean_abstract,
                    min_length=self.shelf.load_config().index.abstract_min_length,
                    max_length=self.shelf.load_config().index.abstract_max_length,
                )
                if lint_issues:
                    raise ValueError("; ".join(lint_issues))
                self.shelf.update_index_abstract(
                    file_path=file_path, new_abstract=clean_abstract,
                )
                report.applied_changes.append({
                    "type": "abstract_update",
                    "file": file_path,
                    "old_abstract": update.get("current_abstract", ""),
                    "new_abstract": clean_abstract,
                })
            except Exception as e:
                logger.error(f"Failed to update abstract for {file_path}: {e}")
                report.errors.append({"type": "abstract_update", "error": str(e)})

    def _preview_changes(self, extraction: Extraction) -> list:
        """Generate a preview of changes for dry-run mode."""
        changes = []

        for pref in extraction.user_preferences:
            changes.append({
                "type": "user_preference", "action": "append",
                "file": "memory/user/preferences.md",
                "content": pref.get("content", ""),
                "confidence": pref.get("confidence", "medium"),
            })

        for lesson in extraction.agent_lessons:
            changes.append({
                "type": "agent_lesson", "action": "append",
                "file": "memory/agent/lessons.md",
                "content": lesson.get("content", ""),
                "confidence": lesson.get("confidence", "medium"),
            })

        for pattern in extraction.task_patterns:
            changes.append({
                "type": "task_pattern", "action": "append",
                "file": "memory/agent/patterns.md",
                "content": pattern.get("content", ""),
                "confidence": pattern.get("confidence", "medium"),
            })

        for correction in extraction.knowledge_corrections:
            changes.append({
                "type": "knowledge_correction", "action": "flag_for_review",
                "file": correction.get("file", ""),
                "issue": correction.get("issue", ""),
                "correction": correction.get("correction", ""),
            })

        new_context_mode = (self.shelf.load_config().session.new_context_mode or "stage").strip().lower()
        for ctx in extraction.new_context:
            changes.append({
                "type": "staged_new_file" if new_context_mode != "apply" else "new_file",
                "action": "stage" if new_context_mode != "apply" else "create",
                "file": ctx.get("suggested_path", ""),
                "abstract": ctx.get("suggested_abstract", ""),
            })

        for update in extraction.abstract_updates:
            changes.append({
                "type": "abstract_update", "action": "update",
                "file": update.get("file", ""),
                "old_abstract": update.get("current_abstract", "(unknown)"),
                "new_abstract": update.get("suggested_abstract", ""),
                "reason": update.get("reason", ""),
            })

        return changes

    # ──────────────────────────────────────────────
    # Helpers
    # ──────────────────────────────────────────────

    def _generate_new_file_content(self, ctx: dict, date_str: str) -> str:
        """Generate markdown content for a new shelf file."""
        filename = ctx.get("suggested_path", "untitled.md")
        title = Path(filename).stem.replace("_", " ").replace("-", " ").title()
        # Fix #20: Sanitize title for YAML frontmatter safety
        title = title.replace("\n", " ").replace(":", " -").replace("---", "").strip()
        # Fix #35: Guard against empty title after sanitization
        if not title:
            title = "Untitled"
        category = (
            Path(filename).parts[0] if len(Path(filename).parts) > 1 else "unknown"
        )

        return (
            f"---\n"
            f"title: {title}\n"
            f"type: {category}\n"
            f"created: {date_str}\n"
            f"updated: {date_str}\n"
            f"source: session-manager\n"
            f"---\n\n"
            f"# {title}\n\n"
            f"{ctx.get('content_summary', '(Content to be filled in.)')}\n"
        )
