#!/usr/bin/env python3
from __future__ import annotations

import argparse
import fnmatch
import hashlib
import json
import os
import sys
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable

# NOTE: extract deps (bs4/pypdf) are only needed for `condense`.
# We import extract_file lazily inside the condense path so `index/query` can run without them.

DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434"
DEFAULT_OLLAMA_MODEL = "llama3.2:3b"
DEFAULT_GATEWAY_URL = "http://127.0.0.1:18789"


@dataclass
class Prompts:
    outline: str
    execution_notes: str
    tool_summary: str


def build_prompts(*, token_cap_hint: str, outline_max_tokens: int, exec_max_tokens: int, summary_max_tokens: int) -> Prompts:
    # Stage A: loss-minimized outline. Keep very explicit, ban prose.
    outline = f"""You are converting documentation into a LOSS-MINIMIZED, low-fluff OUTLINE for an executor AI.

Return ONLY markdown.

FORMAT:
- Use headings to preserve structure.
- Under each heading, only use bullet lists.
- Prefer exact names, symbols, commands, function names, parameters, file paths.

For each section, prioritize these bullets (when present):
- Definitions / key terms
- Interfaces (commands/APIs/classes/functions) + important fields/flags
- Procedures (step lists)
- Constraints/assumptions
- Failure modes / gotchas
- Examples (short)

RULES:
- No marketing prose. No table of contents. No change log.
- Keep as much factual content as possible.
- Target <= {outline_max_tokens} output tokens.

SOURCE (extracted text) below:
---
"""

    # Stage B1: execution notes (from outline if enabled)
    execution_notes = f"""You are an expert at converting documentation into EXECUTION-RELEVANT notes for an AI agent that can run tools (CLI commands, HTTP calls, scripts, functions).

Return ONLY markdown.

OUTPUT FORMAT (Markdown):
- Title
- What this tool/service is
- When to use
- Inputs (required/optional)
- Outputs
- Preconditions / assumptions
- Golden path (numbered steps)
- Verification checks
- Common errors + fixes
- Safety/rollback notes

RULES:
- Be concise and operational; no marketing.
- Prefer concrete commands, flags, endpoints, example payloads.
- Keep within {token_cap_hint}. Target <= {exec_max_tokens} output tokens.

SOURCE (extracted text) below:
---
"""

    # Stage B2: tool index entry (template)
    tool_summary = f"""You are an expert at writing ULTRA-CONDENSED, AI-readable TOOL INDEX entries.

Return ONLY markdown, and ONLY the filled-in TEMPLATE below.

TEMPLATE (replace the angle-bracket placeholders; keep headings verbatim):

# <TOOL_NAME>

**Purpose:** <ONE_SENTENCE>

**Capabilities:**
- <BULLET>
- <BULLET>
- <BULLET>

**Requires:**
- <BULLET_OR_Unknown>

**Entrypoints:**
- <BULLET_OR_Unknown>

**Limits / footguns:**
- <BULLET>

RULES:
- No table of contents, no prose summary, no history, no change log.
- Prefer symbols, backticks, and short bullets over sentences.
- Rewrite commands is OK, but reference the real command/name/path when the doc provides it.
- If the doc describes multiple components, choose the primary "tool" and mention others only as bullets.
- If a dependency/entrypoint is not explicitly in the source, write `Unknown`.
- Target <= {summary_max_tokens} output tokens.

SOURCE (extracted text) below:
---
"""

    return Prompts(outline=outline, execution_notes=execution_notes, tool_summary=tool_summary)


def iter_input_files(root: Path) -> Iterable[Path]:
    if root.is_file():
        yield root
        return

    exts = {".pdf", ".txt", ".md", ".html", ".htm"}
    for p in root.rglob("*"):
        if p.is_file() and p.suffix.lower() in exts:
            yield p


def post_json(url: str, data: dict, timeout: int = 120, headers: dict[str, str] | None = None) -> dict:
    body = json.dumps(data).encode("utf-8")
    h = {"Content-Type": "application/json"}
    if headers:
        h.update(headers)
    req = urllib.request.Request(url, data=body, headers=h)
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return json.loads(resp.read().decode("utf-8"))


def sha256_text(s: str) -> str:
    return hashlib.sha256(s.encode("utf-8", errors="ignore")).hexdigest()


def ollama_generate(*, ollama_url: str, model: str, prompt: str, num_predict: int) -> str:
    data = post_json(
        f"{ollama_url}/api/generate",
        {
            "model": model,
            "prompt": prompt,
            "stream": False,
            "options": {
                "num_predict": num_predict,
                "temperature": 0.2,
            },
        },
        timeout=600,
    )
    return (data.get("response") or "").strip()


def openclaw_generate(*, gateway_url: str, gateway_token: str, agent_id: str, prompt: str, max_output_tokens: int) -> str:
    data = post_json(
        f"{gateway_url}/v1/responses",
        {
            "model": "openclaw",
            "input": prompt,
            "max_output_tokens": max_output_tokens,
        },
        timeout=600,
        headers={
            "Authorization": f"Bearer {gateway_token}",
            "x-openclaw-agent-id": agent_id,
        },
    )

    out_parts: list[str] = []
    for item in data.get("output") or []:
        if item.get("type") != "message":
            continue
        for part in item.get("content") or []:
            if part.get("type") == "output_text" and part.get("text"):
                out_parts.append(part["text"])

    return "\n".join(out_parts).strip()


def generate_with_retries(*, engine: str, retries: int, sleep_s: float, prompt: str, max_tokens: int, ollama_url: str, ollama_model: str, gateway_url: str, gateway_token: str, agent_id: str) -> str:
    last_err: Exception | None = None
    for attempt in range(retries + 1):
        try:
            if engine == "ollama":
                return ollama_generate(ollama_url=ollama_url, model=ollama_model, prompt=prompt, num_predict=max_tokens)
            return openclaw_generate(gateway_url=gateway_url, gateway_token=gateway_token, agent_id=agent_id, prompt=prompt, max_output_tokens=max_tokens)
        except Exception as e:
            last_err = e
            if attempt < retries:
                time.sleep(sleep_s * (attempt + 1))
                continue
            raise
    raise RuntimeError(str(last_err))


def rel_output_path(input_path: Path, input_root: Path, out_root: Path, suffix: str) -> Path:
    if input_root.is_file():
        rel = input_path.name
    else:
        rel = str(input_path.relative_to(input_root))

    base = Path(rel)
    out_rel = base.with_suffix("")
    return out_root / out_rel.parent / (out_rel.name + suffix)


def matches_any(path_str: str, patterns: list[str]) -> bool:
    return any(fnmatch.fnmatch(path_str, pat) for pat in patterns)


def validate_tool_summary(md: str) -> list[str]:
    # Minimal structural validation (keeps us from silently accepting garbage).
    problems: list[str] = []
    required = [
        "# ",
        "**Purpose:**",
        "**Capabilities:**",
        "**Requires:**",
        "**Entrypoints:**",
        "**Limits / footguns:**",
    ]
    for r in required:
        if r not in md:
            problems.append(f"missing:{r}")

    # place-holder leakage
    if "<TOOL_NAME>" in md or "<ONE_SENTENCE>" in md or "<BULLET" in md:
        problems.append("contains_placeholders")

    # too short tends to mean it failed / got truncated into junk
    if len(md.strip()) < 120:
        problems.append("too_short")

    return problems


def validate_outline(md: str) -> list[str]:
    problems: list[str] = []
    if len(md.strip()) < 400:
        problems.append("too_short")
    # Require at least one heading to avoid useless blobs
    if "#" not in md:
        problems.append("missing_headings")
    # Often indicates model refused / answered generically
    if md.strip().lower().startswith("i can") or md.strip().lower().startswith("sorry"):
        problems.append("refusal_or_meta")
    return problems


def validate_execution_notes(md: str) -> list[str]:
    problems: list[str] = []
    if not md.lstrip().startswith("#"):
        problems.append("missing_title")
    if "Golden path" not in md and "golden path" not in md:
        problems.append("missing_golden_path")
    if len(md.strip()) < 300:
        problems.append("too_short")
    return problems


def estimate_tokens(text: str) -> int:
    """Very rough token estimate (good enough for sizing nodes)."""
    return max(1, int(len(text) / 4))


def shard_outline_to_nodes(
    *,
    outline_md: str,
    nodes_dir: Path,
    node_min_tokens: int = 200,
    node_max_tokens: int = 600,
    node_max_chars: int = 1200,
) -> list[Path]:
    """Split outline into small, embed-friendly topic nodes.

    Strategy:
    - Split by H2 headings (##)
    - If a section is too large, further split into multiple parts.
    - If the final part is too small (< node_min_tokens), merge it into the previous part.
    """
    nodes_dir.mkdir(parents=True, exist_ok=True)

    if node_min_tokens < 1 or node_max_tokens < 1 or node_max_chars < 200:
        raise ValueError("node_min_tokens/node_max_tokens/node_max_chars must be positive")
    if node_min_tokens > node_max_tokens:
        raise ValueError("node_min_tokens must be <= node_max_tokens")

    lines = outline_md.splitlines()
    sections: list[tuple[str, list[str]]] = []
    cur_title = "overview"
    cur_lines: list[str] = []

    def flush():
        nonlocal cur_title, cur_lines
        if cur_lines:
            sections.append((cur_title, cur_lines))
        cur_lines = []

    for line in lines:
        if line.startswith("## "):
            flush()
            cur_title = line[3:].strip() or "section"
            cur_lines.append(line)
        else:
            cur_lines.append(line)
    flush()

    def slugify(title: str, idx: int) -> str:
        slug = (
            title.lower()
            .replace("/", " ")
            .replace("\\", " ")
            .replace(":", " ")
            .replace("  ", " ")
            .strip()
        )
        slug = "-".join([p for p in slug.split() if p])[:60] or f"section-{idx}"
        return slug

    out_paths: list[Path] = []
    out_i = 1

    for sec_i, (title, sec_lines) in enumerate(sections, start=1):
        slug = slugify(title, sec_i)

        sec_text = "\n".join(sec_lines).strip() + "\n"
        if estimate_tokens(sec_text) <= node_max_tokens and len(sec_text) <= node_max_chars:
            p = nodes_dir / f"{out_i:02d}-{slug}.md"
            p.write_text(sec_text, encoding="utf-8")
            out_paths.append(p)
            out_i += 1
            continue

        heading = sec_lines[0] if sec_lines and sec_lines[0].startswith("## ") else f"## {title}"
        body = sec_lines[1:] if sec_lines and sec_lines[0].startswith("## ") else sec_lines

        parts: list[str] = []
        cur: list[str] = [heading]

        for line in body:
            candidate = "\n".join(cur + [line]).strip() + "\n"
            if (estimate_tokens(candidate) > node_max_tokens or len(candidate) > node_max_chars) and len(cur) > 1:
                parts.append("\n".join(cur).strip() + "\n")
                cur = [heading, line]
            else:
                cur.append(line)

        if len(cur) > 1:
            parts.append("\n".join(cur).strip() + "\n")

        # Merge trailing tiny part into previous part.
        if len(parts) >= 2 and estimate_tokens(parts[-1]) < node_min_tokens:
            parts[-2] = parts[-2].rstrip() + "\n" + parts[-1]
            parts = parts[:-1]

        for part_i, text in enumerate(parts, start=1):
            # Cap to ensure embed-friendly nodes.
            if len(text) > node_max_chars:
                text = text[:node_max_chars] + "\n\n[TRUNCATED]\n"

            suffix = f"--p{part_i:02d}" if len(parts) > 1 else ""
            p = nodes_dir / f"{out_i:02d}-{slug}{suffix}.md"
            p.write_text(text, encoding="utf-8")
            out_paths.append(p)
            out_i += 1

    return out_paths


def write_doc_index(*, doc_index_path: Path, source_path: Path, tool_summary: Path, execution_notes: Path, outline_path: Path | None, node_paths: list[Path]) -> None:
    doc_index_path.parent.mkdir(parents=True, exist_ok=True)

    rel = lambda p: p.name if p.parent == doc_index_path.parent else str(p.relative_to(doc_index_path.parent))

    lines: list[str] = []
    lines.append(f"# {source_path.stem}")
    lines.append("")
    lines.append("## Outputs")
    lines.append(f"- Tool summary: [{tool_summary.name}]({rel(tool_summary)})")
    lines.append(f"- Execution notes: [{execution_notes.name}]({rel(execution_notes)})")
    if outline_path:
        lines.append(f"- Outline: [{outline_path.name}]({rel(outline_path)})")
    if node_paths:
        lines.append("")
        lines.append("## Nodes")
        for p in node_paths:
            lines.append(f"- [{p.name}]({rel(p)})")
    lines.append("")
    lines.append("## Source")
    lines.append(f"- `{source_path}`")
    lines.append("")

    doc_index_path.write_text("\n".join(lines), encoding="utf-8")


def write_root_index(*, out_root: Path, input_root: Path, doc_indices: list[Path]) -> None:
    """Human-browsable top index that links to per-doc indices."""
    index_path = out_root / "index.md"
    lines: list[str] = []
    lines.append("# RAGLite Index")
    lines.append("")
    lines.append(f"- Source: `{input_root}`")
    lines.append(f"- Generated: {time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime())}")
    lines.append("")

    if not doc_indices:
        lines.append("(No per-doc indices were generated. Run with `--nodes`.)")
        index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
        return

    lines.append("## Documents")
    for p in sorted(doc_indices):
        rel = str(p.relative_to(out_root))
        lines.append(f"- [{p.stem}]({rel})")

    index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def load_cache(cache_path: Path) -> dict:
    try:
        return json.loads(cache_path.read_text(encoding="utf-8"))
    except Exception:
        return {"files": {}}


def save_cache(cache_path: Path, cache: dict) -> None:
    cache_path.parent.mkdir(parents=True, exist_ok=True)
    cache_path.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n", encoding="utf-8")


def cli() -> None:
    """Console-script entrypoint."""
    raise SystemExit(main())


def main() -> int:
    ap = argparse.ArgumentParser(description="RAGLite CLI")
    sub = ap.add_subparsers(dest="cmd", required=True)

    # --- Condense ---
    ap_c = sub.add_parser("condense", help="Extract + distill docs into markdown artifacts")
    ap_c.add_argument("input", help="File or directory to process")
    ap_c.add_argument("--out", default="./raglite_out", help="Output directory")

    ap_c.add_argument("--engine", choices=["ollama", "openclaw"], default="ollama", help="Generation engine")

    ap_c.add_argument("--ollama-url", default=DEFAULT_OLLAMA_URL)
    ap_c.add_argument("--ollama-model", default=DEFAULT_OLLAMA_MODEL)

    ap_c.add_argument("--gateway-url", default=DEFAULT_GATEWAY_URL)
    ap_c.add_argument("--gateway-token", default=os.environ.get("OPENCLAW_GATEWAY_TOKEN", ""))
    ap_c.add_argument("--gateway-agent-id", default="main")

    ap_c.add_argument("--exec-max-tokens", type=int, default=1200)
    ap_c.add_argument("--summary-max-tokens", type=int, default=350)
    ap_c.add_argument("--outline", action="store_true", help="Generate a loss-minimized outline first and summarize from it")
    ap_c.add_argument("--outline-max-tokens", type=int, default=5000)
    ap_c.add_argument("--nodes", action="store_true", help="Write outline shards + per-doc index linking nodes")
    ap_c.add_argument("--node-min-tokens", type=int, default=200, help="Minimum target size for a node (approx tokens). Small trailing parts are merged.")
    ap_c.add_argument("--node-max-tokens", type=int, default=600, help="Maximum target size for a node (approx tokens)")
    ap_c.add_argument("--node-max-chars", type=int, default=1200, help="Hard cap for node size (chars) to keep nodes embed-friendly")

    ap_c.add_argument("--max-chars", type=int, default=180_000, help="Max extracted chars per file")
    ap_c.add_argument("--sleep-ms", type=int, default=0, help="Sleep between files (throttle)")

    ap_c.add_argument("--skip-existing", action="store_true")
    ap_c.add_argument("--only", choices=["all", "tool-summary", "execution-notes"], default="all")

    ap_c.add_argument("--include", action="append", default=[], help="Glob(s) to include (default: all)")
    ap_c.add_argument("--exclude", action="append", default=[], help="Glob(s) to exclude")
    ap_c.add_argument("--max-files", type=int, default=0, help="Process at most N files (0 = no limit)")

    ap_c.add_argument("--retries", type=int, default=2)
    ap_c.add_argument("--retry-sleep", type=float, default=1.5)
    ap_c.add_argument("--max-errors", type=int, default=10)
    ap_c.add_argument("--fail-fast", action="store_true")
    ap_c.add_argument("--validate", action="store_true", help="Validate outputs; retry once if invalid")

    # --- Index ---
    ap_i = sub.add_parser("index", help="Embed + store a distilled directory into Chroma")
    ap_i.add_argument("distilled", help="Distilled output directory (from condense)")
    ap_i.add_argument("--collection", required=True, help="Chroma collection name")
    ap_i.add_argument("--chroma-url", default="http://127.0.0.1:8100")
    ap_i.add_argument("--ollama-url", default="http://127.0.0.1:11434")
    ap_i.add_argument("--embed-model", default="nomic-embed-text")
    ap_i.add_argument("--embed-max-chars", type=int, default=800, help="Max chars passed into embedding model")
    ap_i.add_argument("--sleep-ms", type=int, default=0)
    ap_i.add_argument("--include-outlines", action="store_true", help="Also index *.outline.md (default: skip)")
    ap_i.add_argument(
        "--include-kinds",
        default="",
        help="Comma-separated kinds to include (filters default set). Kinds: node,tool-summary,execution-notes,index,root-index,outline,md",
    )
    ap_i.add_argument(
        "--exclude-kinds",
        default="",
        help="Comma-separated kinds to exclude. Kinds: node,tool-summary,execution-notes,index,root-index,outline,md",
    )
    ap_i.add_argument("--skip-indexed", action="store_true", help="Skip chunks already indexed (via .raglite/index_cache.json)")

    # --- Run (condense + index) ---
    ap_r = sub.add_parser("run", help="One-command pipeline: condense then index into a single Chroma collection")
    ap_r.add_argument("input", help="File or directory to process")
    ap_r.add_argument("--out", default="./raglite_out", help="Output directory")
    ap_r.add_argument("--collection", required=True, help="Chroma collection name")
    ap_r.add_argument("--chroma-url", default="http://127.0.0.1:8100")
    ap_r.add_argument("--ollama-url", default=DEFAULT_OLLAMA_URL)
    ap_r.add_argument("--embed-model", default="nomic-embed-text")
    ap_r.add_argument("--embed-max-chars", type=int, default=800)
    ap_r.add_argument(
        "--skip-indexed",
        action="store_true",
        help="Skip chunks already indexed (via .raglite/index_cache.json)",
    )
    ap_r.add_argument("--include-outlines", action="store_true")
    ap_r.add_argument("--include-kinds", default="")
    ap_r.add_argument("--exclude-kinds", default="")

    # Condense options (same as condense)
    ap_r.add_argument("--engine", choices=["ollama", "openclaw"], default="ollama")
    ap_r.add_argument("--ollama-model", default=DEFAULT_OLLAMA_MODEL)
    ap_r.add_argument("--gateway-url", default=DEFAULT_GATEWAY_URL)
    ap_r.add_argument("--gateway-token", default=os.environ.get("OPENCLAW_GATEWAY_TOKEN", ""))
    ap_r.add_argument("--gateway-agent-id", default="main")
    ap_r.add_argument("--exec-max-tokens", type=int, default=1200)
    ap_r.add_argument("--summary-max-tokens", type=int, default=350)
    ap_r.add_argument("--outline", action="store_true")
    ap_r.add_argument("--outline-max-tokens", type=int, default=5000)
    ap_r.add_argument("--nodes", action="store_true")
    ap_r.add_argument("--node-min-tokens", type=int, default=200)
    ap_r.add_argument("--node-max-tokens", type=int, default=600)
    ap_r.add_argument("--node-max-chars", type=int, default=1200)
    ap_r.add_argument("--max-chars", type=int, default=180_000)
    ap_r.add_argument("--sleep-ms", type=int, default=0)
    ap_r.add_argument("--skip-existing", action="store_true")
    ap_r.add_argument("--only", choices=["all", "tool-summary", "execution-notes"], default="all")
    ap_r.add_argument("--include", action="append", default=[])
    ap_r.add_argument("--exclude", action="append", default=[])
    ap_r.add_argument("--max-files", type=int, default=0)
    ap_r.add_argument("--retries", type=int, default=2)
    ap_r.add_argument("--retry-sleep", type=float, default=1.5)
    ap_r.add_argument("--max-errors", type=int, default=10)
    ap_r.add_argument("--fail-fast", action="store_true")
    ap_r.add_argument("--validate", action="store_true")

    # --- Query ---
    ap_q = sub.add_parser("query", help="Hybrid search (vector + keyword) over an indexed distilled directory")
    ap_q.add_argument("distilled", help="Distilled output directory")
    ap_q.add_argument("--collection", required=True)
    ap_q.add_argument("--chroma-url", default="http://127.0.0.1:8100")
    ap_q.add_argument("--ollama-url", default="http://127.0.0.1:11434")
    ap_q.add_argument("--embed-model", default="nomic-embed-text")
    ap_q.add_argument("--embed-max-chars", type=int, default=800, help="Max chars passed into embedding model")
    ap_q.add_argument("--top-k", type=int, default=10)
    ap_q.add_argument("--keyword-top-k", type=int, default=10)
    ap_q.add_argument("query", help="Search query")

    args = ap.parse_args()

    # --- Index command ---
    if args.cmd == "index":
        try:
            from .vector_index import index_distilled_dir
        except ImportError:  # pragma: no cover
            from raglite.vector_index import index_distilled_dir

        def _parse_csv_set(s: str) -> set[str] | None:
            parts = [p.strip() for p in (s or "").split(",") if p.strip()]
            return set(parts) if parts else None

        distilled_root = Path(args.distilled).expanduser().resolve()
        index_cache_path = (distilled_root / ".raglite" / "index_cache.json") if args.skip_indexed else None

        res = index_distilled_dir(
            distilled_root=distilled_root,
            chroma_url=args.chroma_url,
            collection=args.collection,
            ollama_url=args.ollama_url,
            embed_model=args.embed_model,
            embed_max_chars=args.embed_max_chars,
            sleep_ms=args.sleep_ms,
            include_outlines=bool(args.include_outlines),
            include_kinds=_parse_csv_set(args.include_kinds),
            exclude_kinds=_parse_csv_set(args.exclude_kinds),
            index_cache_path=index_cache_path,
            skip_indexed=bool(args.skip_indexed),
        )
        print(json.dumps(res, indent=2))
        return 0

    # --- Query command ---
    if args.cmd == "query":
        try:
            from .vector_index import query_distilled
        except ImportError:  # pragma: no cover
            from raglite.vector_index import query_distilled

        res = query_distilled(
            query=args.query,
            distilled_root=Path(args.distilled).expanduser().resolve(),
            chroma_url=args.chroma_url,
            collection=args.collection,
            ollama_url=args.ollama_url,
            embed_model=args.embed_model,
            embed_max_chars=args.embed_max_chars,
            top_k=args.top_k,
            keyword_top_k=args.keyword_top_k,
        )
        print(json.dumps(res, indent=2))
        return 0

    # --- Condense / Run command ---
    if args.cmd == "run":
        # 1) condense into --out
        run_condense_args = args
        # reuse the same condense path by setting fields as expected below
        args = run_condense_args
        args.cmd = "condense"  # type: ignore[attr-defined]

        # After condense finishes, we index the output.
        should_index_after = True
    else:
        should_index_after = False

    assert args.cmd == "condense"

    input_root = Path(args.input).expanduser().resolve()
    out_root = Path(args.out).expanduser().resolve()
    out_root.mkdir(parents=True, exist_ok=True)

    meta_dir = out_root / ".raglite"
    cache_path = meta_dir / "cache.json"
    errors_log = meta_dir / "errors.log"
    run_stats_path = meta_dir / "run_stats.json"

    cache = load_cache(cache_path)

    gateway_token = args.gateway_token
    if args.engine == "openclaw" and not gateway_token:
        try:
            cfg_path = Path("~/.openclaw/openclaw.json").expanduser()
            cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
            gateway_token = str(cfg.get("gateway", {}).get("auth", {}).get("token", ""))
        except Exception:
            gateway_token = ""

    if args.engine == "openclaw" and not gateway_token:
        print("ERROR: --gateway-token (or OPENCLAW_GATEWAY_TOKEN) is required for engine=openclaw", file=sys.stderr)
        return 2

    prompts = build_prompts(
        token_cap_hint=f"~{args.exec_max_tokens} tokens max",
        outline_max_tokens=args.outline_max_tokens,
        exec_max_tokens=args.exec_max_tokens,
        summary_max_tokens=args.summary_max_tokens,
    )

    all_files = list(iter_input_files(input_root))
    if not all_files:
        print("No matching files found.", file=sys.stderr)
        return 2

    # include/exclude filtering is based on path relative to input root
    files: list[Path] = []
    for p in all_files:
        rel = p.name if input_root.is_file() else str(p.relative_to(input_root))
        if args.include and not matches_any(rel, args.include):
            continue
        if args.exclude and matches_any(rel, args.exclude):
            continue
        files.append(p)

    total = len(files)
    if args.max_files and args.max_files > 0:
        files = files[: args.max_files]

    start = time.time()
    stats = {
        "engine": args.engine,
        "ollama_model": args.ollama_model,
        "outline": bool(args.outline),
        "nodes": bool(args.nodes),
        "input": str(input_root),
        "out": str(out_root),
        "total_candidates": total,
        "processed": 0,
        "ok": 0,
        "skipped": 0,
        "errors": 0,
        "startedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
    }

    def log_error(msg: str) -> None:
        errors_log.parent.mkdir(parents=True, exist_ok=True)
        with errors_log.open("a", encoding="utf-8", errors="ignore") as f:
            f.write(msg + "\n")

    doc_indices: list[Path] = []

    for idx, p in enumerate(files, start=1):
        stats["processed"] += 1
        rel = p.name if input_root.is_file() else str(p.relative_to(input_root))

        try:
            try:
                from .extract import extract_file
            except ImportError:  # pragma: no cover
                from raglite.extract import extract_file

            extracted = extract_file(p)
            text = extracted.text
            if len(text) > args.max_chars:
                text = text[: args.max_chars] + "\n\n[TRUNCATED]"

            text_hash = sha256_text(text)

            exec_out = rel_output_path(p, input_root, out_root, ".execution-notes.md")
            sum_out = rel_output_path(p, input_root, out_root, ".tool-summary.md")
            outline_out = rel_output_path(p, input_root, out_root, ".outline.md")
            nodes_dir = rel_output_path(p, input_root, out_root, "")
            nodes_dir = nodes_dir.parent / nodes_dir.name  # directory for this doc
            nodes_dir = nodes_dir / "nodes"
            doc_index = rel_output_path(p, input_root, out_root, "")
            doc_index = doc_index.parent / (doc_index.name + ".index.md")

            exec_out.parent.mkdir(parents=True, exist_ok=True)

            cached = cache.get("files", {}).get(rel)
            if args.skip_existing and exec_out.exists() and sum_out.exists():
                stats["skipped"] += 1
                continue
            if cached and cached.get("hash") == text_hash and exec_out.exists() and sum_out.exists() and args.skip_existing:
                stats["skipped"] += 1
                continue

            print(f"[{idx}/{len(files)}] {rel}", flush=True)

            source_for_stage_b = text

            node_paths: list[Path] = []
            if args.outline or args.nodes:
                outline_md = generate_with_retries(
                    engine=args.engine,
                    retries=args.retries,
                    sleep_s=args.retry_sleep,
                    prompt=prompts.outline + text,
                    max_tokens=args.outline_max_tokens,
                    ollama_url=args.ollama_url,
                    ollama_model=args.ollama_model,
                    gateway_url=args.gateway_url,
                    gateway_token=gateway_token,
                    agent_id=args.gateway_agent_id,
                )

                if args.validate:
                    probs = validate_outline(outline_md)
                    if probs:
                        outline_md = generate_with_retries(
                            engine=args.engine,
                            retries=0,
                            sleep_s=args.retry_sleep,
                            prompt=prompts.outline + text,
                            max_tokens=args.outline_max_tokens,
                            ollama_url=args.ollama_url,
                            ollama_model=args.ollama_model,
                            gateway_url=args.gateway_url,
                            gateway_token=gateway_token,
                            agent_id=args.gateway_agent_id,
                        )

                outline_out.write_text(outline_md + "\n", encoding="utf-8")
                source_for_stage_b = outline_md

                if args.nodes:
                    node_paths = shard_outline_to_nodes(
                        outline_md=outline_md,
                        nodes_dir=nodes_dir,
                        node_min_tokens=args.node_min_tokens,
                        node_max_tokens=args.node_max_tokens,
                        node_max_chars=args.node_max_chars,
                    )

            exec_md = ""
            sum_md = ""

            if args.only in ("all", "execution-notes"):
                exec_md = generate_with_retries(
                    engine=args.engine,
                    retries=args.retries,
                    sleep_s=args.retry_sleep,
                    prompt=prompts.execution_notes + source_for_stage_b,
                    max_tokens=args.exec_max_tokens,
                    ollama_url=args.ollama_url,
                    ollama_model=args.ollama_model,
                    gateway_url=args.gateway_url,
                    gateway_token=gateway_token,
                    agent_id=args.gateway_agent_id,
                )

            if args.only in ("all", "tool-summary"):
                sum_md = generate_with_retries(
                    engine=args.engine,
                    retries=args.retries,
                    sleep_s=args.retry_sleep,
                    prompt=prompts.tool_summary + source_for_stage_b,
                    max_tokens=args.summary_max_tokens,
                    ollama_url=args.ollama_url,
                    ollama_model=args.ollama_model,
                    gateway_url=args.gateway_url,
                    gateway_token=gateway_token,
                    agent_id=args.gateway_agent_id,
                )

            # Validation + one extra retry if requested
            if args.validate:
                if sum_md:
                    probs = validate_tool_summary(sum_md)
                    if probs:
                        sum_md = generate_with_retries(
                            engine=args.engine,
                            retries=0,
                            sleep_s=args.retry_sleep,
                            prompt=prompts.tool_summary + source_for_stage_b,
                            max_tokens=args.summary_max_tokens,
                            ollama_url=args.ollama_url,
                            ollama_model=args.ollama_model,
                            gateway_url=args.gateway_url,
                            gateway_token=gateway_token,
                            agent_id=args.gateway_agent_id,
                        )
                if exec_md:
                    probs = validate_execution_notes(exec_md)
                    if probs:
                        exec_md = generate_with_retries(
                            engine=args.engine,
                            retries=0,
                            sleep_s=args.retry_sleep,
                            prompt=prompts.execution_notes + source_for_stage_b,
                            max_tokens=args.exec_max_tokens,
                            ollama_url=args.ollama_url,
                            ollama_model=args.ollama_model,
                            gateway_url=args.gateway_url,
                            gateway_token=gateway_token,
                            agent_id=args.gateway_agent_id,
                        )

            if exec_md:
                exec_out.write_text(exec_md.strip() + "\n", encoding="utf-8")
            if sum_md:
                sum_out.write_text(sum_md.strip() + "\n", encoding="utf-8")

            if args.nodes:
                write_doc_index(
                    doc_index_path=doc_index,
                    source_path=p,
                    tool_summary=sum_out,
                    execution_notes=exec_out,
                    outline_path=outline_out if outline_out.exists() else None,
                    node_paths=node_paths,
                )
                doc_indices.append(doc_index)

            cache.setdefault("files", {})[rel] = {
                "hash": text_hash,
                "engine": args.engine,
                "ollama_model": args.ollama_model,
                "outline": bool(args.outline),
                "nodes": bool(args.nodes),
                "ts": time.time(),
            }
            save_cache(cache_path, cache)

            stats["ok"] += 1

            if args.sleep_ms:
                time.sleep(args.sleep_ms / 1000.0)

        except Exception as e:
            stats["errors"] += 1
            msg = f"ERROR {rel}: {e!r}"
            print(msg, file=sys.stderr, flush=True)
            try:
                log_error(msg)
            except Exception:
                pass
            if args.fail_fast or stats["errors"] >= args.max_errors:
                break

    # Root index is generated when --nodes is enabled.
    if args.nodes:
        try:
            write_root_index(out_root=out_root, input_root=input_root, doc_indices=doc_indices)
        except Exception as e:
            log_error(f"ERROR write_root_index: {e!r}")

    stats["durationSeconds"] = round(time.time() - start, 2)
    run_stats_path.parent.mkdir(parents=True, exist_ok=True)
    run_stats_path.write_text(json.dumps(stats, indent=2) + "\n", encoding="utf-8")

    # If this was `run`, perform indexing as step 2.
    if should_index_after and stats["errors"] == 0:
        try:
            try:
                from .vector_index import index_distilled_dir
            except ImportError:  # pragma: no cover
                from raglite.vector_index import index_distilled_dir

            def _parse_csv_set(s: str) -> set[str] | None:
                parts = [p.strip() for p in (s or "").split(",") if p.strip()]
                return set(parts) if parts else None

            index_cache_path = (out_root / ".raglite" / "index_cache.json") if bool(run_condense_args.skip_indexed) else None

            index_res = index_distilled_dir(
                distilled_root=out_root,
                chroma_url=run_condense_args.chroma_url,
                collection=run_condense_args.collection,
                ollama_url=run_condense_args.ollama_url,
                embed_model=run_condense_args.embed_model,
                embed_max_chars=run_condense_args.embed_max_chars,
                include_outlines=bool(run_condense_args.include_outlines),
                include_kinds=_parse_csv_set(run_condense_args.include_kinds),
                exclude_kinds=_parse_csv_set(run_condense_args.exclude_kinds),
                index_cache_path=index_cache_path,
                skip_indexed=bool(run_condense_args.skip_indexed),
            )
            # attach indexing stats to run_stats.json for visibility
            stats["index"] = index_res
            run_stats_path.write_text(json.dumps(stats, indent=2) + "\n", encoding="utf-8")
        except Exception as e:
            print(f"ERROR: indexing failed: {e!r}", file=sys.stderr)
            return 1
    elif should_index_after and stats["errors"] != 0:
        print("NOTE: skipping indexing because condense reported errors", file=sys.stderr)

    return 0 if stats["errors"] == 0 else 1


if __name__ == "__main__":
    raise SystemExit(main())
