#!/usr/bin/env python3
"""
meshdb_agent.py - MeshPOP Distributed Search Engine v3.0
Full-peer model: every server can search the entire mesh.

Local commands:
    meshdb_agent.py index [path]       Index a directory (default: $HOME)
    meshdb_agent.py index-all          Auto-discover and index all projects
    meshdb_agent.py index-full [home]  Deep recursive scan of entire home
    meshdb_agent.py search <query>     Search LOCAL database
    meshdb_agent.py find <query>       Filename search LOCAL
    meshdb_agent.py read <query>       Read best matching document LOCAL
    meshdb_agent.py status             Show LOCAL database statistics

Distributed commands (new in v3):
    meshdb_agent.py search-all <query>   Search ALL servers in parallel
    meshdb_agent.py find-all <query>     Filename search ALL servers
    meshdb_agent.py status-all           Status of ALL servers
    meshdb_agent.py read-remote <server> <path>  Read file from specific server

JSON output:
    meshdb_agent.py json <any-command>   JSON output mode
"""

import os
import sys
import json
import time
import hashlib
import sqlite3
import socket
import platform
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
# Semantic search imports
try:
    import numpy as np
    HAS_NUMPY = True
except ImportError:
    HAS_NUMPY = False


VERSION = "3.4.0"
DB_DIR = os.path.expanduser("~/.mpop")
DB_PATH = os.path.join(DB_DIR, "meshdb.db")
MAX_CONTENT = 500_000  # 500KB per file
MAX_FILE_SIZE = 2_000_000  # 2MB - skip larger files

# ==================== Distributed Config ====================

# Config paths: try user home first, then common locations
MPOP_CONFIG_PATHS = [
    os.path.expanduser("~/.mpop/config.json"),
    "/home/dragon/.mpop/config.json",
    "/home/dell/.mpop/config.json",
    "/Users/dragon/.mpop/config.json",
    "/root/.mpop/config.json",
]
VSSH_PORT = 48291
AGENT_PATH_LINUX = "/home/dragon/.local/bin/meshdb_agent.py"
AGENT_PATH_MAC = "/Users/dragon/.local/bin/meshdb_agent.py"
MESHDB_SERVERS = ["m1", "g1", "g2", "g3", "g4", "d1", "d2"]

def _load_mpop_config():
    for path in MPOP_CONFIG_PATHS:
        try:
            with open(path, 'r') as f:
                config = json.load(f)
                if config.get('servers'):
                    return config
        except Exception:
            continue
    return {}

def _get_server_ips():
    config = _load_mpop_config()
    servers = config.get("servers", {})
    return {name: srv.get("ip", "") for name, srv in servers.items()}

def _get_vssh_secret():
    config = _load_mpop_config()
    return config.get("vssh_secret", "")

def _get_local_hostname():
    hostname = socket.gethostname()
    if "mac" in hostname.lower() or platform.system() == "Darwin":
        return "m1"
    # Try matching against config
    config = _load_mpop_config()
    servers = config.get("servers", {})
    local_ips = _get_local_ips()
    for name, srv in servers.items():
        if srv.get("ip") in local_ips:
            return name
    # Fallback: match by hostname prefix
    for name in servers:
        if hostname.startswith(name) or name in hostname.lower():
            return name
    return hostname

def _get_local_ips():
    """Get all local IP addresses."""
    ips = set(["127.0.0.1"])
    try:
        for info in socket.getaddrinfo(socket.gethostname(), None):
            ips.add(info[4][0])
    except Exception:
        pass
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        ips.add(s.getsockname()[0])
        s.close()
    except Exception:
        pass
    return ips


# ==================== VSSH Protocol ====================

def vssh_exec(ip, cmd, secret, timeout=15):
    """Execute command on remote server via vssh protocol."""
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(timeout)
        sock.connect((ip, VSSH_PORT))
        sock.sendall(f"SSH:{secret}:{cmd}\n".encode())

        resp = sock.recv(64)
        if not resp.startswith(b'OK'):
            sock.close()
            return None

        data = b''
        while True:
            chunk = sock.recv(8192)
            if not chunk:
                break
            data += chunk
            if b'\x04' in chunk:
                data = data.replace(b'\x04', b'')
                break

        sock.close()
        return data.decode('utf-8', errors='replace').strip()
    except Exception:
        return None


def remote_exec_json(server, cmd, timeout=15):
    """Execute meshdb_agent.py json <cmd> on remote server, return parsed JSON."""
    server_ips = _get_server_ips()
    secret = _get_vssh_secret()

    ip = server_ips.get(server, "")
    if not ip:
        return {"error": f"Unknown server: {server}"}
    if not secret:
        return {"error": "vssh_secret not configured"}

    agent_path = AGENT_PATH_MAC if server == "m1" else AGENT_PATH_LINUX
    full_cmd = f"python3 {agent_path} json {cmd}"

    result = vssh_exec(ip, full_cmd, secret, timeout=timeout)
    if result is None:
        return {"error": f"vssh connection failed to {server}"}

    try:
        if '__END__' in result:
            result = result[:result.index('__END__')].strip()
        # Find JSON start
        for i, ch in enumerate(result):
            if ch in ('{', '['):
                return json.loads(result[i:])
        return {"error": "No JSON in response", "raw": result[:300]}
    except json.JSONDecodeError as e:
        return {"error": f"JSON parse error: {e}", "raw": result[:300]}


# ==================== Multi-Server Operations ====================

def multi_search(query, servers=None, limit=10, doc_type=None, source_dir=None):
    """Search across all servers in parallel. Full peer model."""
    if servers is None:
        servers = MESHDB_SERVERS.copy()

    local_server = _get_local_hostname()
    all_results = []
    server_stats = {}
    t0 = time.time()

    def search_one(srv):
        if srv == local_server:
            res = api_search(query, limit=limit, doc_type=doc_type, source_dir=source_dir)
        else:
            res = remote_exec_json(srv, f'search "{query}"', timeout=15)
            # Post-filter for remote (agent doesn't support --type/--dir in json mode yet)
            if "results" in res and (doc_type or source_dir):
                filtered = res["results"]
                if doc_type:
                    filtered = [r for r in filtered if r.get("doc_type") == doc_type]
                if source_dir:
                    filtered = [r for r in filtered 
                               if source_dir in r.get("source_dir", "") 
                               or source_dir in r.get("filepath", "")]
                res["results"] = filtered[:limit]
                res["count"] = len(res["results"])
        return srv, res

    with ThreadPoolExecutor(max_workers=min(len(servers), 8)) as executor:
        futures = {executor.submit(search_one, srv): srv for srv in servers}
        try:
            for future in as_completed(futures, timeout=20):
                srv = futures[future]
                try:
                    srv_name, result = future.result(timeout=5)
                    if "error" in result:
                        server_stats[srv_name] = {"count": 0, "error": result["error"]}
                    else:
                        count = result.get("count", 0)
                        elapsed = result.get("elapsed_ms", 0)
                        server_stats[srv_name] = {"count": count, "elapsed_ms": elapsed}
                        for r in result.get("results", [])[:limit]:
                            r["server"] = srv_name
                            all_results.append(r)
                except Exception as e:
                    server_stats[srv] = {"count": 0, "error": str(e)}
        except TimeoutError:
            for fut, srv in futures.items():
                if srv not in server_stats:
                    server_stats[srv] = {"count": 0, "error": "timeout"}

    # Sort by BM25 rank (lower = better)
    all_results.sort(key=lambda x: x.get("rank", 0))
    total_count = sum(s.get("count", 0) for s in server_stats.values())
    total_elapsed = (time.time() - t0) * 1000

    return {
        "query": query,
        "total_results": total_count,
        "returned": len(all_results[:limit * 3]),
        "elapsed_ms": round(total_elapsed, 1),
        "results": all_results[:limit * 3],
        "servers": server_stats
    }


def multi_find(query, servers=None, limit=10):
    """Find files by name across all servers in parallel."""
    if servers is None:
        servers = MESHDB_SERVERS.copy()

    local_server = _get_local_hostname()
    all_results = []
    server_stats = {}
    t0 = time.time()

    def find_one(srv):
        if srv == local_server:
            res = api_find(query, limit=limit)
        else:
            res = remote_exec_json(srv, f'find "{query}"', timeout=10)
        return srv, res

    with ThreadPoolExecutor(max_workers=min(len(servers), 8)) as executor:
        futures = {executor.submit(find_one, srv): srv for srv in servers}
        try:
            for future in as_completed(futures, timeout=15):
                srv = futures[future]
                try:
                    srv_name, result = future.result(timeout=5)
                    if "error" in result:
                        server_stats[srv_name] = {"count": 0, "error": result["error"]}
                    else:
                        count = result.get("count", 0)
                        server_stats[srv_name] = {"count": count}
                        for r in result.get("results", [])[:limit]:
                            r["server"] = srv_name
                            all_results.append(r)
                except Exception as e:
                    server_stats[srv] = {"count": 0, "error": str(e)}
        except TimeoutError:
            for fut, srv in futures.items():
                if srv not in server_stats:
                    server_stats[srv] = {"count": 0, "error": "timeout"}

    total_elapsed = (time.time() - t0) * 1000
    return {
        "query": query,
        "total_results": sum(s.get("count", 0) for s in server_stats.values()),
        "elapsed_ms": round(total_elapsed, 1),
        "results": all_results,
        "servers": server_stats
    }


def multi_status(servers=None):
    """Get meshdb status from all servers in parallel."""
    if servers is None:
        servers = MESHDB_SERVERS.copy()

    local_server = _get_local_hostname()
    all_status = {}
    t0 = time.time()

    def get_one(srv):
        if srv == local_server:
            return srv, api_status()
        else:
            return srv, remote_exec_json(srv, "status", timeout=10)

    with ThreadPoolExecutor(max_workers=min(len(servers), 8)) as executor:
        futures = {executor.submit(get_one, srv): srv for srv in servers}
        try:
            for future in as_completed(futures, timeout=20):
                srv = futures[future]
                try:
                    srv_name, result = future.result(timeout=5)
                    all_status[srv_name] = result
                except Exception as e:
                    all_status[srv] = {"error": str(e)}
        except TimeoutError:
            for fut, srv in futures.items():
                if srv not in all_status:
                    all_status[srv] = {"error": "timeout"}

    total_docs = sum(
        s.get("total_documents", 0) for s in all_status.values()
        if isinstance(s, dict) and "error" not in s
    )
    total_elapsed = (time.time() - t0) * 1000
    return {
        "total_documents": total_docs,
        "elapsed_ms": round(total_elapsed, 1),
        "servers": all_status
    }


def remote_read(server, filepath):
    """Read a specific file from a specific server."""
    local_server = _get_local_hostname()
    if server == local_server:
        return api_read(filepath)
    else:
        return remote_exec_json(server, f'read "{filepath}"', timeout=10)


# ==================== File Type Detection ====================

TYPE_MAP = {
    ".py": "python", ".js": "javascript", ".ts": "typescript",
    ".jsx": "react", ".tsx": "react", ".rs": "rust", ".go": "go",
    ".java": "java", ".rb": "ruby", ".php": "php", ".c": "c",
    ".cpp": "cpp", ".h": "header", ".hpp": "header",
    ".sh": "shell", ".bash": "shell", ".zsh": "shell", ".fish": "shell",
    ".md": "markdown", ".txt": "text", ".rst": "text",
    ".json": "json", ".yaml": "yaml", ".yml": "yaml", ".toml": "toml",
    ".xml": "xml", ".html": "html", ".css": "css", ".scss": "scss",
    ".sql": "sql", ".r": "r", ".R": "r",
    ".dockerfile": "docker", ".conf": "config", ".cfg": "config",
    ".ini": "config", ".env": "config", ".properties": "config",
    ".csv": "data", ".tsv": "data",
    ".tex": "latex", ".bib": "latex",
    ".vue": "vue", ".svelte": "svelte",
    ".swift": "swift", ".kt": "kotlin", ".scala": "scala",
    ".lua": "lua", ".perl": "perl", ".pl": "perl",
    ".nginx": "config", ".tf": "terraform", ".hcl": "terraform",
    ".log": "log",
}

INDEXABLE_EXTENSIONS = set(TYPE_MAP.keys()) | {
    ".gitignore", ".dockerignore", ".editorconfig",
    ".prettierrc", ".eslintrc", ".babelrc",
}

SKIP_DIRS = {
    "node_modules", ".git", "__pycache__", ".venv", "venv",
    ".tox", ".mypy_cache", ".pytest_cache", "dist", "build",
    ".next", ".nuxt", ".cache", ".parcel-cache",
    "target", ".cargo", "vendor", ".gradle",
    ".idea", ".vscode", ".DS_Store",
    "env", ".env", "eggs", "*.egg-info",
    ".terraform", ".serverless",
}

SKIP_HOME_DIRS = {
    "Library", "Applications", ".Trash", "Movies", "Music", "Pictures",
    "Downloads", "Desktop", "Documents", "Public",
    "공개", "다운로드", "문서", "바탕화면", "비디오", "사진", "서식", "음악",
    ".local", ".npm", ".cargo", ".rustup", ".gradle", ".m2",
    ".docker", ".kube", ".minikube", ".cache", ".gem",
    ".cpan", ".config", ".ssh", ".gnupg", "go",
    ".pyenv", ".nvm", ".rbenv", ".sdkman",
    ".mpop", ".vscode", ".cursor",
    "stanza_resources", "models", "open-webui-env",
    "executor_venv", "llama.cpp", "stable-diffusion-webui",
    "snap", "chromadb-data",
}

SHALLOW_SCAN_DIRS = {
    "corpus", "corpus_raw", "corpus_raw2", "corpus_backup", "corpus_fast",
    "corpus_catalog", "builds", "builds_wiki_full", "builds_wiki_mini",
    "wiki_ppmi", "wiki_ppmi_large", "wiki_ppmi_out",
    "ppmi", "ppmi_out", "ppmi_archive", "ppmi_all", "ppmi_backup", "ppmi_data",
    "cc100_data", "cc100_ppmi_out", "cc_news",
    "news_data", "news_pipeline",
    "wikidata", "wikidata_output", "wikipedia",
}


def detect_type(filepath):
    basename = os.path.basename(filepath).lower()
    if basename == "dockerfile":
        return "docker"
    if basename == "makefile":
        return "makefile"
    if basename == "jenkinsfile":
        return "jenkins"
    if basename in (".gitignore", ".dockerignore"):
        return "config"
    ext = os.path.splitext(filepath)[1].lower()
    return TYPE_MAP.get(ext, "unknown")


def file_hash(filepath):
    try:
        h = hashlib.md5()
        with open(filepath, "rb") as f:
            for chunk in iter(lambda: f.read(8192), b""):
                h.update(chunk)
        return h.hexdigest()
    except Exception:
        return None


def read_file_content(filepath, max_length=None):
    max_length = max_length or MAX_CONTENT
    try:
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            return f.read(max_length)
    except Exception:
        return ""


def format_size(size):
    for unit in ["B", "KB", "MB", "GB"]:
        if size < 1024:
            return f"{size:.1f}{unit}"
        size /= 1024
    return f"{size:.1f}TB"


# ==================== Database ====================

def init_db(db_path=None):
    db_path = db_path or DB_PATH
    os.makedirs(os.path.dirname(db_path), exist_ok=True)
    conn = sqlite3.connect(db_path)
    conn.row_factory = sqlite3.Row
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=10000")

    conn.execute("""
        CREATE TABLE IF NOT EXISTS documents (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            filepath TEXT UNIQUE,
            filename TEXT,
            content TEXT,
            doc_type TEXT,
            file_size INTEGER,
            file_hash TEXT,
            source_dir TEXT,
            hostname TEXT,
            indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            modified_at TIMESTAMP
        )
    """)

    # FTS5 full-text search (optional - graceful fallback to LIKE search)
    has_fts5 = False
    try:
        conn.execute("""
            CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
                filename, content, doc_type,
                content='documents',
                content_rowid='id',
                tokenize='unicode61'
            )
        """)
        has_fts5 = True
    except sqlite3.OperationalError:
        pass

    if has_fts5:
        # Create sync triggers only when FTS5 table exists
        for trigger_sql in [
            """CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
                INSERT INTO documents_fts(rowid, filename, content, doc_type)
                VALUES (new.id, new.filename, new.content, new.doc_type);
            END""",
            """CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
                INSERT INTO documents_fts(documents_fts, rowid, filename, content, doc_type)
                VALUES ('delete', old.id, old.filename, old.content, old.doc_type);
            END""",
            """CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
                INSERT INTO documents_fts(documents_fts, rowid, filename, content, doc_type)
                VALUES ('delete', old.id, old.filename, old.content, old.doc_type);
                INSERT INTO documents_fts(rowid, filename, content, doc_type)
                VALUES (new.id, new.filename, new.content, new.doc_type);
            END""",
        ]:
            try:
                conn.execute(trigger_sql)
            except sqlite3.OperationalError:
                pass
    else:
        # No FTS5: drop any stale triggers that reference documents_fts
        for tname in ["documents_ai", "documents_ad", "documents_au"]:
            try:
                conn.execute(f"DROP TRIGGER IF EXISTS {tname}")
            except sqlite3.OperationalError:
                pass
        # Also drop stale FTS table if it somehow exists but is broken
        try:
            conn.execute("DROP TABLE IF EXISTS documents_fts")
        except sqlite3.OperationalError:
            pass


    # Semantic search: embeddings table
    conn.execute("""CREATE TABLE IF NOT EXISTS embeddings (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        doc_id INTEGER,
        chunk_text TEXT,
        chunk_start_line INTEGER,
        chunk_end_line INTEGER,
        embedding BLOB,
        model TEXT DEFAULT 'nomic-embed-text',
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        FOREIGN KEY (doc_id) REFERENCES documents(id) ON DELETE CASCADE
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_embeddings_doc ON embeddings(doc_id)")

    conn.commit()
    return conn


# ==================== Scanning ====================

def should_skip_dir(dirname):
    return dirname in SKIP_DIRS or dirname.startswith(".")


def should_index_file(filepath):
    ext = os.path.splitext(filepath)[1].lower()
    basename = os.path.basename(filepath).lower()
    if basename in ("dockerfile", "makefile", "jenkinsfile", ".gitignore", ".dockerignore"):
        return True
    return ext in INDEXABLE_EXTENSIONS


def scan_directory(dirpath):
    files = []
    for root, dirs, filenames in os.walk(dirpath):
        dirs[:] = [d for d in dirs if not should_skip_dir(d)]
        for fname in filenames:
            fpath = os.path.join(root, fname)
            if should_index_file(fpath):
                try:
                    sz = os.path.getsize(fpath)
                    if sz <= MAX_FILE_SIZE:
                        files.append(fpath)
                except OSError:
                    pass
    return files


def scan_directory_shallow(dirpath):
    files = []
    try:
        for fname in os.listdir(dirpath):
            fpath = os.path.join(dirpath, fname)
            if os.path.isfile(fpath) and should_index_file(fpath):
                try:
                    sz = os.path.getsize(fpath)
                    if sz <= MAX_FILE_SIZE:
                        files.append(fpath)
                except OSError:
                    pass
    except PermissionError:
        pass
    return files


def scan_home_full(home_dir):
    home_dir = os.path.abspath(os.path.expanduser(home_dir))
    all_files = []

    for fname in os.listdir(home_dir):
        fpath = os.path.join(home_dir, fname)
        if os.path.isfile(fpath) and should_index_file(fpath):
            try:
                sz = os.path.getsize(fpath)
                if sz <= MAX_FILE_SIZE:
                    all_files.append(fpath)
            except OSError:
                pass

    for entry in os.listdir(home_dir):
        subpath = os.path.join(home_dir, entry)
        if not os.path.isdir(subpath):
            continue
        if entry.startswith(".") or entry in SKIP_HOME_DIRS:
            continue
        if entry in SHALLOW_SCAN_DIRS:
            shallow = scan_directory_shallow(subpath)
            all_files.extend(shallow)
        else:
            deep = scan_directory(subpath)
            all_files.extend(deep)

    return all_files


# ==================== Indexing ====================

def index_files(conn, files, source_dir, hostname=None):
    hostname = hostname or platform.node()
    stats = {"total": len(files), "new": 0, "updated": 0, "skipped": 0, "errors": 0}

    for fpath in files:
        try:
            fhash = file_hash(fpath)
            existing = conn.execute(
                "SELECT file_hash FROM documents WHERE filepath = ?", (fpath,)
            ).fetchone()
            if existing and existing["file_hash"] == fhash:
                stats["skipped"] += 1
                continue

            content = read_file_content(fpath)
            fstat = os.stat(fpath)
            doc_type = detect_type(fpath)
            filename = os.path.basename(fpath)
            is_update = existing is not None

            if is_update:
                conn.execute("""
                    UPDATE documents SET filename=?, content=?, doc_type=?,
                    file_size=?, file_hash=?, source_dir=?, hostname=?,
                    indexed_at=CURRENT_TIMESTAMP, modified_at=?
                    WHERE filepath=?
                """, (filename, content, doc_type, fstat.st_size, fhash,
                      source_dir, hostname,
                      datetime.fromtimestamp(fstat.st_mtime).isoformat(),
                      fpath))
                stats["updated"] += 1
            else:
                conn.execute("""
                    INSERT INTO documents (filepath, filename, content, doc_type,
                    file_size, file_hash, source_dir, hostname, modified_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                """, (fpath, filename, content, doc_type, fstat.st_size, fhash,
                      source_dir, hostname,
                      datetime.fromtimestamp(fstat.st_mtime).isoformat()))
                stats["new"] += 1

            if (stats["new"] + stats["updated"]) % 500 == 0:
                conn.commit()

        except Exception as e:
            stats["errors"] += 1
            if stats["errors"] <= 5:
                stats.setdefault("error_samples", [])
                stats["error_samples"].append(f"{os.path.basename(fpath)}: {e}")

    conn.commit()
    return stats


def index_directory(conn, dirpath, replace=False):
    hostname = platform.node()
    dirpath = os.path.abspath(os.path.expanduser(dirpath))
    if not os.path.isdir(dirpath):
        return {"error": f"Not a directory: {dirpath}"}
    files = scan_directory(dirpath)
    return index_files(conn, files, dirpath, hostname)


# ==================== Search ====================

def search(conn, query, limit=20, doc_type=None, source_dir=None):
    where_clauses = []
    params = []
    if doc_type:
        where_clauses.append("d.doc_type = ?")
        params.append(doc_type)
    if source_dir:
        where_clauses.append("d.source_dir LIKE ?")
        params.append(f"%{source_dir}%")
    where_sql = (" AND " + " AND ".join(where_clauses)) if where_clauses else ""

    sql = f"""
        SELECT d.filepath, d.filename, d.doc_type, d.file_size, d.source_dir,
               d.content,
               bm25(documents_fts, 10.0, 5.0, 1.0) as rank,
               snippet(documents_fts, 1, '>>>', '<<<', '...', 40) as snippet
        FROM documents_fts fts
        JOIN documents d ON d.id = fts.rowid
        WHERE documents_fts MATCH ?
        {where_sql}
        ORDER BY rank
        LIMIT ?
    """
    params = [query] + params + [limit]

    try:
        rows = conn.execute(sql, params).fetchall()
        return [dict(r) for r in rows]
    except sqlite3.OperationalError:
        return search_like(conn, query, limit, doc_type, source_dir)


def search_like(conn, query, limit=20, doc_type=None, source_dir=None):
    where_clauses = ["(content LIKE ? OR filename LIKE ?)"]
    params = [f"%{query}%", f"%{query}%"]
    if doc_type:
        where_clauses.append("doc_type = ?")
        params.append(doc_type)
    if source_dir:
        where_clauses.append("source_dir LIKE ?")
        params.append(f"%{source_dir}%")
    where_sql = " AND ".join(where_clauses)
    sql = f"SELECT * FROM documents WHERE {where_sql} ORDER BY file_size ASC LIMIT ?"
    params.append(limit)
    rows = conn.execute(sql, params).fetchall()
    results = []
    for r in rows:
        d = dict(r)
        # Generate snippet around matching text for LIKE fallback
        content_text = d.get("content", "")
        query_lower = query.lower()
        pos = content_text.lower().find(query_lower)
        if pos >= 0:
            start = max(0, pos - 80)
            end = min(len(content_text), pos + len(query) + 80)
            snip = content_text[start:end].replace("\n", " ")
            if start > 0:
                snip = "..." + snip
            if end < len(content_text):
                snip = snip + "..."
            d["snippet"] = snip
        else:
            d["snippet"] = content_text[:200]
        results.append(d)
    return results


def search_filename(conn, query, limit=10):
    rows = conn.execute(
        "SELECT * FROM documents WHERE filename LIKE ? ORDER BY file_size ASC LIMIT ?",
        (f"%{query}%", limit)
    ).fetchall()
    return [dict(r) for r in rows]


def read_doc(conn, query):
    row = conn.execute(
        "SELECT * FROM documents WHERE filepath LIKE ? OR filename LIKE ? LIMIT 1",
        (f"%{query}%", f"%{query}%")
    ).fetchone()
    return dict(row) if row else None


def get_status(conn):
    total = conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0]
    types = conn.execute(
        "SELECT doc_type, COUNT(*) as cnt FROM documents GROUP BY doc_type ORDER BY cnt DESC"
    ).fetchall()
    dirs = conn.execute(
        "SELECT source_dir, COUNT(*) as cnt FROM documents GROUP BY source_dir ORDER BY cnt DESC"
    ).fetchall()
    db_size = os.path.getsize(DB_PATH) if os.path.exists(DB_PATH) else 0
    return {
        "total_documents": total,
        "database_size": format_size(db_size),
        "database_path": DB_PATH,
        "hostname": platform.node(),
        "types": {r[0]: r[1] for r in types},
        "directories": {r[0]: r[1] for r in dirs},
    }


# ==================== Auto-discover Projects ====================

def discover_projects(home_dir=None):
    home_dir = home_dir or os.path.expanduser("~")
    markers = {".git", "package.json", "Cargo.toml", "go.mod", "setup.py",
               "pyproject.toml", "Makefile", "CMakeLists.txt", "pom.xml",
               "build.gradle", "Gemfile", "requirements.txt", "docker-compose.yml"}
    projects = []
    for entry in os.scandir(home_dir):
        if not entry.is_dir() or entry.name in SKIP_HOME_DIRS or entry.name.startswith("."):
            continue
        try:
            children = set(os.listdir(entry.path))
            if children & markers:
                projects.append(entry.path)
            else:
                for sub in os.scandir(entry.path):
                    if sub.is_dir() and not sub.name.startswith(".") and sub.name not in SKIP_DIRS:
                        try:
                            sub_children = set(os.listdir(sub.path))
                            if sub_children & markers:
                                projects.append(sub.path)
                        except PermissionError:
                            pass
        except PermissionError:
            pass
    return sorted(projects)




# ==================== Semantic Search ====================

OLLAMA_URLS = [
    "http://10.99.74.111:11434",   # g1
    "http://10.99.106.230:11434",  # g2
    "http://10.99.202.68:11434",   # g3
    "http://10.99.175.0:11434",    # g4
    "http://localhost:11434",       # local
]

EMBED_MODEL = "nomic-embed-text"
CHUNK_LINES = 30
CHUNK_OVERLAP = 10

def _get_ollama_url():
    """Find first available Ollama instance."""
    import urllib.request
    for url in OLLAMA_URLS:
        try:
            req = urllib.request.Request(f"{url}/api/tags", method="GET")
            resp = urllib.request.urlopen(req, timeout=3)
            if resp.status == 200:
                return url
        except:
            continue
    return None

def _get_embedding(text, ollama_url=None):
    """Get embedding vector from Ollama."""
    import urllib.request
    if not ollama_url:
        ollama_url = _get_ollama_url()
    if not ollama_url:
        return None
    
    data = json.dumps({"model": EMBED_MODEL, "prompt": text[:2000]}).encode()
    req = urllib.request.Request(
        f"{ollama_url}/api/embeddings",
        data=data,
        headers={"Content-Type": "application/json"},
        method="POST"
    )
    try:
        resp = urllib.request.urlopen(req, timeout=30)
        result = json.loads(resp.read())
        return result.get("embedding")
    except Exception as e:
        return None

def _chunk_content(content, chunk_lines=CHUNK_LINES, overlap=CHUNK_OVERLAP):
    """Split content into overlapping chunks by lines."""
    lines = content.split("\n")
    chunks = []
    i = 0
    while i < len(lines):
        end = min(i + chunk_lines, len(lines))
        chunk_text = "\n".join(lines[i:end])
        if chunk_text.strip():  # Skip empty chunks
            chunks.append({
                "text": chunk_text,
                "start_line": i + 1,
                "end_line": end
            })
        i += chunk_lines - overlap
    return chunks

def embed_document(conn, doc_id, ollama_url=None):
    """Generate embeddings for a single document's chunks."""
    if not HAS_NUMPY:
        return {"error": "numpy not installed"}
    
    row = conn.execute("SELECT id, filepath, content, doc_type FROM documents WHERE id = ?", (doc_id,)).fetchone()
    if not row:
        return {"error": f"Document {doc_id} not found"}
    
    doc = dict(row)
    content_text = doc.get("content", "")
    if not content_text or len(content_text) < 20:
        return {"skipped": True, "reason": "too short"}
    
    # Skip binary-looking content
    if "\x00" in content_text[:100]:
        return {"skipped": True, "reason": "binary"}
    
    # Skip minified/bundled files (very long lines = likely bundled)
    first_lines = content_text[:5000].split("\n")
    if first_lines and max(len(l) for l in first_lines[:5]) > 1000:
        return {"skipped": True, "reason": "minified/bundled"}
    
    # Remove old embeddings for this doc
    conn.execute("DELETE FROM embeddings WHERE doc_id = ?", (doc_id,))
    
    chunks = _chunk_content(content_text)
    # For very large files, limit to first 50 chunks (1500 lines)
    if len(chunks) > 50:
        chunks = chunks[:50]
    embedded = 0
    
    for chunk in chunks:
        vec = _get_embedding(chunk["text"], ollama_url)
        if vec:
            blob = np.array(vec, dtype=np.float32).tobytes()
            conn.execute(
                "INSERT INTO embeddings (doc_id, chunk_text, chunk_start_line, chunk_end_line, embedding, model) VALUES (?,?,?,?,?,?)",
                (doc_id, chunk["text"][:500], chunk["start_line"], chunk["end_line"], blob, EMBED_MODEL)
            )
            embedded += 1
    
    conn.commit()
    return {"doc_id": doc_id, "filepath": doc["filepath"], "chunks": len(chunks), "embedded": embedded}

def embed_all(conn, doc_type=None, limit=100, ollama_url=None):
    """Generate embeddings for documents that don't have them yet."""
    if not HAS_NUMPY:
        return {"error": "numpy not installed"}
    
    if not ollama_url:
        ollama_url = _get_ollama_url()
    if not ollama_url:
        return {"error": "No Ollama instance available"}
    
    # Find docs without embeddings
    where = ""
    params = []
    if doc_type:
        where = "AND d.doc_type = ?"
        params.append(doc_type)
    
    sql = f"""
        SELECT d.id, d.filepath, d.doc_type, d.file_size
        FROM documents d
        LEFT JOIN (SELECT DISTINCT doc_id FROM embeddings) e ON d.id = e.doc_id
        WHERE e.doc_id IS NULL AND d.file_size > 50 AND d.file_size < 2000000
        AND d.filepath NOT LIKE '%/site-packages/%'
        AND d.filepath NOT LIKE '%/node_modules/%'
        AND d.filepath NOT LIKE '%/chat_dist/%'
        AND d.filepath NOT LIKE '%/.venv/%'
        AND d.filepath NOT LIKE '%/venv/%'
        AND d.filepath NOT LIKE '%/oi-env/%'
        AND d.filepath NOT LIKE '%package-lock.json'
        AND d.filepath NOT LIKE '%/corpus/%'
        AND d.filepath NOT LIKE '%/gazetteers/%'
        AND d.filepath NOT LIKE '%/pmi_final/%'
        AND d.filepath NOT LIKE '%.min.js'
        AND d.filepath NOT LIKE '%.min.css'
        {where}
        ORDER BY 
            CASE WHEN d.filepath LIKE '%MeshPOP%' THEN 0 ELSE 1 END,
            d.file_size DESC
        LIMIT ?
    """
    params.append(limit)
    
    rows = conn.execute(sql, params).fetchall()
    
    # Filter out non-essential paths
    skip_patterns = ["/site-packages/", "/node_modules/", "/.venv/", "/venv/", "/oi-env/",
                     "/__pycache__/", "/.git/", "/dist/", "/build/", "/.tox/",
                     "/chat_dist/", "/bundle/", ".min.js", ".min.css",
                     "package-lock.json", "yarn.lock", "Cargo.lock",
                     "/corpus/", "/corpora/", "/gazetteers/", "/pmi_final/"]
    filtered = [r for r in rows if not any(p in dict(r)["filepath"] for p in skip_patterns)]
    
    results = {"total": len(filtered), "embedded": 0, "chunks": 0, "errors": 0, "ollama": ollama_url}
    
    for row in filtered:
        doc = dict(row)
        try:
            r = embed_document(conn, doc["id"], ollama_url)
            if r.get("embedded", 0) > 0:
                results["embedded"] += 1
                results["chunks"] += r["embedded"]
        except Exception as e:
            results["errors"] += 1
            if results["errors"] <= 3:
                results["first_errors"] = results.get("first_errors", [])
                results["first_errors"].append(f"{doc.get('filepath','?')[-50:]}: {str(e)[:80]}")
    
    return results

def semantic_search(conn, query, limit=10, doc_type=None):
    """Search by meaning using cosine similarity."""
    if not HAS_NUMPY:
        return {"error": "numpy not installed"}
    
    # Get query embedding
    query_vec = _get_embedding(query)
    if not query_vec:
        return {"error": "Failed to get query embedding (no Ollama available)"}
    
    query_np = np.array(query_vec, dtype=np.float32)
    query_norm = np.linalg.norm(query_np)
    if query_norm == 0:
        return {"error": "Zero query vector"}
    
    # Load all embeddings
    where = ""
    params = []
    if doc_type:
        where = "WHERE d.doc_type = ?"
        params.append(doc_type)
    
    sql = f"""
        SELECT e.id, e.doc_id, e.chunk_text, e.chunk_start_line, e.chunk_end_line,
               e.embedding, d.filepath, d.filename, d.doc_type
        FROM embeddings e
        JOIN documents d ON d.id = e.doc_id
        {where}
    """
    
    rows = conn.execute(sql, params).fetchall()
    if not rows:
        return {"error": "No embeddings found. Run 'embed' first.", "count": 0}
    
    # Calculate cosine similarities
    results = []
    dim = len(query_vec)
    
    for row in rows:
        r = dict(row)
        blob = r["embedding"]
        doc_vec = np.frombuffer(blob, dtype=np.float32)
        if len(doc_vec) != dim:
            continue
        
        similarity = float(np.dot(query_np, doc_vec) / (query_norm * np.linalg.norm(doc_vec) + 1e-10))
        results.append({
            "filepath": r["filepath"],
            "filename": r["filename"],
            "doc_type": r["doc_type"],
            "chunk_text": r["chunk_text"],
            "start_line": r["chunk_start_line"],
            "end_line": r["chunk_end_line"],
            "similarity": round(similarity, 4)
        })
    
    # Sort by similarity descending
    results.sort(key=lambda x: x["similarity"], reverse=True)
    
    return {
        "query": query,
        "total_embeddings": len(rows),
        "count": min(limit, len(results)),
        "results": results[:limit]
    }

def embed_status(conn):
    """Get embedding statistics."""
    total_docs = conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0]
    embedded_docs = conn.execute("SELECT COUNT(DISTINCT doc_id) FROM embeddings").fetchone()[0]
    total_chunks = conn.execute("SELECT COUNT(*) FROM embeddings").fetchone()[0]
    
    # By type
    by_type = {}
    for r in conn.execute("""
        SELECT d.doc_type, COUNT(DISTINCT e.doc_id) as docs, COUNT(e.id) as chunks
        FROM embeddings e JOIN documents d ON d.id = e.doc_id
        GROUP BY d.doc_type ORDER BY docs DESC
    """):
        by_type[r[0]] = {"docs": r[1], "chunks": r[2]}
    
    return {
        "total_documents": total_docs,
        "embedded_documents": embedded_docs,
        "total_chunks": total_chunks,
        "coverage": f"{embedded_docs}/{total_docs} ({round(embedded_docs/total_docs*100,1)}%)" if total_docs > 0 else "0/0",
        "by_type": by_type
    }


# ==================== API Layer ====================


def _find_snippet_line(content, snippet_text):
    """Find the approximate line number of a snippet within content."""
    if not content or not snippet_text:
        return None
    
    # Clean snippet markers and take first meaningful segment
    clean = snippet_text.replace(">>>", "").replace("<<<", "")
    # Take first non-ellipsis segment
    segments = [s.strip() for s in clean.split("...") if s.strip()]
    if not segments:
        return None
    
    search_text = segments[0][:60]  # First 60 chars of first segment
    pos = content.find(search_text)
    if pos < 0:
        # Try case-insensitive
        pos = content.lower().find(search_text.lower())
    if pos < 0:
        return None
    
    line_num = content[:pos].count("\n") + 1
    return line_num

def api_search(query, limit=20, doc_type=None, source_dir=None, db_path=None):
    conn = init_db(db_path)
    t0 = time.time()
    results = search(conn, query, limit, doc_type, source_dir)
    elapsed = (time.time() - t0) * 1000
    conn.close()
    return {"query": query, "count": len(results), "elapsed_ms": round(elapsed, 1),
            "results": [_format_search_result(r) for r in results]}

def _format_search_result(r):
    snippet = r.get("snippet") or (r.get("content") or "")[:200]
    line = _find_snippet_line(r.get("content", ""), snippet)
    result = {
        "filepath": r["filepath"], "filename": r["filename"],
        "doc_type": r["doc_type"], "file_size": r.get("file_size", 0),
        "rank": r.get("rank", 0),
        "snippet": snippet
    }
    if line:
        result["line"] = line
    return result


def api_index(dirpath, replace=False, db_path=None):
    conn = init_db(db_path)
    t0 = time.time()
    stats = index_directory(conn, dirpath, replace)
    elapsed = time.time() - t0
    conn.close()
    if "error" in stats:
        stats["elapsed_seconds"] = round(elapsed, 1)
        stats["directory"] = dirpath
        stats.setdefault("total", 0)
        stats.setdefault("new", 0)
        stats.setdefault("updated", 0)
        stats.setdefault("skipped", 0)
        stats.setdefault("errors", 0)
        return stats
    stats["elapsed_seconds"] = round(elapsed, 1)
    stats["directory"] = dirpath
    return stats


def api_find(query, limit=10, db_path=None):
    conn = init_db(db_path)
    results = search_filename(conn, query, limit)
    conn.close()
    return {"query": query, "count": len(results),
            "results": [{"filepath": r["filepath"], "filename": r["filename"],
                         "doc_type": r["doc_type"], "file_size": r.get("file_size", 0)}
                        for r in results]}


def api_read(query, db_path=None):
    conn = init_db(db_path)
    doc = read_doc(conn, query)
    conn.close()
    if doc:
        return {"filepath": doc["filepath"], "filename": doc["filename"],
                "doc_type": doc["doc_type"], "content": doc.get("content", ""),
                "file_size": doc.get("file_size", 0)}
    return {"error": f"No document found matching: {query}"}


def api_status(db_path=None):
    conn = init_db(db_path)
    status = get_status(conn)
    conn.close()
    return status



def api_embed(doc_type=None, limit=100, db_path=None):
    conn = init_db(db_path)
    result = embed_all(conn, doc_type, limit)
    conn.close()
    return result

def api_semantic_search(query, limit=10, doc_type=None, db_path=None):
    conn = init_db(db_path)
    t0 = time.time()
    result = semantic_search(conn, query, limit, doc_type)
    result["elapsed_ms"] = round((time.time() - t0) * 1000, 1)
    conn.close()
    return result

def api_embed_status(db_path=None):
    conn = init_db(db_path)
    result = embed_status(conn)
    conn.close()
    return result


def api_index_all(home_dir=None, replace=False, db_path=None):
    projects = discover_projects(home_dir)
    conn = init_db(db_path)
    results = []
    total_stats = {"total": 0, "new": 0, "updated": 0, "skipped": 0, "errors": 0}
    for proj in projects:
        t0 = time.time()
        stats = index_directory(conn, proj, replace)
        elapsed = time.time() - t0
        stats["directory"] = proj
        stats["elapsed_seconds"] = round(elapsed, 1)
        results.append(stats)
        for k in total_stats:
            total_stats[k] += stats.get(k, 0)
        print(f"  [{len(results):3d}/{len(projects)}] {proj} "
              f"({stats.get('new',0)} new, {stats.get('updated',0)} updated, "
              f"{elapsed:.1f}s)")
    conn.close()
    return {"projects": len(projects), "totals": total_stats, "details": results}


def api_index_full(home_dir=None, db_path=None):
    home_dir = home_dir or os.path.expanduser("~")
    print(f"  Scanning {home_dir} (full deep scan)...")
    t0 = time.time()
    files = scan_home_full(home_dir)
    scan_time = time.time() - t0
    print(f"  Found {len(files)} files in {scan_time:.1f}s")
    conn = init_db(db_path)
    hostname = platform.node()
    groups = {}
    for fpath in files:
        rel = os.path.relpath(fpath, home_dir)
        parts = rel.split(os.sep)
        source = os.path.join(home_dir, parts[0]) if len(parts) > 1 else home_dir
        groups.setdefault(source, []).append(fpath)
    total_stats = {"total": 0, "new": 0, "updated": 0, "skipped": 0, "errors": 0}
    done = 0
    total_dirs = len(groups)
    for source_dir, group_files in sorted(groups.items()):
        done += 1
        t1 = time.time()
        stats = index_files(conn, group_files, source_dir, hostname)
        elapsed = time.time() - t1
        for k in total_stats:
            total_stats[k] += stats.get(k, 0)
        if stats.get("new", 0) > 0 or stats.get("updated", 0) > 0:
            print(f"  [{done:3d}/{total_dirs}] {source_dir} "
                  f"({len(group_files)} files, {stats.get('new',0)} new, "
                  f"{stats.get('updated',0)} updated, {elapsed:.1f}s)")
    conn.close()
    total_elapsed = time.time() - t0
    total_stats["elapsed_seconds"] = round(total_elapsed, 1)
    total_stats["home_dir"] = home_dir
    total_stats["groups"] = total_dirs
    return total_stats


# ==================== CLI ====================

def parse_cli_options(args):
    opts = {"limit": 20, "type": None, "dir": None, "servers": None}
    query_parts = []
    i = 0
    while i < len(args):
        if args[i] == "--limit" and i + 1 < len(args):
            opts["limit"] = int(args[i + 1])
            i += 2
        elif args[i] == "--type" and i + 1 < len(args):
            opts["type"] = args[i + 1]
            i += 2
        elif args[i] == "--dir" and i + 1 < len(args):
            opts["dir"] = args[i + 1]
            i += 2
        elif args[i] == "--servers" and i + 1 < len(args):
            opts["servers"] = [s.strip() for s in args[i + 1].split(",")]
            i += 2
        elif args[i].startswith("--"):
            i += 1
        else:
            query_parts.append(args[i])
            i += 1
    return " ".join(query_parts), opts


def main():
    args = sys.argv[1:]
    if not args:
        local = _get_local_hostname()
        print(f"meshdb v{VERSION} - MeshPOP Distributed Search Engine")
        print(f"Host: {platform.node()} ({local})")
        print(f"DB: {DB_PATH}")
        print(f"Mesh: {', '.join(MESHDB_SERVERS)}")
        print()
        print("Local commands:")
        print("  search <query>     Search local database")
        print("  find <query>       Filename search local")
        print("  read <query>       Read matching document")
        print("  status             Local database statistics")
        print("  index [path]       Index a single directory")
        print("  index-all          Auto-discover and index projects")
        print("  index-full [home]  Deep scan entire home directory")
        print()
        print("Distributed commands:")
        print("  search-all <query>             Search ALL servers")
        print("  find-all <query>               Filename search ALL servers")
        print("  status-all                     Status of ALL servers")
        print("  read-remote <server> <path>    Read file from server")
        print()
        print("Options: --limit N  --type T  --dir D  --servers s1,s2")
        print("JSON:    json <any-command>")
        return

    cmd = args[0]
    json_mode = False

    if cmd == "json":
        json_mode = True
        args = args[1:]
        cmd = args[0] if args else "status"

    # ---- LOCAL COMMANDS ----

    if cmd == "index":
        dirpath = args[1] if len(args) > 1 else os.path.expanduser("~")
        replace = "--replace" in args
        print(f"Indexing: {dirpath}")
        result = api_index(dirpath, replace)
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"  Total: {result['total']} files")
            print(f"  New: {result['new']}, Updated: {result['updated']}, "
                  f"Skipped: {result['skipped']}, Errors: {result['errors']}")
            print(f"  Time: {result['elapsed_seconds']}s")

    elif cmd == "index-all":
        home_dir = args[1] if len(args) > 1 else None
        replace = "--replace" in args
        print(f"Auto-discovering projects in {home_dir or '~'}...")
        result = api_index_all(home_dir, replace)
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            t = result["totals"]
            print(f"\nDone! {result['projects']} projects, "
                  f"{t['total']} files ({t['new']} new, {t['updated']} updated, "
                  f"{t['errors']} errors)")

    elif cmd == "index-full":
        home_dir = args[1] if len(args) > 1 else None
        print(f"Full deep scan of {home_dir or '~'}...")
        result = api_index_full(home_dir)
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"\nDone! {result['total']} files in {result['groups']} directories")
            print(f"  New: {result['new']}, Updated: {result['updated']}, "
                  f"Skipped: {result['skipped']}, Errors: {result['errors']}")
            print(f"  Time: {result['elapsed_seconds']}s")

    elif cmd == "search":
        query, opts = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py search <query> [--limit N] [--type T] [--dir D]")
            return
        result = api_search(query, limit=opts["limit"], doc_type=opts["type"], source_dir=opts["dir"])
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"Search: '{query}' ({result['count']} results, {result['elapsed_ms']}ms)\n")
            for i, r in enumerate(result["results"], 1):
                print(f"  {i}. [{r['doc_type']}] {r['filepath']}")
                snippet = r.get("snippet", "").strip().replace("\n", " ")[:120]
                if snippet:
                    print(f"     {snippet}")

    elif cmd == "find":
        query, opts = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py find <query> [--limit N]")
            return
        result = api_find(query, limit=opts["limit"])
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"Find: '{query}' ({result['count']} results)\n")
            for i, r in enumerate(result["results"], 1):
                print(f"  {i}. [{r['doc_type']}] {r['filepath']}")

    elif cmd == "read":
        query, _ = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py read <filepath>")
            return
        result = api_read(query)
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            if "error" in result:
                print(f"  {result['error']}")
            else:
                print(f"File: {result['filepath']} [{result['doc_type']}]")
                print("=" * 60)
                print(result.get("content", "")[:5000])

    elif cmd == "status":
        result = api_status()
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"meshdb v{VERSION} (local)")
            print(f"  Host: {result['hostname']}")
            print(f"  DB: {result['database_path']} ({result['database_size']})")
            print(f"  Documents: {result['total_documents']}")
            print(f"\n  Types:")
            for dtype, cnt in sorted(result["types"].items(), key=lambda x: -x[1])[:15]:
                print(f"    {dtype:<15} {cnt:>6}")
            print(f"\n  Directories ({len(result['directories'])}):")
            for d, cnt in sorted(result["directories"].items(), key=lambda x: -x[1])[:15]:
                print(f"    {d:<50} {cnt:>6}")

    # ---- DISTRIBUTED COMMANDS ----

    elif cmd == "search-all":
        query, opts = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py search-all <query> [--limit N] [--type T] [--servers s1,s2]")
            return
        result = multi_search(query, servers=opts["servers"], limit=opts["limit"],
                             doc_type=opts["type"], source_dir=opts["dir"])
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"Search-All: '{query}' ({result['total_results']} results, "
                  f"{result['elapsed_ms']}ms across {len(result['servers'])} servers)\n")
            # Server summary
            for srv, stat in sorted(result["servers"].items()):
                if "error" in stat:
                    print(f"  {srv}: ERROR - {stat['error']}")
                else:
                    ms = stat.get('elapsed_ms', '?')
                    print(f"  {srv}: {stat['count']} results ({ms}ms)")
            print()
            # Results
            for i, r in enumerate(result["results"], 1):
                srv = r.get("server", "?")
                print(f"  {i}. [{srv}] [{r['doc_type']}] {r['filepath']}")
                snippet = r.get("snippet", "").strip().replace("\n", " ")[:120]
                if snippet:
                    print(f"     {snippet}")

    elif cmd == "find-all":
        query, opts = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py find-all <query> [--limit N] [--servers s1,s2]")
            return
        result = multi_find(query, servers=opts["servers"], limit=opts["limit"])
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"Find-All: '{query}' ({result['total_results']} results, "
                  f"{result['elapsed_ms']}ms)\n")
            for srv, stat in sorted(result["servers"].items()):
                if "error" in stat:
                    print(f"  {srv}: ERROR - {stat['error']}")
                else:
                    print(f"  {srv}: {stat['count']} files")
            print()
            for i, r in enumerate(result["results"], 1):
                srv = r.get("server", "?")
                print(f"  {i}. [{srv}] [{r['doc_type']}] {r['filepath']}")

    elif cmd == "status-all":
        _, opts = parse_cli_options(args[1:])
        result = multi_status(servers=opts["servers"])
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            print(f"meshdb v{VERSION} - Mesh Status ({result['elapsed_ms']}ms)")
            print(f"  Total documents: {result['total_documents']:,}")
            print()
            for srv in MESHDB_SERVERS:
                stat = result["servers"].get(srv, {})
                if "error" in stat:
                    print(f"  {srv}: ERROR - {stat['error']}")
                else:
                    docs = stat.get("total_documents", 0)
                    size = stat.get("database_size", "?")
                    print(f"  {srv}: {docs:>7,} docs ({size})")

    elif cmd == "read-remote":
        if len(args) < 3:
            print("Usage: meshdb_agent.py read-remote <server> <filepath>")
            return
        server = args[1]
        filepath = " ".join(args[2:])
        result = remote_read(server, filepath)
        if json_mode:
            print(json.dumps(result, indent=2))
        else:
            if "error" in result:
                print(f"  {result['error']}")
            else:
                print(f"File: {result['filepath']} [{result['doc_type']}] @ {server}")
                print("=" * 60)
                print(result.get("content", "")[:5000])

    elif cmd == "embed":
        query, opts = parse_cli_options(args[1:])
        result = api_embed(opts["type"], opts["limit"])
        if json_mode:
            print(json.dumps(result, indent=2, ensure_ascii=False))
        else:
            if "error" in result:
                print(f"Error: {result['error']}")
            else:
                print(f"Embedded: {result['embedded']}/{result['total']} docs, {result['chunks']} chunks")
                print(f"Ollama: {result.get('ollama', 'N/A')}")
                if result.get('errors', 0) > 0:
                    print(f"Errors: {result['errors']}")

    elif cmd == "semantic" or cmd == "sem":
        query, opts = parse_cli_options(args[1:])
        if not query:
            print("Usage: meshdb_agent.py semantic <query> [--type TYPE] [--limit N]")
            sys.exit(1)
        result = api_semantic_search(query, opts["limit"], opts["type"])
        if json_mode:
            print(json.dumps(result, indent=2, ensure_ascii=False))
        else:
            if "error" in result:
                print(f"Error: {result['error']}")
            else:
                print(f"Semantic search: \"{query}\" ({result['elapsed_ms']}ms, {result['total_embeddings']} embeddings)")
                print(f"Results: {result['count']}")
                for i, r in enumerate(result.get('results', []), 1):
                    sim = r['similarity']
                    bar = '#' * int(sim * 20)
                    print(f"\n  [{i}] {r['filepath']}:{r['start_line']}-{r['end_line']} (sim: {sim:.3f}) {bar}")
                    snippet = r.get('chunk_text', '').strip().replace('\n', ' ')[:120]
                    if snippet:
                        print(f"      {snippet}")

    elif cmd == "embed-status":
        result = api_embed_status()
        if json_mode:
            print(json.dumps(result, indent=2, ensure_ascii=False))
        else:
            print(f"Embeddings: {result['coverage']}")
            print(f"Total chunks: {result['total_chunks']}")
            for t, info in result.get('by_type', {}).items():
                print(f"  {t}: {info['docs']} docs, {info['chunks']} chunks")

    else:
        print(f"Unknown command: {cmd}")
        print("Commands: search, find, read, status, index, index-all, index-full")
        print("          search-all, find-all, status-all, read-remote")
        print("          embed, semantic (sem), embed-status")


if __name__ == "__main__":
    main()
