#!/usr/bin/env python3
"""
meshdb - MeshPOP Search Engine

SQLite FTS5-based full-text search. File contents are NOT stored in DB.
Supports three interfaces: CLI, Library API, and MCP (for mpop integration).

Interfaces:
  1. CLI:     meshdb search [--smart] <query>
  2. Library: import meshdb; meshdb.api_search(query)
  3. MCP:     meshdb_mcp.py (mpop MCP server integration)

CLI Usage:
    meshdb index <path>              # Index a directory
    meshdb search <query>            # Full-text search
    meshdb search --smart <query>    # LLM-expanded keyword search
    meshdb find <filename>           # Filename search
    meshdb read <path>               # Read file content
    meshdb status                    # DB status
    meshdb reindex <path>            # Re-index (delete + rebuild)

Options:
    --json       Output as JSON (for programmatic use)
    --type X     Filter by type: docs|code|config|log
    --limit N    Max results (default: 20)

Environment:
    MESHDB_DB           DB path (default: ~/.mpop/meshdb.db)
    MESHDB_OLLAMA_URL   Ollama endpoint for --smart (default: http://localhost:11434)
    MESHDB_OLLAMA_MODEL Ollama model for --smart (default: qwen2:7b-instruct)
"""

import sqlite3
import os
import sys
import time
import hashlib
import json as _json
from pathlib import Path
from datetime import datetime

# ═══════════════════════════════════════════════════════════
#  Configuration
# ═══════════════════════════════════════════════════════════

DB_PATH = os.environ.get("MESHDB_DB", os.path.expanduser("~/.mpop/meshdb.db"))
MAX_FILE_SIZE = 10 * 1024 * 1024    # 10MB
MAX_CONTENT_LENGTH = 500_000         # 500K chars
PREVIEW_LENGTH = 1024                # 1KB preview

# LLM settings (CLI --smart only; MCP uses Claude directly)
OLLAMA_URL = os.environ.get("MESHDB_OLLAMA_URL", "http://localhost:11434")
OLLAMA_MODEL = os.environ.get("MESHDB_OLLAMA_MODEL", "qwen2:7b-instruct")
SMART_TIMEOUT = 10  # seconds

# Indexable file extensions
TEXT_EXTENSIONS = {
    '.md', '.txt', '.rst', '.org', '.adoc',
    '.py', '.rs', '.js', '.ts', '.jsx', '.tsx',
    '.go', '.java', '.c', '.cpp', '.h', '.hpp',
    '.rb', '.php', '.swift', '.kt', '.scala',
    '.sh', '.bash', '.zsh', '.fish',
    '.sql', '.r', '.lua', '.vim',
    '.html', '.css', '.scss', '.less',
    '.json', '.yaml', '.yml', '.toml', '.ini', '.cfg',
    '.env', '.conf', '.properties',
    '.xml', '.plist',
    '.csv', '.tsv', '.log',
    '.gitignore', '.dockerignore', '.editorconfig',
}

# Directories to skip during indexing
SKIP_DIRS = {
    '.git', '.svn', '.hg',
    'node_modules', '__pycache__', '.mypy_cache', '.pytest_cache',
    '.tox', '.eggs', '*.egg-info',
    'venv', '.venv', 'env',
    '.next', '.nuxt', 'dist', 'build',
    'target', '.cargo', '.DS_Store',
}

# Remote server configuration (for cross-server search)
REMOTE_TIMEOUT = 30  # seconds for remote operations

# Server registry: server -> list of home directories to index
# Configure in ~/.mpop/config.json under "meshdb_servers":
#   { "meshdb_servers": { "myserver": ["/home"], "local": ["~"] } }
def _load_server_registry() -> dict:
    config_path = os.path.expanduser('~/.mpop/config.json')
    try:
        import json
        with open(config_path) as f:
            cfg = json.load(f)
        reg = cfg.get('meshdb_servers')
        if isinstance(reg, dict):
            return {k: [os.path.expanduser(p) for p in v] for k, v in reg.items()}
    except Exception:
        pass
    return {}

SERVER_REGISTRY = _load_server_registry()

# Directories to skip during remote scan (system/data-only/env dirs)
SKIP_HOME_DIRS = {
    'snap', 'bin', 'logs', '__pycache__', 'stanza_resources', 'nltk_data',
    'Desktop', 'Downloads', 'Documents', 'Pictures', 'Videos', 'Music', 'Public',
    'Library', 'Applications', 'Movies',
    # Korean system dirs
    'Public', 'Downloads', 'Documents', 'Desktop', 'Videos', 'Photos', 'Templates', 'Music',
    # Virtual envs and build artifacts
    'venv', '.venv', 'env', 'open-webui-env', 'edge-tts-env', 'executor_venv',
    'radio_api_env', 'oi-env',
    # Large data dirs (data only, no scripts)
    'SynologyDrive', 'SynologyTM', 'synology_mount', 'mnt',
    'chromadb-data', 'models',
    # macOS specific
    'iCloud Drive(Archive)',
}


TYPE_ICONS = {'docs': '\U0001f4c4', 'code': '\U0001f4bb',
              'config': '\u2699\ufe0f', 'log': '\U0001f4cb'}


# ═══════════════════════════════════════════════════════════
#  i18n - Language Pack System
# ═══════════════════════════════════════════════════════════

# Embedded English strings
_EN_STRINGS = {
    "cli": {
        "search_result": "search results",
        "no_results": "no results found",
        "search_time": "search time",
        "smart_expanding": "smart mode: expanding keywords",
        "expanded": "expanded",
        "indexing": "indexing", "reindexing": "reindexing",
        "scanned": "scanned", "files": "files",
        "new": "new", "updated": "updated", "moved": "moved",
        "skipped": "skipped", "errors": "errors", "time": "time",
        "filename_search": "filename search",
        "not_found": "not found", "unreadable": "unreadable",
        "status": "status", "docs": "documents", "embedded": "embedded",
        "db_size": "DB size", "last_indexed": "last indexed",
        "by_type": "by type", "by_source": "by source",
        "unknown_cmd": "unknown command",
    },
    "types": {"docs": "docs", "code": "code", "config": "config", "log": "log"},
}


def t(key):
    """Return English string for key."""
    parts = key.split(".", 1)
    if len(parts) == 2:
        return _EN_STRINGS.get(parts[0], {}).get(parts[1], key)
    return _EN_STRINGS.get(key, key)


def detect_type(filepath):
    """Detect file type from extension."""
    ext = Path(filepath).suffix.lower()
    name = Path(filepath).name.lower()
    if ext in {'.md', '.txt', '.rst', '.org', '.adoc'}:
        return 'docs'
    if ext in {'.log'}:
        return 'log'
    if ext in {'.json', '.yaml', '.yml', '.toml', '.ini', '.cfg',
               '.conf', '.env', '.properties', '.xml', '.plist'}:
        return 'config'
    if name in {'makefile', 'dockerfile', 'docker-compose.yml',
                '.gitignore', '.dockerignore', '.editorconfig'}:
        return 'config'
    return 'code'


def file_hash(filepath):
    """Compute fast hash using head+tail+size."""
    try:
        size = os.path.getsize(filepath)
        with open(filepath, 'rb') as f:
            head = f.read(4096)
            if size > 8192:
                f.seek(-4096, 2)
            tail = f.read(4096)
        return hashlib.md5(head + tail + str(size).encode()).hexdigest()
    except Exception:
        return None


def read_file_content(filepath, max_length=None):
    """Read file with multi-encoding fallback."""
    max_length = max_length or MAX_CONTENT_LENGTH
    for enc in ['utf-8', 'euc-kr', 'cp949', 'latin-1']:
        try:
            with open(filepath, 'r', encoding=enc) as f:
                return f.read(max_length)
        except (UnicodeDecodeError, UnicodeError):
            continue
    return None


def format_size(size):
    """Human-readable file size."""
    for unit in ['B', 'KB', 'MB', 'GB']:
        if size < 1024:
            return f"{size:.1f}{unit}"
        size /= 1024
    return f"{size:.1f}TB"


def init_db(db_path=None):
    """Initialize SQLite database with FTS5 index."""
    path = db_path or DB_PATH
    os.makedirs(os.path.dirname(path), exist_ok=True)
    conn = sqlite3.connect(path)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS docs (
            id TEXT PRIMARY KEY, path TEXT NOT NULL, filename TEXT NOT NULL,
            preview TEXT, type TEXT, size INTEGER, file_hash TEXT,
            mtime REAL, indexed_at REAL, source_dir TEXT, embedding BLOB
        );
        CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(path);
        CREATE INDEX IF NOT EXISTS idx_docs_type ON docs(type);
        CREATE INDEX IF NOT EXISTS idx_docs_source ON docs(source_dir);
        CREATE INDEX IF NOT EXISTS idx_docs_hash ON docs(file_hash);
    """)
    try:
        conn.execute("""
            CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
                filename, content, type,
                tokenize='unicode61 remove_diacritics 2'
            );
        """)
    except sqlite3.OperationalError:
        pass
    conn.commit()
    return conn


# ─── Indexing ─────────────────────────────────────────────

def should_skip_dir(dirname):
    return dirname in SKIP_DIRS or dirname.startswith('.')

def should_index_file(filepath):
    path = Path(filepath)
    if path.suffix.lower() not in TEXT_EXTENSIONS:
        if path.name.lower() not in {'makefile', 'dockerfile', 'procfile',
                                       'gemfile', 'rakefile', 'vagrantfile'}:
            return False
    try:
        size = os.path.getsize(filepath)
        return 0 < size <= MAX_FILE_SIZE
    except OSError:
        return False

def scan_directory(dirpath):
    """Walk directory and collect indexable files."""
    files = []
    dirpath = os.path.abspath(os.path.expanduser(dirpath))
    for root, dirs, filenames in os.walk(dirpath):
        dirs[:] = [d for d in dirs if not should_skip_dir(d)]
        for fname in filenames:
            fp = os.path.join(root, fname)
            if should_index_file(fp):
                files.append(fp)
    return files

def index_directory(conn, dirpath, replace=False):
    """Index all text files in a directory. Returns stats dict."""
    dirpath = os.path.abspath(os.path.expanduser(dirpath))

    if replace:
        cursor = conn.execute(
            "SELECT rowid FROM docs WHERE source_dir = ?", (dirpath,))
        rowids = [r[0] for r in cursor]
        if rowids:
            ph = ','.join('?' * len(rowids))
            conn.execute(f"DELETE FROM search_index WHERE rowid IN ({ph})", rowids)
        conn.execute("DELETE FROM docs WHERE source_dir = ?", (dirpath,))
        conn.commit()

    # Build existing hash map for change detection
    existing = {}
    hash_to_path = {}
    for row in conn.execute(
        "SELECT path, file_hash, mtime, rowid FROM docs WHERE source_dir = ?", (dirpath,)):
        existing[row[0]] = {'hash': row[1], 'mtime': row[2], 'rowid': row[3]}
        if row[1]:
            hash_to_path[row[1]] = row[0]

    files = scan_directory(dirpath)
    new_c = upd_c = skip_c = mov_c = err_c = 0
    now = time.time()

    for fp in files:
        try:
            stat = os.stat(fp)
            fh = file_hash(fp)

            # Skip unchanged files
            if fp in existing:
                ex = existing[fp]
                if ex['hash'] == fh and ex['mtime'] == stat.st_mtime:
                    skip_c += 1; continue

            content = read_file_content(fp)
            if content is None:
                err_c += 1; continue

            fname = os.path.basename(fp)
            ftype = detect_type(fp)
            doc_id = hashlib.md5(fp.encode()).hexdigest()
            preview = content[:PREVIEW_LENGTH]

            if fp in existing:
                # Update: content changed
                conn.execute("""
                    UPDATE docs SET filename=?, preview=?, type=?, size=?,
                        file_hash=?, mtime=?, indexed_at=?
                    WHERE path=? AND source_dir=?
                """, (fname, preview, ftype, stat.st_size,
                      fh, stat.st_mtime, now, fp, dirpath))
                row = conn.execute(
                    "SELECT rowid FROM docs WHERE path=? AND source_dir=?",
                    (fp, dirpath)).fetchone()
                if row:
                    conn.execute("DELETE FROM search_index WHERE rowid=?", (row[0],))
                    conn.execute(
                        "INSERT INTO search_index(rowid,filename,content,type) VALUES(?,?,?,?)",
                        (row[0], fname, content, ftype))
                upd_c += 1

            elif fh and fh in hash_to_path:
                # File moved: same hash, different path
                old_path = hash_to_path[fh]
                old_info = existing.get(old_path)
                if old_info and not os.path.exists(old_path):
                    nid = hashlib.md5(fp.encode()).hexdigest()
                    conn.execute("""
                        UPDATE docs SET id=?, path=?, filename=?, preview=?,
                            mtime=?, indexed_at=?
                        WHERE path=? AND source_dir=?
                    """, (nid, fp, fname, preview, stat.st_mtime, now,
                          old_path, dirpath))
                    rid = old_info['rowid']
                    conn.execute("DELETE FROM search_index WHERE rowid=?", (rid,))
                    conn.execute(
                        "INSERT INTO search_index(rowid,filename,content,type) VALUES(?,?,?,?)",
                        (rid, fname, content, ftype))
                    mov_c += 1
                    hash_to_path[fh] = fp
                    existing[fp] = old_info
                    del existing[old_path]
                    continue

            # New document
            if fp not in existing:
                conn.execute("""
                    INSERT OR REPLACE INTO docs
                    (id,path,filename,preview,type,size,file_hash,mtime,indexed_at,source_dir)
                    VALUES(?,?,?,?,?,?,?,?,?,?)
                """, (doc_id, fp, fname, preview, ftype, stat.st_size,
                      fh, stat.st_mtime, now, dirpath))
                row = conn.execute("SELECT rowid FROM docs WHERE id=?", (doc_id,)).fetchone()
                if row:
                    conn.execute(
                        "INSERT INTO search_index(rowid,filename,content,type) VALUES(?,?,?,?)",
                        (row[0], fname, content, ftype))
                new_c += 1
        except Exception:
            err_c += 1

    conn.commit()
    return {'total_scanned': len(files), 'new': new_c, 'updated': upd_c,
            'moved': mov_c, 'skipped': skip_c, 'errors': err_c,
            'source_dir': dirpath}


# ─── Search ──────────────────────────────────────────────

def search(conn, query, limit=20, doc_type=None, source_dir=None):
    """BM25-based full-text search. Weights: filename 10x, content 5x, type 1x."""
    params = [query]
    sql = """
        SELECT d.path, d.filename, d.type, d.size, d.mtime,
            snippet(search_index, 1, '>>>', '<<<', '...', 64) as snippet,
            bm25(search_index, 10.0, 5.0, 1.0) as rank
        FROM search_index
        JOIN docs d ON d.rowid = search_index.rowid
        WHERE search_index MATCH ?
    """
    if doc_type:
        sql += " AND d.type = ?"; params.append(doc_type)
    if source_dir:
        sql += " AND d.source_dir = ?"
        params.append(os.path.abspath(os.path.expanduser(source_dir)))
    sql += " ORDER BY rank LIMIT ?"; params.append(limit)

    try:
        return [{'path': r[0], 'filename': r[1], 'type': r[2], 'size': r[3],
                 'mtime': datetime.fromtimestamp(r[4]).strftime('%Y-%m-%d %H:%M') if r[4] else '',
                 'snippet': r[5], 'score': abs(r[6])}
                for r in conn.execute(sql, params)]
    except sqlite3.OperationalError:
        return search_like(conn, query, limit, doc_type, source_dir)


def search_like(conn, query, limit=20, doc_type=None, source_dir=None):
    """LIKE fallback when FTS fails."""
    pattern = f"%{query}%"
    params = [pattern, pattern]
    sql = "SELECT path,filename,type,size,mtime,preview FROM docs WHERE (lower(filename) LIKE ? OR lower(preview) LIKE ?)"
    if doc_type:
        sql += " AND type = ?"; params.append(doc_type)
    if source_dir:
        sql += " AND source_dir = ?"
        params.append(os.path.abspath(os.path.expanduser(source_dir)))
    sql += " LIMIT ?"; params.append(limit)
    return [{'path': r[0], 'filename': r[1], 'type': r[2], 'size': r[3],
             'mtime': datetime.fromtimestamp(r[4]).strftime('%Y-%m-%d %H:%M') if r[4] else '',
             'snippet': (r[5] or '')[:200], 'score': 1.0}
            for r in conn.execute(sql, params)]


def search_filename(conn, query, limit=10):
    """Search by filename pattern."""
    return [{'path': r[0], 'filename': r[1], 'type': r[2], 'size': r[3],
             'mtime': datetime.fromtimestamp(r[4]).strftime('%Y-%m-%d %H:%M') if r[4] else ''}
            for r in conn.execute(
                "SELECT path,filename,type,size,mtime FROM docs WHERE lower(filename) LIKE lower(?) ORDER BY mtime DESC LIMIT ?",
                (f"%{query}%", limit))]


def read_doc(conn, query):
    """Read file content by path or filename."""
    if os.path.exists(query):
        return query, read_file_content(query)
    row = conn.execute(
        "SELECT path FROM docs WHERE path=? OR lower(filename) LIKE lower(?) LIMIT 1",
        (query, f"%{query}%")).fetchone()
    if row and os.path.exists(row[0]):
        return row[0], read_file_content(row[0])
    return None, None


def get_status(conn):
    """Get database statistics."""
    stats = {}
    stats['total_docs'] = conn.execute("SELECT COUNT(*) FROM docs").fetchone()[0]
    stats['by_type'] = dict(conn.execute(
        "SELECT type, COUNT(*) FROM docs GROUP BY type ORDER BY COUNT(*) DESC"))
    stats['by_source'] = dict(conn.execute(
        "SELECT source_dir, COUNT(*) FROM docs GROUP BY source_dir ORDER BY COUNT(*) DESC"))
    stats['embedded_docs'] = conn.execute(
        "SELECT COUNT(*) FROM docs WHERE embedding IS NOT NULL").fetchone()[0]
    db_path = conn.execute("PRAGMA database_list").fetchone()[2]
    if db_path and os.path.exists(db_path):
        stats['db_size'] = os.path.getsize(db_path)
        stats['db_size_human'] = format_size(stats['db_size'])
    row = conn.execute("SELECT MAX(indexed_at) FROM docs").fetchone()
    if row[0]:
        stats['last_indexed'] = datetime.fromtimestamp(row[0]).strftime('%Y-%m-%d %H:%M:%S')
    return stats


# ═══════════════════════════════════════════════════════════
#  API Layer (for MCP / Library - returns dicts)
# ═══════════════════════════════════════════════════════════

def api_search(query, limit=20, doc_type=None, source_dir=None, db_path=None):
    """Search API. Returns dict with results."""
    conn = init_db(db_path)
    t0 = time.time()
    results = search(conn, query, limit, doc_type, source_dir)
    elapsed_ms = round((time.time() - t0) * 1000)
    log_search(conn, query, len(results), elapsed_ms, 'search')
    conn.close()
    return {'query': query, 'results': results, 'count': len(results),
            'elapsed_ms': elapsed_ms}

def api_index(dirpath, replace=False, db_path=None):
    """Index API. Supports local and remote paths (server:path). Returns stats dict."""
    conn = init_db(db_path)
    t0 = time.time()
    server, path = parse_remote_path(dirpath)
    if server:
        result = index_remote(conn, server, path, replace)
    else:
        result = index_directory(conn, dirpath, replace)
    result['elapsed_s'] = round(time.time() - t0, 1)
    conn.close()
    return result

def api_find(query, limit=10, db_path=None):
    """Filename search API."""
    conn = init_db(db_path)
    results = search_filename(conn, query, limit)
    conn.close()
    return {'query': query, 'results': results, 'count': len(results)}

def api_read(query, db_path=None):
    """File read API."""
    conn = init_db(db_path)
    filepath, content = read_doc(conn, query)
    conn.close()
    if content:
        return {'path': filepath, 'content': content, 'size': len(content)}
    return {'error': f"'{query}' not found or unreadable"}

def api_status(db_path=None):
    """Status API."""
    conn = init_db(db_path)
    stats = get_status(conn)
    conn.close()
    return stats


def api_analytics(days=30, limit=20, db_path=None):
    """Search analytics API."""
    conn = init_db(db_path)
    stats = get_search_analytics(conn, days, limit)
    conn.close()
    return stats


# ═══════════════════════════════════════════════════════════
#  Full-Cluster Indexing (index-all)
# ═══════════════════════════════════════════════════════════

def discover_project_dirs(server, home_dirs):
    """Discover project directories on a server (1 level deep under home)."""
    import subprocess
    projects = []

    if server == 'm1':
        # Local: direct filesystem access
        for home in home_dirs:
            home = os.path.expanduser(home)
            if not os.path.isdir(home):
                continue
            for name in sorted(os.listdir(home)):
                if name.startswith('.') or name in SKIP_HOME_DIRS:
                    continue
                full = os.path.join(home, name)
                if os.path.isdir(full):
                    projects.append(full)
    else:
        # Remote: via mpop exec
        mpop_bin = os.path.expanduser("~/.local/bin/mpop")
        if not os.path.exists(mpop_bin):
            mpop_bin = os.path.expanduser("~/MeshPOP/mpop/mpop")

        for home in home_dirs:
            cmd = f'find {home} -maxdepth 1 -mindepth 1 -type d 2>/dev/null | sort'
            try:
                result = subprocess.run(
                    [mpop_bin, 'exec', '-t', server, cmd],
                    capture_output=True, text=True, timeout=REMOTE_TIMEOUT)
                for line in result.stdout.strip().split('\n'):
                    line = line.strip()
                    if not line or line.startswith('[') or line.startswith('-'):
                        continue
                    dirname = os.path.basename(line)
                    if dirname.startswith('.') or dirname in SKIP_HOME_DIRS:
                        continue
                    projects.append(line)
            except Exception:
                pass

    return projects


def api_index_all(servers=None, replace=False, db_path=None):
    """Index all project directories across all registered servers.
    Returns summary dict with per-server results.
    """
    import time as _time
    t0 = _time.time()
    target_servers = servers or list(SERVER_REGISTRY.keys())
    summary = {'servers': {}, 'total_projects': 0, 'total_files': 0,
               'total_new': 0, 'total_errors': 0}

    conn = init_db(db_path)

    for srv in target_servers:
        if srv not in SERVER_REGISTRY:
            summary['servers'][srv] = {'error': 'not in registry'}
            continue

        home_dirs = SERVER_REGISTRY[srv]
        projects = discover_project_dirs(srv, home_dirs)
        srv_result = {'projects': len(projects), 'files': 0, 'new': 0,
                      'updated': 0, 'skipped': 0, 'errors': 0, 'dirs': []}

        for proj_dir in projects:
            try:
                if srv == 'local':
                    # Local indexing
                    r = index_directory(conn, proj_dir, replace)
                else:
                    # Remote indexing
                    source_key = f"{srv}:{proj_dir}"
                    r = index_remote(conn, srv, proj_dir, replace)

                srv_result['files'] += r.get('total_scanned', 0)
                srv_result['new'] += r.get('new', 0)
                srv_result['updated'] += r.get('updated', 0)
                srv_result['skipped'] += r.get('skipped', 0)
                srv_result['errors'] += r.get('errors', 0)
                srv_result['dirs'].append({
                    'path': proj_dir if srv == 'local' else f"{srv}:{proj_dir}",
                    'files': r.get('total_scanned', 0),
                    'new': r.get('new', 0)
                })
            except Exception as e:
                srv_result['errors'] += 1
                srv_result['dirs'].append({'path': proj_dir, 'error': str(e)})

        summary['servers'][srv] = srv_result
        summary['total_projects'] += srv_result['projects']
        summary['total_files'] += srv_result['files']
        summary['total_new'] += srv_result['new']
        summary['total_errors'] += srv_result['errors']

    conn.close()
    summary['elapsed_s'] = round(_time.time() - t0, 1)
    return summary


# ═══════════════════════════════════════════════════════════
#  Remote Indexing (Cross-Server Search via mpop)
# ═══════════════════════════════════════════════════════════

def parse_remote_path(path_spec):
    """Parse 'server:path' notation. Returns (server, path) or (None, path)."""
    if ':' in path_spec and not path_spec.startswith('/'):
        parts = path_spec.split(':', 1)
        if len(parts[0]) <= 10 and parts[0].isalnum():
            return parts[0], parts[1]
    return None, path_spec


def remote_list_files(server, dirpath):
    """List indexable files on remote server via mpop exec."""
    import subprocess
    mpop_bin = os.path.expanduser("~/.local/bin/mpop")
    if not os.path.exists(mpop_bin):
        mpop_bin = os.path.expanduser("~/MeshPOP/mpop/mpop")

    # Use find + grep (avoids shell escaping issues with mpop exec)
    ext_re = r'\.(py|sh|bash|js|ts|jsx|tsx|md|txt|rst|json|yaml|yml|toml|ini|cfg|conf|env|html|css|scss|go|rs|java|c|cpp|h|hpp|rb|lua|sql|xml|csv|log|swift|kt)$'
    cmd = (
        f'find {dirpath} '
        f'-name ".git" -prune -o -name "node_modules" -prune -o '
        f'-name "__pycache__" -prune -o -name ".venv" -prune -o '
        f'-name "venv" -prune -o -name ".mypy_cache" -prune -o '
        f'-name "target" -prune -o -name "dist" -prune -o '
        f'-name ".tox" -prune -o -name ".eggs" -prune -o '
        f'-type f -print 2>/dev/null | '
        f'grep -E "{ext_re}" | head -5000'
    )

    try:
        result = subprocess.run(
            [mpop_bin, 'exec', '-t', server, cmd],
            capture_output=True, text=True, timeout=60)
        output = result.stdout.strip()
        if not output:
            return []
        files = []
        found_sep = False
        for line in output.split('\n'):
            raw = line.strip()
            if not raw:
                continue
            if not found_sep:
                if raw.startswith('---'):
                    found_sep = True
                continue
            if raw.startswith('/') and not raw.startswith('/bin/'):
                files.append(raw)
        return files
    except Exception:
        return []

def remote_read_file(server, filepath, max_bytes=None):
    """Read file content from remote server via mpop exec."""
    import subprocess
    mpop_bin = os.path.expanduser("~/.local/bin/mpop")
    if not os.path.exists(mpop_bin):
        mpop_bin = os.path.expanduser("~/MeshPOP/mpop/mpop")

    limit_cmd = f"head -c {max_bytes or MAX_CONTENT_LENGTH}" if max_bytes else f"head -c {MAX_CONTENT_LENGTH}"
    cmd = f'cat "{filepath}" 2>/dev/null | {limit_cmd}'

    try:
        result = subprocess.run(
            [mpop_bin, 'exec', '-t', server, cmd],
            capture_output=True, text=True, timeout=REMOTE_TIMEOUT)
        output = result.stdout
        # Strip mpop header: "[server] cmd..." and "---..." lines
        lines = output.split('\n')
        start = 0
        for i, line in enumerate(lines):
            stripped = line.strip()
            if stripped.startswith('[') or stripped.startswith('---'):
                start = i + 1
                continue
            break
        return '\n'.join(lines[start:])
    except Exception:
        return None


def remote_stat_file(server, filepath):
    """Get file size and mtime from remote server."""
    import subprocess
    mpop_bin = os.path.expanduser("~/.local/bin/mpop")
    if not os.path.exists(mpop_bin):
        mpop_bin = os.path.expanduser("~/MeshPOP/mpop/mpop")

    cmd = f'stat -c "%s %Y" "{filepath}" 2>/dev/null || stat -f "%z %m" "{filepath}" 2>/dev/null'
    try:
        result = subprocess.run(
            [mpop_bin, 'exec', '-t', server, cmd],
            capture_output=True, text=True, timeout=10)
        for line in result.stdout.strip().split('\n'):
            line = line.strip()
            if not line or line.startswith('[') or line.startswith('-'):
                continue
            if line and line[0].isdigit():
                parts = line.split()
                if len(parts) >= 2:
                    return int(parts[0]), float(parts[1])
    except Exception:
        pass
    return 0, 0.0


def index_remote(conn, server, dirpath, replace=False):
    """Index files from a remote server. Returns stats dict."""
    source_key = f"{server}:{dirpath}"

    if replace:
        cursor = conn.execute(
            "SELECT rowid FROM docs WHERE source_dir = ?", (source_key,))
        rowids = [r[0] for r in cursor]
        if rowids:
            ph = ','.join('?' * len(rowids))
            conn.execute(f"DELETE FROM search_index WHERE rowid IN ({ph})", rowids)
        conn.execute("DELETE FROM docs WHERE source_dir = ?", (source_key,))
        conn.commit()

    # Build existing hash map
    existing = {}
    for row in conn.execute(
        "SELECT path, file_hash, mtime, rowid FROM docs WHERE source_dir = ?", (source_key,)):
        existing[row[0]] = {'hash': row[1], 'mtime': row[2], 'rowid': row[3]}

    files = remote_list_files(server, dirpath)
    new_c = upd_c = skip_c = err_c = 0
    now = time.time()

    for fp in files:
        try:
            size, mtime = remote_stat_file(server, fp)
            remote_path = f"{server}:{fp}"

            # Skip unchanged
            if remote_path in existing:
                ex = existing[remote_path]
                if ex['mtime'] == mtime:
                    skip_c += 1
                    continue

            content = remote_read_file(server, fp)
            if not content:
                err_c += 1
                continue

            fname = os.path.basename(fp)
            ftype = detect_type(fp)
            doc_id = hashlib.md5(remote_path.encode()).hexdigest()
            preview = content[:PREVIEW_LENGTH]
            fh = hashlib.md5((content[:4096] + content[-4096:] + str(size)).encode()).hexdigest()

            if remote_path in existing:
                conn.execute("""
                    UPDATE docs SET filename=?, preview=?, type=?, size=?,
                        file_hash=?, mtime=?, indexed_at=?
                    WHERE path=? AND source_dir=?
                """, (fname, preview, ftype, size, fh, mtime, now, remote_path, source_key))
                row = conn.execute(
                    "SELECT rowid FROM docs WHERE path=? AND source_dir=?",
                    (remote_path, source_key)).fetchone()
                if row:
                    conn.execute("DELETE FROM search_index WHERE rowid=?", (row[0],))
                    conn.execute(
                        "INSERT INTO search_index(rowid,filename,content,type) VALUES(?,?,?,?)",
                        (row[0], fname, content, ftype))
                upd_c += 1
            else:
                conn.execute("""
                    INSERT OR REPLACE INTO docs
                    (id,path,filename,preview,type,size,file_hash,mtime,indexed_at,source_dir)
                    VALUES(?,?,?,?,?,?,?,?,?,?)
                """, (doc_id, remote_path, fname, preview, ftype, size, fh, mtime, now, source_key))
                row = conn.execute("SELECT rowid FROM docs WHERE id=?", (doc_id,)).fetchone()
                if row:
                    conn.execute(
                        "INSERT INTO search_index(rowid,filename,content,type) VALUES(?,?,?,?)",
                        (row[0], fname, content, ftype))
                new_c += 1
        except Exception:
            err_c += 1

    conn.commit()
    return {'total_scanned': len(files), 'new': new_c, 'updated': upd_c,
            'moved': 0, 'skipped': skip_c, 'errors': err_c,
            'source_dir': source_key}


# ═══════════════════════════════════════════════════════════
#  Search History & Analytics
# ═══════════════════════════════════════════════════════════

def log_search(conn, query, result_count, elapsed_ms, search_type='search'):
    """Record a search in history."""
    try:
        conn.execute(
            "INSERT INTO search_history(query,result_count,elapsed_ms,search_type,lang) VALUES(?,?,?,?,?)",
            (query, result_count, elapsed_ms, search_type))
        conn.commit()
    except Exception:
        pass


def get_search_analytics(conn, days=30, limit=20):
    """Get search analytics: top queries, trends, zero-result queries."""
    import math
    cutoff = time.time() - (days * 86400)

    stats = {}
    # Total searches
    stats['total_searches'] = conn.execute(
        "SELECT COUNT(*) FROM search_history WHERE ts >= ?", (cutoff,)).fetchone()[0]

    # Average response time
    row = conn.execute(
        "SELECT AVG(elapsed_ms), MAX(elapsed_ms) FROM search_history WHERE ts >= ?",
        (cutoff,)).fetchone()
    stats['avg_ms'] = round(row[0] or 0, 1)
    stats['max_ms'] = row[1] or 0

    # Top queries
    stats['top_queries'] = [
        {'query': r[0], 'count': r[1], 'avg_results': round(r[2] or 0, 1)}
        for r in conn.execute("""
            SELECT query, COUNT(*) as cnt, AVG(result_count)
            FROM search_history WHERE ts >= ?
            GROUP BY lower(query) ORDER BY cnt DESC LIMIT ?
        """, (cutoff, limit))]

    # Zero-result queries (potential gaps)
    stats['zero_result_queries'] = [
        {'query': r[0], 'count': r[1]}
        for r in conn.execute("""
            SELECT query, COUNT(*) as cnt
            FROM search_history WHERE ts >= ? AND result_count = 0
            GROUP BY lower(query) ORDER BY cnt DESC LIMIT ?
        """, (cutoff, limit))]

    # Daily search volume
    stats['daily_volume'] = [
        {'date': r[0], 'count': r[1]}
        for r in conn.execute("""
            SELECT date(ts, 'unixepoch') as day, COUNT(*)
            FROM search_history WHERE ts >= ?
            GROUP BY day ORDER BY day DESC LIMIT 30
        """, (cutoff,))]

    # Search type breakdown
    stats['by_type'] = dict(conn.execute("""
        SELECT search_type, COUNT(*)
        FROM search_history WHERE ts >= ?
        GROUP BY search_type
    """, (cutoff,)))

    return stats


# ═══════════════════════════════════════════════════════════
#  Smart Search (CLI --smart uses local Ollama; MCP: Claude expands)
# ═══════════════════════════════════════════════════════════

def smart_expand(query, ollama_url=None, model=None):
    """Expand search query using LLM. Returns list of keywords."""
    import urllib.request
    url = (ollama_url or OLLAMA_URL) + "/api/generate"
    mdl = model or OLLAMA_MODEL
    prompt = (
        "You are a search keyword expander. Given a search query, "
        "output ONLY a comma-separated list of related keywords, "
        "synonyms, and translations (Korean<->English).\n"
        "Rules: Output ONLY keywords, no explanation. Max 8 keywords. "
        "Include synonyms, related technical terms, Korean/English translations. "
        "Keep original query terms.\n\n"
        f"Query: {query}\nKeywords:"
    )
    payload = _json.dumps({
        "model": mdl, "prompt": prompt, "stream": False,
        "options": {"temperature": 0.3, "num_predict": 100}
    }).encode()
    try:
        req = urllib.request.Request(
            url, data=payload, headers={"Content-Type": "application/json"})
        with urllib.request.urlopen(req, timeout=SMART_TIMEOUT) as resp:
            raw = _json.loads(resp.read()).get("response", "").strip()
            keywords = [k.strip().strip('"').strip("'")
                       for k in raw.split(",") if k.strip()]
            all_terms = list(dict.fromkeys(query.split() + keywords))
            return all_terms[:12]
    except Exception:
        return query.split()


def build_fts_or_query(terms):
    """Build FTS5 OR query from keyword list."""
    safe = []
    for t in terms:
        t = t.replace('"', '').replace("'", "").strip()
        if t:
            safe.append(f'"{t}"' if ' ' in t else t)
    return " OR ".join(safe) if safe else (terms[0] if terms else "")


# ═══════════════════════════════════════════════════════════
#  CLI Interface
# ═══════════════════════════════════════════════════════════

def _parse_args(argv):
    """Parse CLI arguments into (cmd, query, options)."""
    opts = {
        'json': False, 'smart': False, 
        'type': None, 'limit': 20, 'all': False,
    }
    args = []
    i = 0
    while i < len(argv):
        a = argv[i]
        if a == '--json': opts['json'] = True
        elif a == '--smart': opts['smart'] = True
        elif a == '--all': opts['all'] = True
        elif a == '--type' and i+1 < len(argv):
            opts['type'] = argv[i+1]; i += 1
        elif a == '--limit' and i+1 < len(argv):
            opts['limit'] = int(argv[i+1]); i += 1
        elif not a.startswith('--'):
            args.append(a)
        i += 1
    return args, opts


# ========= Distributed Search =========
# port 51830 = vssh, others = SSH
DISTRIBUTED_SERVERS = {
    'v1': ('10.99.x.x', 51830, 'root'),
    'v2': ('10.99.x.x', 51830, 'root'),
    'v3': ('10.99.x.x', 51830, 'root'),
    'v4': ('10.99.x.x', 51830, 'root'),
    'g1': ('10.99.x.x', 51830, 'user'),
    'g2': ('10.99.x.x', 51830, 'user'),
    'g3': ('10.99.x.x', 51830, 'user'),
    'g4': ('10.99.x.x', 51830, 'user'),
    'd1': ('10.99.x.x', 51830, 'root'),
    'd2': ('10.99.x.x', 51830, 'user'),
    's1': ('10.99.x.x', 51830, 'user'),  # Synology NAS
}

def _get_local_server():
    """Detect which server we're on."""
    import socket
    hostname = socket.gethostname()
    if 'user' in hostname.lower() or 'mac' in hostname.lower():
        return 'm1'
    for name, (ip, _, _) in DISTRIBUTED_SERVERS.items():
        try:
            local_ips = socket.gethostbyname_ex(socket.gethostname())[2]
            if ip in local_ips:
                return name
        except:
            pass
    return None

def distributed_search(query, limit=20, doc_type=None):
    """Search across all servers in parallel."""
    import subprocess
    from concurrent.futures import ThreadPoolExecutor, as_completed

    local_server = _get_local_server()
    all_results = []
    server_stats = {}

    def search_one(srv_name, srv_info):
        ip, port, user = srv_info
        try:
            # Synology NAS uses different home path
            if srv_name == 's1':
                home = f'/var/services/homes/{user}'
            elif user == 'root':
                home = '/root'
            else:
                home = f'/home/{user}'
            # use vssh (port 51830)
            if port == 51830:
                # vssh: use config alias, secret auto-loaded from config
                vssh_secret = os.environ.get('VSSH_SECRET', '')
                if not vssh_secret:
                    # read secret from config
                    cfg_path = os.path.expanduser('~/.vssh/config')
                    if os.path.exists(cfg_path):
                        with open(cfg_path) as f:
                            for line in f:
                                if line.startswith('SECRET='):
                                    vssh_secret = line.split('=', 1)[1].strip()
                                    break
                cmd = ['vssh', 'ssh', srv_name, f'{home}/.local/bin/meshdb search --json "{query}"']
                env = os.environ.copy()
                env['VSSH_SECRET'] = vssh_secret
                result = subprocess.run(cmd, capture_output=True, text=True, timeout=15, env=env)
            else:
                # regular SSH
                cmd = ['ssh', '-o', 'ConnectTimeout=5', '-p', str(port),
                       f'{user}@{ip}', f'{home}/.local/bin/meshdb search --json "{query}"']
                result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
            if result.returncode == 0:
                data = _json.loads(result.stdout)
                return srv_name, data.get('results', []), None
        except Exception as e:
            return srv_name, [], str(e)
        return srv_name, [], 'failed'

    # local search
    local_result = api_search(query, limit=limit*2, doc_type=doc_type)
    local_name = local_server or 'm1'
    all_results.extend([(local_name, r) for r in local_result.get('results', [])])
    server_stats[local_name] = len(local_result.get('results', []))

    # remote search (parallel)
    with ThreadPoolExecutor(max_workers=10) as ex:
        futures = {ex.submit(search_one, name, info): name
                   for name, info in DISTRIBUTED_SERVERS.items()}
        for f in as_completed(futures, timeout=20):
            try:
                srv, results, err = f.result()
                if results:
                    all_results.extend([(srv, r) for r in results])
                    server_stats[srv] = len(results)
                elif err:
                    server_stats[srv] = f'error: {err}'
                else:
                    server_stats[srv] = 0  # No results, no error
            except Exception as e:
                pass

    # sort by score, top N
    all_results.sort(key=lambda x: x[1].get('score', 0), reverse=True)
    final = []
    for srv, r in all_results[:limit]:
        r['server'] = srv
        final.append(r)

    return {'results': final, 'server_stats': server_stats, 'total': len(all_results)}


def cli_print_results(results, query):
    """Print search results with i18n."""
    if not results:
        print(f"  '{query}' - {t('cli.no_results')}")
        return
    print(f"  '{query}' {t('cli.search_result')}: {len(results)}\n")
    for i, r in enumerate(results, 1):
        score = f"[{r['score']:.2f}]" if 'score' in r else ""
        icon = TYPE_ICONS.get(r['type'], '\U0001f4c1')
        print(f"  {i}. {icon} {r['filename']} {score}")
        print(f"     {r['path']}")
        if r.get('snippet'):
            snip = r['snippet'].replace('\n', ' ').strip()
            print(f"     {snip[:150]}{'...' if len(snip) > 150 else ''}")
        print(f"     [{r['type']}] {r.get('mtime', '')} "
              f"{format_size(r.get('size', 0)) if r.get('size') else ''}")
        print()


def cli_print_status(stats):
    """Print DB status with i18n."""
    print(f"\n  meshdb {t('cli.status')}")
    print(f"  {'=' * 50}")
    print(f"  {t('cli.docs')}: {stats['total_docs']:,}")
    print(f"  {t('cli.embedded')}: {stats.get('embedded_docs', 0):,}")
    print(f"  {t('cli.db_size')}: {stats.get('db_size_human', 'N/A')}")
    print(f"  {t('cli.last_indexed')}: {stats.get('last_indexed', 'N/A')}")
    if stats.get('by_type'):
        print(f"\n  {t('cli.by_type')}:")
        for tp, c in stats['by_type'].items():
            print(f"    {TYPE_ICONS.get(tp, '')} {tp}: {c:,}")
    if stats.get('by_source'):
        print(f"\n  {t('cli.by_source')}:")
        for s, c in stats['by_source'].items():
            print(f"    {s}: {c:,}")
    print()


def main():
    if len(sys.argv) < 2:
        print(__doc__); sys.exit(1)

    cmd = sys.argv[1]
    rest, opts = _parse_args(sys.argv[2:])
    jm = opts['json']

    if cmd in ('index', 'reindex'):
        if not rest:
            print(f"usage: meshdb {cmd} <path>"); sys.exit(1)
        result = api_index(rest[0], replace=(cmd == 'reindex'))
        if jm:
            print(_json.dumps(result, ensure_ascii=False))
        else:
            label = t('cli.reindexing' if cmd == 'reindex' else 'cli.indexing')
            print(f"\n  {label}: {result['source_dir']}")
            print(f"  {t('cli.scanned')}: {result['total_scanned']}, "
                  f"{t('cli.new')}: {result['new']}, "
                  f"{t('cli.updated')}: {result['updated']}, "
                  f"{t('cli.moved')}: {result['moved']}, "
                  f"{t('cli.skipped')}: {result['skipped']}, "
                  f"{t('cli.errors')}: {result['errors']}")
            print(f"  {t('cli.time')}: {result['elapsed_s']}s\n")

    elif cmd == 'search':
        query = ' '.join(rest)
        if not query:
            print("usage: meshdb search [--smart] [--all] <query>"); sys.exit(1)

        # distributed search
        if opts['all']:
            if not jm:
                print(f"\n  🌐 Distributed search: '{query}'...")
            result = distributed_search(query, limit=opts['limit'], doc_type=opts['type'])
            if jm:
                print(_json.dumps(result, ensure_ascii=False))
            else:
                print(f"  Servers: {len(result.get('server_stats', {}))} | Total: {result.get('total', 0)}")
                for srv, stat in result.get('server_stats', {}).items():
                    print(f"    {srv}: {stat}")
                print()
                cli_print_results(result['results'], query)
        elif opts['smart']:
            if not jm:
                print(f"\n  \U0001f9e0 {t('cli.smart_expanding')}: '{query}'...")
            expanded = smart_expand(query)
            fts_query = build_fts_or_query(expanded)
            if not jm:
                print(f"  {t('cli.expanded')}: {', '.join(expanded)}")
                print(f"  FTS: {fts_query}\n")
            result = api_search(fts_query, limit=opts['limit'], doc_type=opts['type'])
            result['smart'] = True
            result['original_query'] = query
            result['expanded_terms'] = expanded
            if jm:
                print(_json.dumps(result, ensure_ascii=False))
            else:
                print(f"  {t('cli.search_time')}: {result['elapsed_ms']}ms")
                cli_print_results(result['results'], query)
        else:
            result = api_search(query, limit=opts['limit'], doc_type=opts['type'])
            if jm:
                print(_json.dumps(result, ensure_ascii=False))
            else:
                print(f"  {t('cli.search_time')}: {result['elapsed_ms']}ms")
                cli_print_results(result['results'], query)

    elif cmd == 'find':
        if not rest:
            print("usage: meshdb find <filename>"); sys.exit(1)
        result = api_find(rest[0], limit=opts['limit'])
        if jm:
            print(_json.dumps(result, ensure_ascii=False))
        elif result['results']:
            print(f"\n  {t('cli.filename_search')}: '{rest[0]}' -> {result['count']}\n")
            for r in result['results']:
                icon = TYPE_ICONS.get(r['type'], '\U0001f4c1')
                print(f"  {icon} {r['filename']}\n    {r['path']}\n")
        else:
            print(f"\n  '{rest[0]}' - {t('cli.not_found')}\n")

    elif cmd == 'read':
        if not rest:
            print("usage: meshdb read <path_or_filename>"); sys.exit(1)
        result = api_read(rest[0])
        if jm:
            print(_json.dumps(result, ensure_ascii=False))
        elif 'error' in result:
            print(f"\n  {result['error']}\n")
        else:
            print(f"\n  --- {result['path']} ---\n{result['content']}")

    elif cmd == 'status':
        result = api_status()
        if jm:
            print(_json.dumps(result, ensure_ascii=False))
        else:
            cli_print_status(result)

    else:
        print(f"{t('cli.unknown_cmd')}: {cmd}")
        print(__doc__); sys.exit(1)


if __name__ == '__main__':
    main()
