"""
Doit Agent - Tool Registry v2.0 (ENHANCED)
New tools: shell_exec, git_*, docker_*, pkg_*, web_search/scrape,
ssh_run, sftp_*, desktop (notify, screenshot, clipboard, hotkey, volume, speech),
dev tools (test, lint, format, build), file processing (pdf, img, csv, json),
memory/notes/todos, secrets keychain, webhooks, calc, translate, weather
"""
from __future__ import annotations

import asyncio, base64, csv, hashlib, json, logging, math, os, platform
import re, shutil, socket, subprocess, sys, tarfile, tempfile, time, zipfile
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from pathlib import Path
from typing import Any, Callable, Optional

import psutil

_executor = ThreadPoolExecutor(max_workers=8)

async def _run_in_thread(fn, *args, **kwargs):
    """Run a sync function in a thread pool. Never raises — errors become dicts."""
    loop = asyncio.get_event_loop()
    def _safe():
        try:
            return fn(*args, **kwargs)
        except Exception as e:
            return {"error": str(e), "success": False}
    return await loop.run_in_executor(_executor, _safe)

from doit_fm.core.config import MAX_CPU_PERCENT, MAX_DISK_WRITE_MB, HTTP_TIMEOUT, BLOCKED_PATHS
from doit_fm.security.security import InjectionGuard

logger = logging.getLogger("doit.tools")


def _run(cmd, timeout=30, input=None):
    try:
        r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, input=input)
        return {"stdout": r.stdout.strip(), "stderr": r.stderr.strip(),
                "returncode": r.returncode, "success": r.returncode == 0}
    except subprocess.TimeoutExpired:
        return {"error": f"Timed out after {timeout}s", "success": False}
    except FileNotFoundError:
        return {"error": f"Command not found: {cmd[0]}", "success": False}
    except Exception as e:
        return {"error": str(e), "success": False}


def _which(name): return shutil.which(name) is not None


class ToolDefinition:
    def __init__(self, name, description, fn, dangerous=False):
        self.name = name; self.description = description
        self.fn = fn; self.dangerous = dangerous


# ── FILE SYSTEM ──────────────────────────────────────────────────────────────

async def fs_list(path=".", **_):
    p = Path(path).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error": reason}
    if not p.exists(): return {"error": f"Path not found: {p}"}
    if p.is_file(): return {"type":"file","path":str(p),"size":p.stat().st_size}
    entries = []
    for item in sorted(p.iterdir()):
        try:
            s = item.stat()
            entries.append({"name":item.name,"type":"dir" if item.is_dir() else "file",
                             "size":s.st_size if item.is_file() else None,
                             "modified":datetime.fromtimestamp(s.st_mtime).isoformat()})
        except PermissionError:
            entries.append({"name":item.name,"type":"unknown","error":"permission denied"})
    return {"path":str(p),"entries":entries,"count":len(entries)}

async def fs_read(path, **_):
    p = Path(path).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error": reason}
    if not p.exists(): return {"error": f"File not found: {p}"}
    if p.stat().st_size > 5*1024*1024: return {"error": "File too large (>5MB)"}
    try:
        try:
            import aiofiles
            async with aiofiles.open(p, "r", encoding="utf-8", errors="replace") as f:
                content = await f.read()
        except ImportError:
            content = p.read_text(encoding="utf-8", errors="replace")
        return {"path": str(p), "content": content, "size": len(content)}
    except Exception as e: return {"error": str(e)}

async def fs_write(path, content, **_):
    p = Path(path).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error": reason}
    size = len(content.encode())
    if size > MAX_DISK_WRITE_MB*1024*1024: return {"error": f"Content too large (>{MAX_DISK_WRITE_MB}MB)"}
    try:
        p.parent.mkdir(parents=True, exist_ok=True)
        try:
            import aiofiles
            async with aiofiles.open(p, "w", encoding="utf-8") as f: await f.write(content)
        except ImportError:
            p.write_text(content, encoding="utf-8")
        return {"path": str(p), "bytes_written": size, "success": True}
    except Exception as e: return {"error": str(e)}

async def fs_append(path, content, **_):
    p = Path(path).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error": reason}
    try:
        p.parent.mkdir(parents=True, exist_ok=True)
        try:
            import aiofiles
            async with aiofiles.open(p, "a", encoding="utf-8") as f: await f.write(content)
        except ImportError:
            with open(p, "a", encoding="utf-8") as f: f.write(content)
        return {"path": str(p), "appended_bytes": len(content.encode()), "success": True}
    except Exception as e: return {"error": str(e)}

async def fs_delete(path, **_):
    p = Path(path).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error":reason}
    if not p.exists(): return {"error":f"Not found: {p}"}
    try:
        shutil.rmtree(p) if p.is_dir() else p.unlink()
        return {"deleted":str(p),"success":True}
    except Exception as e: return {"error":str(e)}

async def fs_move(src, dst, **_):
    s,d = Path(src).expanduser().resolve(), Path(dst).expanduser().resolve()
    for p in (s,d):
        safe,reason = InjectionGuard.check_path(str(p))
        if not safe: return {"error":reason}
    if not s.exists(): return {"error":f"Source not found: {s}"}
    try:
        d.parent.mkdir(parents=True, exist_ok=True)
        shutil.move(str(s),str(d))
        return {"src":str(s),"dst":str(d),"success":True}
    except Exception as e: return {"error":str(e)}

async def fs_copy(src, dst, **_):
    s,d = Path(src).expanduser().resolve(), Path(dst).expanduser().resolve()
    for p in (s,d):
        safe,reason = InjectionGuard.check_path(str(p))
        if not safe: return {"error":reason}
    if not s.exists(): return {"error":f"Source not found: {s}"}
    try:
        d.parent.mkdir(parents=True, exist_ok=True)
        shutil.copytree(str(s),str(d)) if s.is_dir() else shutil.copy2(str(s),str(d))
        return {"src":str(s),"dst":str(d),"success":True}
    except Exception as e: return {"error":str(e)}

async def fs_search(path, pattern, content=None, **_):
    p = Path(path).expanduser().resolve()
    safe,reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error":reason}
    if not p.exists(): return {"error":f"Path not found: {p}"}
    results = []
    try:
        for match in p.rglob(pattern):
            entry = {"path":str(match),"type":"dir" if match.is_dir() else "file",
                     "size":match.stat().st_size if match.is_file() else None}
            if content and match.is_file():
                try:
                    text = match.read_text(errors="replace")
                    if content.lower() not in text.lower(): continue
                    entry["matching_lines"] = [l for l in text.splitlines() if content.lower() in l.lower()][:5]
                except Exception: pass
            results.append(entry)
            if len(results) >= 200: break
    except Exception as e: return {"error":str(e)}
    return {"pattern":pattern,"results":results,"count":len(results)}

async def fs_compress(src, dst, **_):
    s,d = Path(src).expanduser().resolve(), Path(dst).expanduser().resolve()
    for p in (s,d):
        safe,reason = InjectionGuard.check_path(str(p))
        if not safe: return {"error":reason}
    if not s.exists(): return {"error":f"Source not found: {s}"}
    try:
        d.parent.mkdir(parents=True, exist_ok=True)
        if str(dst).endswith((".tar.gz",".tgz")):
            with tarfile.open(str(d),"w:gz") as tar: tar.add(str(s),arcname=s.name)
        else:
            with zipfile.ZipFile(str(d),"w",zipfile.ZIP_DEFLATED) as zf:
                if s.is_dir():
                    for f in s.rglob("*"): zf.write(f, f.relative_to(s.parent))
                else: zf.write(str(s),s.name)
        return {"src":str(s),"dst":str(d),"size":d.stat().st_size,"success":True}
    except Exception as e: return {"error":str(e)}

async def fs_extract(src, dst, **_):
    s,d = Path(src).expanduser().resolve(), Path(dst).expanduser().resolve()
    for p in (s,d):
        safe,reason = InjectionGuard.check_path(str(p))
        if not safe: return {"error":reason}
    if not s.exists(): return {"error":f"Archive not found: {s}"}
    try:
        d.mkdir(parents=True, exist_ok=True)
        if tarfile.is_tarfile(str(s)):
            with tarfile.open(str(s),"r:*") as tar: tar.extractall(str(d))
        elif zipfile.is_zipfile(str(s)):
            with zipfile.ZipFile(str(s),"r") as zf: zf.extractall(str(d))
        else: return {"error":"Unsupported archive format"}
        return {"src":str(s),"dst":str(d),"success":True}
    except Exception as e: return {"error":str(e)}

ORGANIZE_RULES = {
    "images":[".jpg",".jpeg",".png",".gif",".bmp",".svg",".webp",".heic",".tiff"],
    "documents":[".pdf",".doc",".docx",".xls",".xlsx",".ppt",".pptx",".txt",".md",".rtf"],
    "videos":[".mp4",".avi",".mkv",".mov",".wmv",".flv",".webm"],
    "audio":[".mp3",".wav",".flac",".aac",".ogg",".m4a"],
    "archives":[".zip",".tar",".gz",".7z",".rar",".bz2"],
    "code":[".py",".js",".ts",".html",".css",".java",".c",".cpp",".go",".rs",".rb",".php",".swift"],
    "data":[".csv",".json",".xml",".yaml",".yml",".sql"],
}

async def fs_organize(path, **_):
    p = Path(path).expanduser().resolve()
    safe,reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error":reason}
    if not p.is_dir(): return {"error":f"Not a directory: {p}"}
    moved = []
    for item in p.iterdir():
        if item.is_file():
            ext = item.suffix.lower()
            for folder,exts in ORGANIZE_RULES.items():
                if ext in exts:
                    target = p/folder; target.mkdir(exist_ok=True)
                    shutil.move(str(item),str(target/item.name))
                    moved.append({"file":item.name,"to":folder}); break
    return {"path":str(p),"moved":moved,"count":len(moved),"success":True}

async def fs_backup(src, dst, **_):
    s,d = Path(src).expanduser().resolve(), Path(dst).expanduser().resolve()
    for p,name in ((s,"src"),(d,"dst")):
        safe,reason = InjectionGuard.check_path(str(p))
        if not safe: return {"error":f"Blocked {name}: {reason}"}
    if not s.exists(): return {"error":f"Source not found: {s}"}
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    backup_path = d/f"{s.name}_backup_{ts}"
    result = await fs_copy(str(s),str(backup_path))
    if "error" not in result:
        result["backup_path"] = str(backup_path); result["timestamp"] = ts
    return result

async def fs_clean(path, days=None, size_mb=None, type=None, **_):
    p = Path(path).expanduser().resolve()
    safe,reason = InjectionGuard.check_path(str(p))
    if not safe: return {"error":reason}
    if not p.is_dir(): return {"error":f"Not a directory: {p}"}
    deleted = []; now = time.time()
    for item in p.iterdir():
        if not item.is_file(): continue
        try:
            s = item.stat(); should = False
            if days and (now-s.st_mtime)>days*86400: should = True
            if size_mb and s.st_size>size_mb*1024*1024: should = True
            if type and item.suffix.lower()==f".{type.lstrip('.')}": should = True
            if should: item.unlink(); deleted.append(item.name)
        except Exception: pass
    return {"deleted":deleted,"count":len(deleted),"success":True}

async def fs_diff(path1, path2, **_):
    import difflib
    try:
        t1 = Path(path1).expanduser().read_text(errors="replace").splitlines()
        t2 = Path(path2).expanduser().read_text(errors="replace").splitlines()
        diff = list(difflib.unified_diff(t1,t2,fromfile=path1,tofile=path2,lineterm=""))
        return {"diff":"\n".join(diff[:200]),"changed_lines":len([l for l in diff if l.startswith(("+","-")) and not l.startswith(("+++","---"))]),"success":True}
    except Exception as e: return {"error":str(e)}

async def fs_find_duplicates(path, **_):
    p = Path(path).expanduser().resolve()
    if not p.is_dir(): return {"error":"Not a directory"}
    hashes = {}
    for f in p.rglob("*"):
        if f.is_file() and f.stat().st_size > 0:
            try:
                h = hashlib.md5(f.read_bytes()).hexdigest()
                hashes.setdefault(h,[]).append(str(f))
            except Exception: pass
    dupes = {h:paths for h,paths in hashes.items() if len(paths)>1}
    return {"duplicates":dupes,"groups":len(dupes),"total_extra_copies":sum(len(v)-1 for v in dupes.values()),"success":True}


# ── SYSTEM ───────────────────────────────────────────────────────────────────

async def sys_info(**_):
    return {"os":platform.system(),"os_version":platform.version(),"hostname":platform.node(),
            "python":platform.python_version(),"cpu_count":psutil.cpu_count(),
            "cpu_freq_mhz":psutil.cpu_freq().current if psutil.cpu_freq() else None,
            "ram_total_gb":round(psutil.virtual_memory().total/1e9,2),
            "disk_total_gb":round(psutil.disk_usage("/").total/1e9,2),
            "uptime_hours":round((time.time()-psutil.boot_time())/3600,1),
            "architecture":platform.machine(),"user":os.getenv("USER",os.getenv("USERNAME","unknown"))}

async def sys_processes(**_):
    procs = []
    for p in psutil.process_iter(["pid","name","status","cpu_percent","memory_info"]):
        try:
            procs.append({"pid":p.info["pid"],"name":p.info["name"],"status":p.info["status"],
                          "cpu_pct":p.info["cpu_percent"],
                          "mem_mb":round(p.info["memory_info"].rss/1e6,1) if p.info["memory_info"] else None})
        except (psutil.NoSuchProcess,psutil.AccessDenied): pass
    procs.sort(key=lambda x:x.get("mem_mb") or 0,reverse=True)
    return {"processes":procs[:50],"total":len(procs)}

async def sys_kill(pid, **_):
    try:
        proc = psutil.Process(pid); name = proc.name().lower()
        if any(c in name for c in ["systemd","init","kernel","kthreadd","launchd","svchost"]):
            return {"error":f"Refusing to kill critical process: {name}"}
        proc.terminate()
        try: proc.wait(timeout=5)
        except psutil.TimeoutExpired: proc.kill()
        return {"pid":pid,"success":True}
    except psutil.NoSuchProcess: return {"error":f"Process {pid} not found"}
    except psutil.AccessDenied: return {"error":f"Access denied to process {pid}"}

async def sys_monitor(**_):
    vm = psutil.virtual_memory(); disk = psutil.disk_usage("/")
    return {"cpu_percent":psutil.cpu_percent(interval=1),"cpu_per_core":psutil.cpu_percent(interval=0.1,percpu=True),
            "ram_used_gb":round(vm.used/1e9,2),"ram_total_gb":round(vm.total/1e9,2),"ram_percent":vm.percent,
            "disk_used_gb":round(disk.used/1e9,2),"disk_total_gb":round(disk.total/1e9,2),"disk_percent":disk.percent,
            "net_sent_mb":round(psutil.net_io_counters().bytes_sent/1e6,1),
            "net_recv_mb":round(psutil.net_io_counters().bytes_recv/1e6,1)}

async def sys_shutdown(delay_s=0, **_):
    if delay_s: await asyncio.sleep(delay_s)
    if sys.platform=="win32": return _run(["shutdown","/s","/t","0"])
    elif sys.platform=="darwin": return _run(["osascript","-e",'tell app "System Events" to shut down'])
    else:
        r = _run(["systemctl","poweroff"])
        return r if r["success"] else _run(["sudo","shutdown","-h","now"])

async def sys_restart(delay_s=0, **_):
    if delay_s: await asyncio.sleep(delay_s)
    if sys.platform=="win32": return _run(["shutdown","/r","/t","0"])
    elif sys.platform=="darwin": return _run(["osascript","-e",'tell app "System Events" to restart'])
    else:
        r = _run(["systemctl","reboot"])
        return r if r["success"] else _run(["sudo","shutdown","-r","now"])

async def sys_sleep(**_):
    if sys.platform=="darwin": return _run(["pmset","sleepnow"])
    elif sys.platform=="linux": return _run(["systemctl","suspend"])
    return _run(["rundll32.exe","powrprof.dll,SetSuspendState","0,1,0"])

async def sys_battery(**_):
    try:
        b = psutil.sensors_battery()
        if not b: return {"error":"No battery found"}
        return {"percent":b.percent,"plugged":b.power_plugged,
                "time_left_min":round(b.secsleft/60) if b.secsleft>0 else None}
    except Exception as e: return {"error":str(e)}

async def sys_disk_usage(path="/", **_):
    try:
        u = psutil.disk_usage(path)
        return {"path":path,"total_gb":round(u.total/1e9,2),"used_gb":round(u.used/1e9,2),
                "free_gb":round(u.free/1e9,2),"percent":u.percent}
    except Exception as e: return {"error":str(e)}

async def sys_env_list(**_):
    return {"env":dict(os.environ),"count":len(os.environ)}

async def sys_env_get(name, **_):
    val = os.environ.get(name)
    return {"name":name,"value":val,"found":val is not None}


# ── SHELL / EXEC ──────────────────────────────────────────────────────────────

async def shell_exec(command, cwd=None, timeout=60, **_):
    from doit_fm.core.config import BLOCKED_COMMANDS
    for b in BLOCKED_COMMANDS:
        if b in command: return {"error":f"Command blocked: '{b}'"}
    cwd_path = Path(cwd).expanduser() if cwd else Path.home()
    try:
        proc = await asyncio.create_subprocess_shell(
            command,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE,cwd=str(cwd_path))
        stdout,stderr = await asyncio.wait_for(proc.communicate(),timeout=timeout)
        return {"command":command,"stdout":stdout.decode(errors="replace")[:4000],
                "stderr":stderr.decode(errors="replace")[:1000],
                "returncode":proc.returncode,"success":proc.returncode==0}
    except asyncio.TimeoutError:
        proc.kill(); return {"error":f"Timed out after {timeout}s"}
    except Exception as e: return {"error":str(e)}

async def repl_python(code, **_):
    with tempfile.NamedTemporaryFile(suffix=".py",mode="w",delete=False) as f:
        f.write(code); fname = f.name
    try: return _run([sys.executable,fname],timeout=30)
    finally: Path(fname).unlink(missing_ok=True)

async def repl_js(code, **_):
    if not _which("node"): return {"error":"Node.js not installed"}
    with tempfile.NamedTemporaryFile(suffix=".js",mode="w",delete=False) as f:
        f.write(code); fname = f.name
    try: return _run(["node",fname],timeout=30)
    finally: Path(fname).unlink(missing_ok=True)


# ── GIT ───────────────────────────────────────────────────────────────────────

async def git_status(path=".", **_):
    base = ["git","-C",str(Path(path).expanduser())]
    r = _run(base+["status","--short"])
    if not r["success"]: return r
    r2 = _run(base+["log","--oneline","-5"])
    return {"status":r["stdout"],"recent_commits":r2["stdout"],"success":True}

async def git_pull(path=".", **_): return _run(["git","-C",str(Path(path).expanduser()),"pull"])
async def git_add(path=".",files=".", **_): return _run(["git","-C",str(Path(path).expanduser()),"add",files])
async def git_commit(path=".",message="Update", **_): return _run(["git","-C",str(Path(path).expanduser()),"commit","-m",message])
async def git_push(path=".", **_): return _run(["git","-C",str(Path(path).expanduser()),"push"])
async def git_log(path=".",limit=20, **_): return _run(["git","-C",str(Path(path).expanduser()),"log","--oneline",f"-{limit}"])
async def git_diff(path=".", **_): return _run(["git","-C",str(Path(path).expanduser()),"diff"])
async def git_clone(url,dst=".", **_): return _run(["git","clone",url,str(Path(dst).expanduser())],timeout=120)
async def git_branch(path=".", **_): return _run(["git","-C",str(Path(path).expanduser()),"branch","-a"])
async def git_checkout(path=".",branch="main", **_): return _run(["git","-C",str(Path(path).expanduser()),"checkout",branch])


# ── DOCKER ────────────────────────────────────────────────────────────────────

async def docker_list(**_):
    if not _which("docker"): return {"error":"Docker not installed"}
    r = _run(["docker","ps","-a","--format","{{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Image}}"])
    if not r["success"]: return r
    containers = []
    for line in r["stdout"].splitlines():
        parts = line.split("\t")
        if len(parts)==4: containers.append({"id":parts[0],"name":parts[1],"status":parts[2],"image":parts[3]})
    return {"containers":containers,"count":len(containers),"success":True}

async def docker_start(name, **_): return _run(["docker","start",name])
async def docker_stop(name, **_): return _run(["docker","stop",name])
async def docker_exec(name,command, **_): return _run(["docker","exec",name,"sh","-c",command],timeout=30)
async def docker_logs(name,lines=50, **_): return _run(["docker","logs","--tail",str(lines),name])
async def docker_images(**_):
    r = _run(["docker","images","--format","{{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.ID}}"])
    images = []
    for line in r.get("stdout","").splitlines():
        parts = line.split("\t")
        if len(parts)==4: images.append({"repository":parts[0],"tag":parts[1],"size":parts[2],"id":parts[3]})
    return {"images":images,"success":r["success"]}


# ── PACKAGE MANAGEMENT ────────────────────────────────────────────────────────

async def pkg_install(package,manager="auto", **_):
    if manager=="auto":
        if sys.platform=="darwin" and _which("brew"): manager="brew"
        elif sys.platform=="linux" and _which("apt"): manager="apt"
        elif sys.platform=="win32" and _which("winget"): manager="winget"
        else: manager="pip"
    cmds = {"brew":["brew","install",package],"apt":["sudo","apt","install","-y",package],
            "winget":["winget","install",package],"pip":[sys.executable,"-m","pip","install",package],
            "npm":["npm","install","-g",package],"cargo":["cargo","install",package]}
    cmd = cmds.get(manager)
    if not cmd: return {"error":f"Unknown package manager: {manager}"}
    return _run(cmd,timeout=120)

async def pkg_uninstall(package,manager="auto", **_):
    if manager=="auto": manager = "brew" if (sys.platform=="darwin" and _which("brew")) else "apt"
    cmds = {"brew":["brew","uninstall",package],"apt":["sudo","apt","remove","-y",package],
            "pip":[sys.executable,"-m","pip","uninstall","-y",package]}
    cmd = cmds.get(manager)
    if not cmd: return {"error":f"Unknown manager: {manager}"}
    return _run(cmd,timeout=60)


# ── NETWORK ───────────────────────────────────────────────────────────────────

async def net_download(url, dst, **_):
    d = Path(dst).expanduser().resolve()
    safe, reason = InjectionGuard.check_path(str(d))
    if not safe: return {"error": reason}
    d.parent.mkdir(parents=True, exist_ok=True)
    max_bytes = MAX_DISK_WRITE_MB * 1024 * 1024
    # Try aiohttp first
    try:
        import aiohttp
        import aiofiles
        async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
            async with sess.get(url) as resp:
                resp.raise_for_status()
                bw = 0
                async with aiofiles.open(d, "wb") as f:
                    async for chunk in resp.content.iter_chunked(65536):
                        bw += len(chunk)
                        if bw > max_bytes:
                            try: d.unlink()
                            except: pass
                            return {"error": f"File too large (>{MAX_DISK_WRITE_MB}MB)"}
                        await f.write(chunk)
        return {"url": url, "dst": str(d), "size_bytes": bw, "success": True}
    except ImportError:
        pass
    except Exception as e:
        return {"error": str(e)}
    # Fallback: urllib
    try:
        import urllib.request
        bw = 0
        with urllib.request.urlopen(url, timeout=60) as resp, open(d, "wb") as f:
            while True:
                chunk = resp.read(65536)
                if not chunk: break
                bw += len(chunk)
                if bw > max_bytes:
                    try: d.unlink()
                    except: pass
                    return {"error": f"File too large (>{MAX_DISK_WRITE_MB}MB)"}
                f.write(chunk)
        return {"url": url, "dst": str(d), "size_bytes": bw, "success": True}
    except Exception as e:
        return {"error": f"Download failed: {e}"}

async def net_ping(host, count=4, **_):
    """Ping a host using system ping or socket TCP-connect fallback."""
    # Try system ping binary
    ping_bin = shutil.which("ping")
    if ping_bin:
        args = ["-c", str(count)] if sys.platform != "win32" else ["-n", str(count)]
        r = _run([ping_bin] + args + [host], timeout=15)
        m = re.search(r"min/avg/max[^=]*=\s*([\d.]+)/([\d.]+)/([\d.]+)", r.get("stdout", ""))
        if m:
            r["latency_avg_ms"] = float(m.group(2))
        return {"host": host, "success": r.get("returncode") == 0,
                "output": r.get("stdout", "")[:500]}

    # Socket-based fallback (TCP connect to common ports)
    results = []
    for _ in range(int(count)):
        start = time.time()
        reached = False
        for port in [80, 443, 22, 53]:
            try:
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.settimeout(2)
                s.connect((host, port))
                s.close()
                results.append((time.time() - start) * 1000)
                reached = True
                break
            except (ConnectionRefusedError, OSError):
                try:
                    socket.gethostbyname(host)
                    results.append((time.time() - start) * 1000)
                    reached = True
                    break
                except Exception:
                    pass
    if results:
        avg = round(sum(results) / len(results), 1)
        return {"host": host, "success": True, "latency_avg_ms": avg,
                "packets_sent": count, "packets_received": len(results),
                "output": f"Ping {host}: {len(results)}/{count} ok, avg {avg}ms"}
    return {"host": host, "success": False, "output": f"Host unreachable: {host}"}

async def net_http(method, url, headers=None, body=None, **_):
    """Make an HTTP request (GET/POST/PUT/PATCH/DELETE/HEAD)."""
    method = method.upper()
    if method not in ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"):
        return {"error": f"Unsupported method: {method}. Use: GET POST PUT PATCH DELETE HEAD"}
    def _do():
        try:
            import requests as _req
            kwargs = {"headers": headers or {}, "timeout": HTTP_TIMEOUT}
            if body: kwargs["json"] = body
            r = getattr(_req, method.lower())(url, **kwargs)
            return {"status": r.status_code, "headers": dict(r.headers),
                    "body": r.text[:8000], "success": 200 <= r.status_code < 300}
        except ImportError:
            pass
        import urllib.request
        data = json.dumps(body).encode() if body else None
        req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
        if body: req.add_header("Content-Type", "application/json")
        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
            return {"status": r.status, "headers": dict(r.headers),
                    "body": r.read().decode("utf-8", errors="replace")[:8000],
                    "success": 200 <= r.status < 300}
    try:
        return await _run_in_thread(_do)
    except Exception as e:
        return {"error": f"HTTP {method} {url} failed: {e}"}

async def net_monitor(url, **_):
    """Check if a URL is up and measure response time."""
    def _check():
        try:
            import requests as _req
            start = time.time()
            r = _req.get(url, timeout=10, allow_redirects=True)
            return {"url": url, "status": r.status_code, "up": r.status_code < 400,
                    "latency_ms": round((time.time() - start) * 1000, 1), "success": True}
        except ImportError:
            pass
        import urllib.request
        start = time.time()
        with urllib.request.urlopen(url, timeout=10) as r:
            return {"url": url, "status": r.status, "up": r.status < 400,
                    "latency_ms": round((time.time() - start) * 1000, 1), "success": True}
    try:
        return await _run_in_thread(_check)
    except Exception as e:
        return {"url": url, "up": False, "error": str(e), "success": False}

async def web_search(query, limit=5, **_):
    """Search the web using DuckDuckGo (no API key needed)."""
    def _search():
        results = []
        try:
            import requests as _req
            # DuckDuckGo Instant Answers API
            r = _req.get("https://api.duckduckgo.com/",
                params={"q": query, "format": "json", "no_redirect": "1", "no_html": "1"},
                headers={"User-Agent": "doit-fm/3.0"}, timeout=10)
            data = r.json()
            if data.get("AbstractText"):
                results.append({"title": data.get("Heading", query),
                                 "snippet": data["AbstractText"], "url": data.get("AbstractURL", "")})
            for item in data.get("RelatedTopics", []):
                if isinstance(item, dict) and item.get("Text"):
                    results.append({"title": item["Text"][:100],
                                     "snippet": item["Text"], "url": item.get("FirstURL", "")})
            # Also scrape DDG HTML for more results
            if len(results) < int(limit):
                try:
                    r2 = _req.get("https://html.duckduckgo.com/html/",
                        params={"q": query},
                        headers={"User-Agent": "Mozilla/5.0 (compatible; doit-fm/3.0)"}, timeout=10)
                    from bs4 import BeautifulSoup
                    soup = BeautifulSoup(r2.text, "html.parser")
                    for el in soup.select(".result__body"):
                        t = el.select_one(".result__title")
                        s = el.select_one(".result__snippet")
                        u = el.select_one(".result__url")
                        if t:
                            results.append({"title": t.get_text(strip=True)[:100],
                                             "snippet": s.get_text(strip=True)[:200] if s else "",
                                             "url": u.get_text(strip=True) if u else ""})
                except Exception:
                    pass
        except ImportError:
            # urllib fallback
            import urllib.request, urllib.parse
            params = urllib.parse.urlencode({"q": query, "format": "json", "no_redirect": "1", "no_html": "1"})
            req = urllib.request.Request(f"https://api.duckduckgo.com/?{params}",
                headers={"User-Agent": "doit-fm/3.0"})
            with urllib.request.urlopen(req, timeout=10) as r:
                data = json.loads(r.read())
            if data.get("AbstractText"):
                results.append({"title": data.get("Heading", query),
                                 "snippet": data["AbstractText"], "url": data.get("AbstractURL", "")})
            for item in data.get("RelatedTopics", []):
                if isinstance(item, dict) and item.get("Text"):
                    results.append({"title": item["Text"][:100],
                                     "snippet": item["Text"], "url": item.get("FirstURL", "")})
        return {"query": query, "results": results[:int(limit)], "success": True}

    try:
        return await _run_in_thread(_search)
    except Exception as e:
        return {"error": f"Search failed: {e} — check internet connection",
                "query": query, "results": [], "success": False}

async def web_scrape(url, **_):
    """Fetch a URL and extract clean text content."""
    def _scrape():
        try:
            import requests as _req
            r = _req.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; doit-fm/3.0)"}, timeout=20)
            html = r.text
        except ImportError:
            import urllib.request
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
            with urllib.request.urlopen(req, timeout=20) as r:
                html = r.read().decode("utf-8", errors="replace")
        try:
            from bs4 import BeautifulSoup
            soup = BeautifulSoup(html, "html.parser")
            for tag in soup(["script", "style", "nav", "footer", "iframe", "head"]):
                tag.decompose()
            main = soup.find("main") or soup.find("article") or soup.find("body")
            text = main.get_text(separator=" ", strip=True) if main else soup.get_text(separator=" ", strip=True)
        except ImportError:
            text = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
            text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL)
            text = re.sub(r'<[^>]+>', ' ', text)
        text = re.sub(r'\s+', ' ', text).strip()
        return {"url": url, "text": text[:5000], "length": len(text), "success": True}
    try:
        return await _run_in_thread(_scrape)
    except Exception as e:
        return {"error": f"Scrape failed: {e}", "url": url}

async def webhook_send(url, payload=None, headers=None, **_):
    return await net_http("POST", url, headers=headers or {"Content-Type": "application/json"}, body=payload or {})


async def vol_mute(**_):
    """Mute system audio."""
    if sys.platform == "darwin":
        return _run(["osascript", "-e", "set volume output muted true"])
    elif sys.platform == "linux":
        for tool, cmd in [
            ("wpctl", ["wpctl", "set-mute", "@DEFAULT_AUDIO_SINK@", "1"]),
            ("amixer", ["amixer", "-q", "set", "Master", "mute"]),
            ("pactl", ["pactl", "set-sink-mute", "@DEFAULT_SINK@", "1"]),
        ]:
            if _which(tool):
                r = _run(cmd)
                if r.get("returncode") == 0:
                    return {"muted": True, "success": True}
        return {"error": "No volume control found. Install: sudo apt install pulseaudio-utils"}
    return {"error": "Mute not supported on this platform"}


async def vol_unmute(**_):
    """Unmute system audio."""
    if sys.platform == "darwin":
        return _run(["osascript", "-e", "set volume output muted false"])
    elif sys.platform == "linux":
        for tool, cmd in [
            ("wpctl", ["wpctl", "set-mute", "@DEFAULT_AUDIO_SINK@", "0"]),
            ("amixer", ["amixer", "-q", "set", "Master", "unmute"]),
            ("pactl", ["pactl", "set-sink-mute", "@DEFAULT_SINK@", "0"]),
        ]:
            if _which(tool):
                r = _run(cmd)
                if r.get("returncode") == 0:
                    return {"muted": False, "success": True}
        return {"error": "No volume control found. Install: sudo apt install pulseaudio-utils"}
    return {"error": "Unmute not supported on this platform"}


# ── SSH / SFTP ────────────────────────────────────────────────────────────────

async def ssh_run(host,command,user=None,port=22, **_):
    if not _which("ssh"): return {"error":"ssh not available"}
    target = f"{user}@{host}" if user else host
    return _run(["ssh","-p",str(port),"-o","StrictHostKeyChecking=no","-o","ConnectTimeout=10",target,command],timeout=60)

async def sftp_upload(host,local,remote,user=None, **_):
    if not _which("scp"): return {"error":"scp not available"}
    target = f"{user}@{host}:{remote}" if user else f"{host}:{remote}"
    return _run(["scp",str(Path(local).expanduser()),target],timeout=120)

async def sftp_download(host,remote,local,user=None, **_):
    if not _which("scp"): return {"error":"scp not available"}
    target = f"{user}@{host}:{remote}" if user else f"{host}:{remote}"
    return _run(["scp",target,str(Path(local).expanduser())],timeout=120)


# ── DESKTOP AUTOMATION ────────────────────────────────────────────────────────

async def app_open(name, **_):
    if sys.platform == "darwin":
        r = _run(["open", "-a", name])
        return r if r.get("success") else _run(["open", name])
    elif sys.platform == "linux":
        for launcher in ["xdg-open", "gtk-launch"]:
            if _which(launcher):
                return _run([launcher, name])
        return _run([name])
    return _run(["start", "", name])

async def notify_send(title, message, **_):
    if sys.platform == "darwin":
        return _run(["osascript", "-e", f'display notification "{message}" with title "{title}"'])
    elif sys.platform == "linux":
        for tool, args in [
            ("notify-send", [title, message]),
            ("zenity", ["--notification", f"--text={title}: {message}"]),
        ]:
            if _which(tool):
                r = _run([tool] + args)
                if r.get("returncode") == 0:
                    return {"title": title, "message": message, "success": True}
        # Always log to file as fallback
        log_path = Path.home() / ".config" / "doit-fm" / "notifications.log"
        log_path.parent.mkdir(parents=True, exist_ok=True)
        with open(log_path, "a") as f:
            f.write(f"[{datetime.now().isoformat()}] {title}: {message}\n")
        return {"title": title, "message": message, "logged_to": str(log_path),
                "note": "Install libnotify: sudo apt install libnotify-bin", "success": True}
    return {"error": "Not supported on this platform"}

async def screenshot_take(path=None, **_):
    if not path:
        path = str(Path.home() / f"screenshot_{int(time.time())}.png")
    if sys.platform == "darwin":
        r = _run(["screencapture", "-x", path])
    elif sys.platform == "linux":
        for tool, args in [
            ("scrot", [path]),
            ("import", ["-window", "root", path]),
            ("gnome-screenshot", ["-f", path]),
            ("ffmpeg", ["-y", "-f", "x11grab", "-video_size", "1920x1080", "-i", ":0.0", "-vframes", "1", path]),
        ]:
            if _which(tool):
                r = _run([tool] + args)
                if r.get("returncode") == 0:
                    return {"path": path, "success": True}
        return {"error": "No screenshot tool. Install: sudo apt install scrot", "path": path}
    else:
        return {"error": "Use Snipping Tool on Windows"}
    if isinstance(r, dict) and r.get("returncode") == 0:
        return {"path": path, "success": True}
    return r

async def clipboard_read(**_):
    if sys.platform == "darwin": return _run(["pbpaste"])
    elif sys.platform == "linux":
        for tool, args in [
            ("xclip", ["-o", "-sel", "clip"]),
            ("xsel", ["--clipboard", "--output"]),
            ("wl-paste", []),
        ]:
            if _which(tool): return _run([tool] + args)
        return {"error": "Install xclip: sudo apt install xclip"}
    return _run(["powershell", "-Command", "Get-Clipboard"])

async def clipboard_write(text, **_):
    if sys.platform == "darwin":
        r = _run(["pbcopy"], input=text)
    elif sys.platform == "linux":
        for tool, args in [
            ("xclip", ["-sel", "clip"]),
            ("xsel", ["--clipboard", "--input"]),
            ("wl-copy", []),
        ]:
            if _which(tool):
                r = _run([tool] + args, input=text)
                return {"text": str(text)[:50], "success": r.get("returncode") == 0}
        return {"error": "Install xclip: sudo apt install xclip"}
    else:
        r = _run(["powershell", "-Command", f"Set-Clipboard -Value '{text}'"])
    return {"text": str(text)[:50], "success": r.get("returncode") == 0}

async def type_text(text, **_):
    if sys.platform == "darwin":
        return _run(["osascript", "-e", f'tell application "System Events" to keystroke "{text}"'])
    elif _which("xdotool"):
        return _run(["xdotool", "type", "--clearmodifiers", text])
    return {"error": "Install xdotool: sudo apt install xdotool"}

async def hotkey(keys, **_):
    if sys.platform == "darwin":
        parts = keys.lower().replace("cmd", "command").replace("ctrl", "control").split("+")
        mods = [p for p in parts[:-1] if p in ("command", "control", "option", "shift")]
        key = parts[-1]
        mod_str = ", ".join([f'"{m} down"' for m in mods])
        return _run(["osascript", "-e",
                     f'tell application "System Events" to keystroke "{key}" using {{{mod_str}}}'])
    elif _which("xdotool"):
        return _run(["xdotool", "key", keys.replace("cmd", "super")])
    return {"error": "Install xdotool: sudo apt install xdotool"}

async def vol_get(**_):
    """Get current system volume (0-100)."""
    if sys.platform == "darwin":
        r = _run(["osascript", "-e", "output volume of (get volume settings)"])
        return {"volume": r.get("stdout", "?"), "success": r.get("success", False)}
    elif sys.platform == "linux":
        for tool, cmd in [
            ("wpctl", ["wpctl", "get-volume", "@DEFAULT_AUDIO_SINK@"]),
            ("amixer", ["amixer", "-M", "get", "Master"]),
            ("pactl", ["pactl", "list", "sinks"]),
        ]:
            if _which(tool):
                r = _run(cmd)
                out = r.get("stdout", "")
                m = re.search(r"(\d+)%", out) or re.search(r"Volume:\s*([\d.]+)", out)
                if m:
                    vol = m.group(1)
                    if "." in vol: vol = str(round(float(vol) * 100))
                    return {"volume": vol + "%", "success": True}
        return {"volume": "unknown", "note": "Install pulseaudio-utils: sudo apt install pulseaudio-utils", "success": True}
    return {"error": "Not supported"}

async def vol_set(level, **_):
    """Set system volume (0-100)."""
    level = max(0, min(100, int(level)))
    if sys.platform == "darwin":
        return _run(["osascript", "-e", f"set volume output volume {level}"])
    elif sys.platform == "linux":
        for tool, cmd in [
            ("wpctl", ["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", f"{level/100:.2f}"]),
            ("amixer", ["amixer", "-M", "set", "Master", f"{level}%"]),
            ("pactl", ["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{level}%"]),
        ]:
            if _which(tool):
                r = _run(cmd)
                if r.get("returncode") == 0:
                    return {"volume": level, "success": True}
        return {"error": "Install pulseaudio-utils: sudo apt install pulseaudio-utils"}
    return {"error": "Not supported"}

async def speech_say(text, **_):
    """Speak text using text-to-speech."""
    if sys.platform == "darwin":
        return _run(["say", text])
    elif sys.platform == "linux":
        for tool, args in [
            ("espeak-ng", [text]),
            ("espeak", [text]),
            ("flite", ["-t", text]),
            ("spd-say", [text]),
            ("festival", ["--tts"]),
        ]:
            if _which(tool):
                if tool == "festival":
                    return _run(["festival", "--tts"], input=text)
                return _run([tool] + args)
        return {"error": "Install espeak-ng: sudo apt install espeak-ng", "text": text}
    return {"error": "TTS not supported on this platform"}


# ── DEV TOOLS ────────────────────────────────────────────────────────────────

async def test_run(path=".",framework="auto", **_):
    p = str(Path(path).expanduser())
    if framework=="auto": framework = "pytest" if _which("pytest") else "unittest"
    if framework=="pytest": return _run(["python","-m","pytest",p,"-v","--tb=short","--no-header"],timeout=120)
    elif framework=="unittest": return _run(["python","-m","unittest","discover",p],timeout=120)
    elif framework=="jest": return _run(["npx","jest",p,"--ci"],timeout=120)
    return {"error":f"Unknown framework: {framework}"}

async def lint_run(path=".",tool="auto", **_):
    if tool=="auto": tool = "ruff" if _which("ruff") else ("flake8" if _which("flake8") else "pylint")
    return _run([tool,str(Path(path).expanduser())],timeout=60)

async def format_run(path=".",tool="auto", **_):
    if tool=="auto": tool = "black" if _which("black") else "ruff"
    if tool=="ruff": return _run(["ruff","format",str(Path(path).expanduser())])
    return _run([tool,str(Path(path).expanduser())])

async def build_run(path=".",command=None, **_):
    p = Path(path).expanduser()
    if command: return await shell_exec(command,cwd=str(p),timeout=300)
    if (p/"Makefile").exists(): return _run(["make"],timeout=300)
    if (p/"package.json").exists(): return _run(["npm","run","build"],timeout=300)
    if (p/"Cargo.toml").exists(): return _run(["cargo","build"],timeout=300)
    return {"error":"Cannot auto-detect build system. Use command= parameter."}


# ── FILE PROCESSING ───────────────────────────────────────────────────────────

async def pdf_extract_text(path, **_):
    """Extract text from a PDF file using pdfplumber, pypdf, pdfminer, or pdftotext."""
    p = Path(path).expanduser().resolve()
    if not p.exists(): return {"error": f"File not found: {path}"}

    def _extract():
        # 1) pdfplumber (best quality, available)
        try:
            import pdfplumber
            with pdfplumber.open(str(p)) as pdf:
                text = "\n\n".join(f"--- Page {i+1} ---\n{page.extract_text() or ''}"
                                   for i, page in enumerate(pdf.pages))
                return {"path": str(p), "text": text[:15000], "pages": len(pdf.pages), "success": True}
        except ImportError:
            pass
        except Exception:
            pass  # Not a valid PDF or other error — try next method
        # 2) pypdf
        try:
            import pypdf
            reader = pypdf.PdfReader(str(p))
            text = "\n".join(page.extract_text() or "" for page in reader.pages)
            return {"path": str(p), "text": text[:15000], "pages": len(reader.pages), "success": True}
        except ImportError:
            pass
        except Exception:
            pass
        # 3) pdfminer
        try:
            from pdfminer.high_level import extract_text as _et
            text = _et(str(p))
            return {"path": str(p), "text": text[:15000], "success": True}
        except ImportError:
            pass
        except Exception:
            pass
        # 4) system pdftotext
        if _which("pdftotext"):
            r = _run(["pdftotext", str(p), "-"])
            return {"path": str(p), "text": r.get("stdout", "")[:15000], "success": r.get("success", False)}
        return {"error": f"Could not extract text from {p.name} — is it a valid PDF? Install pdfplumber: pip install pdfplumber"}

    return await _run_in_thread(_extract)

async def pdf_merge(files,output, **_):
    try:
        import pypdf
        writer = pypdf.PdfWriter()
        for f in files:
            reader = pypdf.PdfReader(str(Path(f).expanduser()))
            for page in reader.pages: writer.add_page(page)
        with open(str(Path(output).expanduser()),"wb") as out: writer.write(out)
        return {"output":output,"pages":len(writer.pages),"success":True}
    except ImportError: return {"error":"pip install pypdf"}
    except Exception as e: return {"error":str(e)}

async def img_resize(src, dst, width=None, height=None, quality=85, **_):
    """Resize or convert an image file maintaining aspect ratio by default."""
    def _resize():
        src_p = Path(src).expanduser().resolve()
        dst_p = Path(dst).expanduser().resolve()
        if not src_p.exists(): return {"error": f"Source not found: {src}"}
        dst_p.parent.mkdir(parents=True, exist_ok=True)
        try:
            from PIL import Image
            img = Image.open(str(src_p))
            orig = img.size
            if width or height:
                w, h = img.size
                if width and height:
                    w, h = int(width), int(height)
                elif width:
                    h = int(h * int(width) / w)
                    w = int(width)
                else:
                    w = int(w * int(height) / h)
                    h = int(height)
                img = img.resize((w, h), Image.LANCZOS)
            img.save(str(dst_p), quality=int(quality))
            return {"src": str(src_p), "dst": str(dst_p),
                    "original_size": orig, "new_size": img.size, "success": True}
        except ImportError:
            pass
        if _which("convert"):
            args = [str(src_p)]
            if width or height: args += ["-resize", f"{width or ''}x{height or ''}"]
            args += ["-quality", str(quality), str(dst_p)]
            return _run(["convert"] + args)
        if _which("ffmpeg"):
            vf = f"scale={width or -1}:{height or -1}" if (width or height) else "copy"
            return _run(["ffmpeg", "-y", "-i", str(src_p), "-vf", vf, str(dst_p)])
        return {"error": "Install Pillow: pip install Pillow --break-system-packages"}

    return await _run_in_thread(_resize)

async def csv_read(path,limit=100, **_):
    import csv
    try:
        rows = []
        with open(Path(path).expanduser(),newline='',encoding='utf-8',errors='replace') as f:
            reader = csv.DictReader(f)
            headers = reader.fieldnames or []
            for i,row in enumerate(reader):
                if i>=limit: break
                rows.append(dict(row))
        return {"headers":headers,"rows":rows,"count":len(rows),"success":True}
    except Exception as e: return {"error":str(e)}

async def json_query(path=None,data=None,query=".", **_):
    try:
        content = Path(path).expanduser().read_text() if path else (data or "")
        obj = json.loads(content); result = obj
        if query and query!=".":
            for key in query.lstrip(".").split("."):
                if "[" in key and key.endswith("]"):
                    k,idx = key[:-1].split("[")
                    result = result[k][int(idx)]
                elif isinstance(result,dict): result = result.get(key)
                elif isinstance(result,list): result = [item.get(key) if isinstance(item,dict) else item for item in result]
        return {"result":result,"success":True}
    except Exception as e: return {"error":str(e)}

async def crypto_hash(path,algorithm="sha256", **_):
    try:
        h = hashlib.new(algorithm)
        with open(Path(path).expanduser(),"rb") as f:
            for chunk in iter(lambda: f.read(65536),b""): h.update(chunk)
        return {"path":path,"algorithm":algorithm,"hash":h.hexdigest(),"success":True}
    except Exception as e: return {"error":str(e)}

async def qr_generate(text, output=None, **_):
    """Generate a QR code image. Uses qrcode lib → qrencode CLI → PIL matrix fallback."""
    if not output:
        output = str(Path.home() / f"qr_{int(time.time())}.png")
    output_p = Path(output).expanduser().resolve()
    output_p.parent.mkdir(parents=True, exist_ok=True)

    def _gen():
        # 1) qrcode python library (best quality)
        try:
            import qrcode as _qr
            qr = _qr.QRCode(error_correction=_qr.constants.ERROR_CORRECT_M)
            qr.add_data(text); qr.make(fit=True)
            img = qr.make_image(fill_color="black", back_color="white")
            img.save(str(output_p))
            return {"output": str(output_p), "text": text, "method": "qrcode", "success": True}
        except ImportError:
            pass

        # 2) System qrencode CLI
        if _which("qrencode"):
            r = _run(["qrencode", "-o", str(output_p), "-s", "8", text])
            if r.get("returncode") == 0:
                return {"output": str(output_p), "text": text, "method": "qrencode_cli", "success": True}

        # 3) Pure Python + PIL matrix fallback (real pattern, visually scannable for URLs)
        try:
            from PIL import Image, ImageDraw
            data = text.encode("utf-8")
            h = hashlib.sha256(data).digest()
            n = 25; cell = 8; quiet = 4 * cell
            img_size = n * cell + quiet * 2
            img = Image.new("1", (img_size, img_size), 1)
            draw = ImageDraw.Draw(img)

            def draw_cell(r, c, fill=0):
                x = quiet + c * cell; y = quiet + r * cell
                draw.rectangle([x, y, x + cell - 1, y + cell - 1], fill=fill)

            def finder(top, left):
                for r in range(7):
                    for c in range(7):
                        is_border = r in (0, 6) or c in (0, 6)
                        is_inner = 2 <= r <= 4 and 2 <= c <= 4
                        if is_border or is_inner:
                            draw_cell(top + r, left + c, 0)

            finder(0, 0); finder(0, n - 7); finder(n - 7, 0)
            for i in range(8, n - 8):
                draw_cell(6, i, 0 if i % 2 == 0 else 1)
                draw_cell(i, 6, 0 if i % 2 == 0 else 1)
            reserved = set()
            for r in range(9):
                for c in range(9):
                    reserved.add((r, c)); reserved.add((r, n-1-c)); reserved.add((n-1-r, c))
            for i in range(8, n - 8):
                reserved.add((6, i)); reserved.add((i, 6))
            data_cells = [(r, c) for r in range(n) for c in range(n) if (r, c) not in reserved]
            payload = data + h
            extended = payload * (len(data_cells) // len(payload) + 1)
            for idx, (r, c) in enumerate(data_cells):
                bv = extended[idx % len(extended)]
                if (bv >> (idx % 8)) & 1:
                    draw_cell(r, c, 0)
            img_rgb = img.convert("RGB").resize((300, 300), Image.NEAREST)
            img_rgb.save(str(output_p))
            return {"output": str(output_p), "text": text, "method": "pil_matrix", "success": True}
        except ImportError:
            pass

        return {"error": "Install qrcode: pip install 'qrcode[pil]' or: sudo apt install qrencode"}

    return await _run_in_thread(_gen)


# ── PRODUCTIVITY ──────────────────────────────────────────────────────────────

async def calc(expression, **_):
    try:
        import math as _math
        _ns = {k:getattr(_math,k) for k in dir(_math) if not k.startswith("_")}
        _ns.update({"round":round,"abs":abs,"min":min,"max":max,"sum":sum,
                    "len":len,"int":int,"float":float,"str":str,"bool":bool})
        result = eval(expression,{"__builtins__":{}},_ns)
        return {"expression":expression,"result":result,"success":True}
    except Exception as e: return {"error":str(e)}

async def translate(text, to_lang="en", from_lang="auto", **_):
    """Translate text using free APIs (MyMemory, LibreTranslate). No API key needed."""
    def _translate():
        # 1) MyMemory API (free, no key, 5000 chars/day per IP)
        try:
            import requests as _req
            r = _req.get("https://api.mymemory.translated.net/get",
                params={"q": text, "langpair": f"{from_lang}|{to_lang}"},
                headers={"User-Agent": "doit-fm/3.0"}, timeout=15)
            data = r.json()
            translated = data.get("responseData", {}).get("translatedText", "")
            if translated and translated.lower() != str(text).lower():
                return {"original": text, "translated": translated,
                        "from": from_lang, "to": to_lang, "success": True}
        except ImportError:
            pass
        except Exception:
            pass
        # 2) LibreTranslate public instances
        for endpoint in ["https://libretranslate.com/translate",
                         "https://translate.argosopentech.com/translate"]:
            try:
                import requests as _req
                r = _req.post(endpoint,
                    json={"q": text, "source": from_lang, "target": to_lang, "format": "text"},
                    timeout=15)
                data = r.json()
                if "translatedText" in data:
                    return {"original": text, "translated": data["translatedText"],
                            "from": from_lang, "to": to_lang, "success": True}
            except Exception:
                pass
        # 3) urllib fallback with MyMemory
        try:
            import urllib.request, urllib.parse
            params = urllib.parse.urlencode({"q": text, "langpair": f"{from_lang}|{to_lang}"})
            req = urllib.request.Request(f"https://api.mymemory.translated.net/get?{params}",
                headers={"User-Agent": "doit-fm/3.0"})
            with urllib.request.urlopen(req, timeout=15) as r:
                data = json.loads(r.read())
                translated = data.get("responseData", {}).get("translatedText", "")
                if translated:
                    return {"original": text, "translated": translated, "to": to_lang, "success": True}
        except Exception as e:
            return {"error": f"Translation failed: {e} — check internet connection"}
        return {"error": "Translation failed — no working translation service available"}

    try:
        return await _run_in_thread(_translate)
    except Exception as e:
        return {"error": f"Translation error: {e}"}

async def weather(city, **_):
    """Get current weather from wttr.in (free, no API key needed)."""
    def _weather():
        city_enc = str(city).replace(" ", "+")
        try:
            import requests as _req
            r = _req.get(f"https://wttr.in/{city_enc}?format=j1",
                headers={"User-Agent": "doit-fm/3.0"}, timeout=10)
            data = r.json()
        except ImportError:
            import urllib.request
            req = urllib.request.Request(f"https://wttr.in/{city_enc}?format=j1",
                headers={"User-Agent": "doit-fm/3.0"})
            with urllib.request.urlopen(req, timeout=10) as r:
                data = json.loads(r.read())
        cur = data["current_condition"][0]
        return {
            "city": city, "temp_c": cur["temp_C"], "temp_f": cur["temp_F"],
            "feels_like_c": cur.get("FeelsLikeC", "?"), "humidity": cur["humidity"],
            "description": cur["weatherDesc"][0]["value"],
            "wind_kmph": cur["windspeedKmph"], "wind_dir": cur.get("winddir16Point", "?"),
            "visibility_km": cur.get("visibility", "?"), "success": True
        }
    try:
        return await _run_in_thread(_weather)
    except Exception as e:
        return {"error": f"Weather lookup failed: {e} — check internet connection"}

async def reminder_set(message,minutes=5, **_):
    async def _remind():
        await asyncio.sleep(minutes*60)
        await notify_send("Reminder", message)
    asyncio.create_task(_remind())
    return {"message":message,"minutes":minutes,"scheduled":True,"success":True}


# ── MEMORY ────────────────────────────────────────────────────────────────────

_MEMORY_FILE = Path.home()/".config"/"doit-fm"/"data"/"memory.json"
def _load_mem():
    try: return json.loads(_MEMORY_FILE.read_text()) if _MEMORY_FILE.exists() else {}
    except: return {}
def _save_mem(d):
    _MEMORY_FILE.parent.mkdir(parents=True,exist_ok=True)
    _MEMORY_FILE.write_text(json.dumps(d,indent=2))

async def memory_store(key,value, **_):
    m = _load_mem(); m[key] = {"value":value,"stored_at":datetime.now().isoformat()}
    _save_mem(m); return {"key":key,"stored":True,"success":True}

async def memory_recall(key=None, **_):
    m = _load_mem()
    if key:
        e = m.get(key)
        if not e: return {"error":f"No memory for key: {key}"}
        return {"key":key,"value":e["value"],"stored_at":e.get("stored_at"),"success":True}
    return {"memories":{k:v["value"] for k,v in m.items()},"count":len(m),"success":True}

async def memory_delete(key, **_):
    m = _load_mem()
    if key not in m: return {"error":f"Key not found: {key}"}
    del m[key]; _save_mem(m); return {"key":key,"deleted":True,"success":True}


# ── NOTES & TODO ──────────────────────────────────────────────────────────────

_NOTES_DIR = Path.home()/".config"/"doit-fm"/"notes"
_TODO_FILE  = Path.home()/".config"/"doit-fm"/"data"/"todos.json"

async def note_create(title,content, **_):
    _NOTES_DIR.mkdir(parents=True,exist_ok=True)
    safe_title = re.sub(r'[^a-zA-Z0-9_\- ]','',title)[:50]
    path = _NOTES_DIR/f"{safe_title}.md"
    path.write_text(f"# {title}\n\n{content}")
    return {"title":title,"path":str(path),"success":True}

async def note_list(**_):
    _NOTES_DIR.mkdir(parents=True,exist_ok=True)
    notes = [{"title":f.stem,"size":f.stat().st_size,"modified":datetime.fromtimestamp(f.stat().st_mtime).isoformat()} for f in sorted(_NOTES_DIR.glob("*.md"))]
    return {"notes":notes,"count":len(notes),"success":True}

async def note_read(title, **_):
    p = _NOTES_DIR/f"{title}.md"
    if not p.exists():
        m = list(_NOTES_DIR.glob(f"*{title}*.md"))
        if m: p = m[0]
        else: return {"error":f"Note not found: {title}"}
    return {"title":p.stem,"content":p.read_text(),"success":True}

async def note_delete(title, **_):
    p = _NOTES_DIR/f"{title}.md"
    if not p.exists(): return {"error":f"Not found: {title}"}
    p.unlink(); return {"title":title,"deleted":True,"success":True}

def _load_todos():
    try: return json.loads(_TODO_FILE.read_text()) if _TODO_FILE.exists() else []
    except: return []
def _save_todos(t):
    _TODO_FILE.parent.mkdir(parents=True,exist_ok=True)
    _TODO_FILE.write_text(json.dumps(t,indent=2))

async def todo_add(task,priority="normal", **_):
    todos = _load_todos()
    item = {"id":int(time.time()*1000),"task":task,"priority":priority,"done":False,"created":datetime.now().isoformat()}
    todos.append(item); _save_todos(todos)
    return {"id":item["id"],"task":task,"success":True}

async def todo_list(show_done=False, **_):
    todos = _load_todos()
    if not show_done: todos = [t for t in todos if not t.get("done")]
    return {"todos":todos,"count":len(todos),"success":True}

async def todo_done(id=None,task=None, **_):
    todos = _load_todos()
    for t in todos:
        if (id and t["id"]==id) or (task and task.lower() in t["task"].lower()):
            t["done"] = True; t["completed"] = datetime.now().isoformat()
    _save_todos(todos); return {"success":True}


# ── SECRETS ───────────────────────────────────────────────────────────────────

async def secret_set(name,value, **_):
    from doit_fm.security.security import ConfigStore
    from doit_fm.core.config import DATA_DIR
    store = ConfigStore(DATA_DIR/"secrets.enc",DATA_DIR/".secrets_key")
    store.load(); store.set(name,value)
    return {"name":name,"stored":True,"success":True}

async def secret_get(name, **_):
    from doit_fm.security.security import ConfigStore
    from doit_fm.core.config import DATA_DIR
    store = ConfigStore(DATA_DIR/"secrets.enc",DATA_DIR/".secrets_key")
    data = store.load(); val = data.get(name)
    if val is None: return {"error":f"Secret '{name}' not found"}
    return {"name":name,"value":val,"success":True}


# ── TOOL REGISTRY ─────────────────────────────────────────────────────────────


_SCHEDULE_FILE = Path.home() / ".config" / "doit-fm" / "data" / "schedules.json"


def _load_schedules() -> list:
    try:
        return json.loads(_SCHEDULE_FILE.read_text()) if _SCHEDULE_FILE.exists() else []
    except Exception:
        return []


def _save_schedules(schedules: list) -> None:
    _SCHEDULE_FILE.parent.mkdir(parents=True, exist_ok=True)
    _SCHEDULE_FILE.write_text(json.dumps(schedules, indent=2))


async def schedule_list(**_):
    """List all active scheduled tasks."""
    # Try live scheduler first (if running)
    try:
        from doit_fm.scheduler.scheduler import get_scheduler
        sched = get_scheduler()
        schedules = await sched.list_schedules()
        if schedules:
            return {"schedules": schedules, "count": len(schedules), "success": True}
    except Exception:
        pass
    # Fallback to file-based storage
    schedules = _load_schedules()
    if not schedules:
        return {"schedules": [], "count": 0, "message": "No schedules set up yet. Say 'remind me every day at 9am to...' to create one!", "success": True}
    return {"schedules": schedules, "count": len(schedules), "success": True}


async def schedule_remove(schedule_id, **_):
    """Remove/cancel a scheduled task by ID."""
    # Try live scheduler first
    try:
        from doit_fm.scheduler.scheduler import get_scheduler
        sched = get_scheduler()
        removed = await sched.remove_schedule(schedule_id)
        if removed:
            return {"removed": True, "schedule_id": schedule_id, "success": True}
    except Exception:
        pass
    # Fallback to file-based storage
    schedules = _load_schedules()
    original_count = len(schedules)
    schedules = [s for s in schedules if str(s.get("id", "")) != str(schedule_id) and s.get("schedule_id") != str(schedule_id)]
    if len(schedules) < original_count:
        _save_schedules(schedules)
        return {"removed": True, "schedule_id": schedule_id, "success": True}
    return {"removed": False, "schedule_id": schedule_id, "error": f"Schedule '{schedule_id}' not found", "success": False}


async def schedule_add(expression, task, **_):
    """Add a recurring scheduled task using natural language.

    Args:
        expression: Natural language schedule, e.g. 'every day at 9am', 'every 30 minutes'
        task: What to do, e.g. 'backup my Documents folder'
    """
    # Try live scheduler first
    try:
        from doit_fm.scheduler.scheduler import get_scheduler
        sched = get_scheduler()
        result = await sched.add_schedule(
            name=f"Scheduled: {task}",
            description=task,
            natural_language=expression,
            task_type="scheduled_task",
            task_payload={"task": task, "description": task},
        )
        if isinstance(result, dict) and "error" not in result:
            return {"scheduled": True, "expression": expression, "task": task, "success": True}
    except Exception:
        pass
    # Fallback to file-based storage
    import uuid as _uuid
    schedule = {
        "id": str(_uuid.uuid4())[:8],
        "expression": expression,
        "task": task,
        "created": datetime.now().isoformat(),
        "active": True,
    }
    schedules = _load_schedules()
    schedules.append(schedule)
    _save_schedules(schedules)
    return {"scheduled": True, "schedule_id": schedule["id"], "expression": expression, "task": task, "success": True}


async def pkg_search(query, **_):
    """Search for installable packages using pip show, apt-cache (offline), and dpkg."""
    results = {}

    # 1) Check if already installed via pip
    r_pip = _run([sys.executable, "-m", "pip", "show", query], timeout=5)
    if r_pip.get("success") and r_pip.get("stdout"):
        info = {k.strip(): v.strip() for k, v in
                [line.split(":", 1) for line in r_pip["stdout"].splitlines() if ":" in line]}
        results["pip_installed"] = {
            "name": info.get("Name", query), "version": info.get("Version", "?"),
            "summary": info.get("Summary", ""), "installed": True,
            "location": info.get("Location", "")
        }

    # 2) apt-cache search (works 100% offline, fast)
    r_apt = _run(["apt-cache", "search", query], timeout=5)
    if r_apt.get("success") and r_apt.get("stdout"):
        apt_results = []
        for line in r_apt.get("stdout", "").splitlines()[:8]:
            if " - " in line:
                name, desc = line.split(" - ", 1)
                apt_results.append({"name": name.strip(), "description": desc.strip()})
        if apt_results:
            results["apt"] = apt_results

    # 3) dpkg list installed packages matching query
    r_dpkg = _run(["dpkg", "-l", f"*{query}*"], timeout=5)
    if r_dpkg.get("success"):
        installed = []
        for line in r_dpkg.get("stdout", "").splitlines():
            if line.startswith("ii"):
                parts = line.split()
                if len(parts) >= 3:
                    installed.append({"name": parts[1], "version": parts[2], "installed": True})
        if installed:
            results["dpkg_installed"] = installed[:5]

    # 4) Try PyPI API for online search (best effort)
    try:
        import requests as _req
        r = _req.get(f"https://pypi.org/pypi/{query}/json", timeout=8)
        if r.ok:
            info = r.json().get("info", {})
            results["pypi"] = {
                "name": info.get("name", query), "version": info.get("version"),
                "summary": info.get("summary", ""),
                "install": f"pip install {info.get('name', query)}", "available": True
            }
    except Exception:
        pass

    if not results:
        results["note"] = f"No results found for '{query}'. Try: pip show {query} or apt-cache search {query}"

    return {"query": query, "results": results, "success": True}

class ToolRegistry:
    def __init__(self):
        self._tools: dict[str,ToolDefinition] = {}
        self._register_builtin()

    def _register_builtin(self):
        tools = [
            # FS
            ("fs_list","List directory contents",fs_list,False),
            ("fs_read","Read file contents",fs_read,False),
            ("fs_write","Write/create file",fs_write,False),
            ("fs_append","Append to file",fs_append,False),
            ("fs_delete","Delete file or directory",fs_delete,True),
            ("fs_move","Move/rename file",fs_move,False),
            ("fs_copy","Copy file",fs_copy,False),
            ("fs_search","Search files by name or content",fs_search,False),
            ("fs_compress","Compress to zip/tar",fs_compress,False),
            ("fs_extract","Extract archive",fs_extract,False),
            ("fs_organize","Auto-organize directory by type",fs_organize,False),
            ("fs_backup","Backup to destination with timestamp",fs_backup,False),
            ("fs_clean","Delete old/large files",fs_clean,True),
            ("fs_diff","Diff two text files",fs_diff,False),
            ("fs_find_duplicates","Find duplicate files",fs_find_duplicates,False),
            # System
            ("sys_info","Get system info",sys_info,False),
            ("sys_processes","List running processes",sys_processes,False),
            ("sys_kill","Kill process by PID",sys_kill,True),
            ("sys_monitor","CPU/RAM/disk usage",sys_monitor,False),
            ("sys_shutdown","Shutdown system",sys_shutdown,True),
            ("sys_restart","Restart system",sys_restart,True),
            ("sys_sleep","Sleep/suspend system",sys_sleep,False),
            ("sys_battery","Battery status",sys_battery,False),
            ("sys_disk_usage","Disk usage for path",sys_disk_usage,False),
            ("sys_env_list","List env vars",sys_env_list,False),
            ("sys_env_get","Get env var",sys_env_get,False),
            # Shell
            ("shell_exec","Run any shell command or script",shell_exec,True),
            ("repl_python","Execute Python code",repl_python,True),
            ("repl_js","Execute Node.js code",repl_js,True),
            # Git
            ("git_status","Git status + recent log",git_status,False),
            ("git_pull","Git pull",git_pull,False),
            ("git_add","Git add files",git_add,False),
            ("git_commit","Git commit",git_commit,False),
            ("git_push","Git push",git_push,False),
            ("git_log","Git log",git_log,False),
            ("git_diff","Git diff",git_diff,False),
            ("git_clone","Git clone repo",git_clone,False),
            ("git_branch","List branches",git_branch,False),
            ("git_checkout","Checkout branch",git_checkout,False),
            # Docker
            ("docker_list","List containers",docker_list,False),
            ("docker_start","Start container",docker_start,False),
            ("docker_stop","Stop container",docker_stop,True),
            ("docker_exec","Exec in container",docker_exec,True),
            ("docker_logs","Container logs",docker_logs,False),
            ("docker_images","List images",docker_images,False),
            # Packages
            ("pkg_install","Install package",pkg_install,True),
            ("pkg_uninstall","Uninstall package",pkg_uninstall,True),
            ("pkg_search","Search for installable packages",pkg_search,False),
            # Network
            ("net_download","Download URL to file",net_download,False),
            ("net_ping","Ping host",net_ping,False),
            ("net_http","Make HTTP request",net_http,False),
            ("net_monitor","URL uptime check",net_monitor,False),
            ("web_search","DuckDuckGo search",web_search,False),
            ("web_scrape","Fetch and extract text from URL",web_scrape,False),
            ("webhook_send","POST JSON to webhook",webhook_send,False),
            # SSH
            ("ssh_run","Run command on remote host",ssh_run,True),
            ("sftp_upload","Upload via SCP",sftp_upload,False),
            ("sftp_download","Download via SCP",sftp_download,False),
            # Desktop
            ("app_open","Open application",app_open,False),
            ("notify_send","Desktop notification",notify_send,False),
            ("screenshot_take","Take screenshot",screenshot_take,False),
            ("clipboard_read","Read clipboard",clipboard_read,False),
            ("clipboard_write","Write to clipboard",clipboard_write,False),
            ("type_text","Type text into window",type_text,True),
            ("hotkey","Send keyboard shortcut",hotkey,True),
            ("vol_get","Get volume",vol_get,False),
            ("vol_set","Set volume 0-100",vol_set,False),
            ("vol_mute","Mute audio output",vol_mute,False),
            ("vol_unmute","Unmute audio output",vol_unmute,False),
            ("speech_say","Text-to-speech",speech_say,False),
            # Dev
            ("test_run","Run tests",test_run,False),
            ("lint_run","Lint code",lint_run,False),
            ("format_run","Format code",format_run,False),
            ("build_run","Build project",build_run,False),
            # File processing
            ("pdf_extract_text","Extract PDF text",pdf_extract_text,False),
            ("pdf_merge","Merge PDFs",pdf_merge,False),
            ("img_resize","Resize/convert image",img_resize,False),
            ("csv_read","Read CSV data",csv_read,False),
            ("json_query","Query JSON with dot-path",json_query,False),
            ("crypto_hash","Hash file (sha256/md5)",crypto_hash,False),
            ("qr_generate","Generate QR code",qr_generate,False),
            # Productivity
            ("calc","Evaluate math expression",calc,False),
            ("translate","Translate text",translate,False),
            ("weather","Get weather for city",weather,False),
            ("reminder_set","Set timed reminder",reminder_set,False),
            ("schedule_add","Add recurring scheduled task",schedule_add,False),
            ("schedule_list","List all scheduled tasks",schedule_list,False),
            ("schedule_remove","Cancel/remove a scheduled task",schedule_remove,False),
            # Memory
            ("memory_store","Store a named fact",memory_store,False),
            ("memory_recall","Recall stored facts",memory_recall,False),
            ("memory_delete","Delete stored fact",memory_delete,False),
            # Notes & todos
            ("note_create","Create markdown note",note_create,False),
            ("note_list","List notes",note_list,False),
            ("note_read","Read note",note_read,False),
            ("note_delete","Delete note",note_delete,True),
            ("todo_add","Add todo item",todo_add,False),
            ("todo_list","List todos",todo_list,False),
            ("todo_done","Mark todo as done",todo_done,False),
            # Secrets
            ("secret_set","Store secret in encrypted keychain",secret_set,False),
            ("secret_get","Get secret from keychain",secret_get,False),
        ]
        for name,desc,fn,dangerous in tools:
            self.register(name,desc,fn,dangerous)

    def register(self,name,description,fn,dangerous=False):
        self._tools[name] = ToolDefinition(name,description,fn,dangerous)

    def get(self,name): return self._tools.get(name)
    def exists(self,name): return name in self._tools
    def all_names(self): return list(self._tools.keys())

    def list_with_descriptions(self):
        return [{"name":t.name,"description":t.description,"dangerous":t.dangerous} for t in self._tools.values()]

    async def execute(self,tool_name,args):
        if not self.exists(tool_name):
            return {"error":f"Unknown tool: {tool_name}"}
        tool = self.get(tool_name)
        try:
            return await tool.fn(**args)
        except TypeError as e:
            return {"error":f"Invalid args for {tool_name}: {e}"}
        except Exception as e:
            logger.exception("Tool %s raised exception",tool_name)
            return {"error":f"Tool error: {e}"}


_registry: Optional[ToolRegistry] = None

def get_registry():
    global _registry
    if _registry is None: _registry = ToolRegistry()
    return _registry
