#!/bin/bash
# Monit self-heartbeat: Telegram health report with environment integrity.
# Deployed to /usr/local/bin/obdpy-monit-heartbeat.sh
# Called by Monit's "check program" as uid "tca" every 10 cycles (~20 min).
#
# Layout: stable items top, frequently-monitored items bottom.
# Bottom = what you see first in Telegram (IM recency logic).

set -eo pipefail
trap '' PIPE

ENV_FILE="/home/tca/opendeviationbar-py/deploy/opendeviationbar.local.env"
GIT_ENV="/home/tca/opendeviationbar-py/deploy/opendeviationbar.env"
SIDECAR_ENV="/home/tca/opendeviationbar-py/deploy/sidecar.env"
VENV="/home/tca/opendeviationbar-py/.venv"

[ -f "$ENV_FILE" ] && source "$ENV_FILE"
[ -f "$GIT_ENV" ] && source "$GIT_ENV"

if [ -z "${OPENDEVIATIONBAR_TELEGRAM_TOKEN:-}" ] || [ -z "${OPENDEVIATIONBAR_TELEGRAM_CHAT_ID:-}" ]; then
    echo "ERROR: Telegram credentials not set" >&2
    exit 1
fi

UUID=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c 'import uuid; print(uuid.uuid4())')
PKG_VERSION=$("$VENV/bin/python3" -c 'from opendeviationbar import __version__; print(__version__)' 2>/dev/null || echo "unknown")

# Collect system data into temp files (avoids SIGPIPE with heredoc)
# Use $HOME — /tmp and /var/tmp both go read-only during heavy CH merge I/O.
_TMPDIR="${HOME:=/home/tca}/.cache/opendeviationbar/monit-tmp"
mkdir -p "$_TMPDIR"
_TMP_RSS=$(mktemp "$_TMPDIR/hb.XXXXXX")
_TMP_STORM=$(mktemp "$_TMPDIR/hb.XXXXXX")
_TMP_MEMINFO=$(mktemp "$_TMPDIR/hb.XXXXXX")
trap 'rm -f "$_TMP_RSS" "$_TMP_STORM" "$_TMP_MEMINFO"' EXIT

# Top RSS aggregated by user|command
ps axo rss,user,comm --no-headers 2>/dev/null \
    | awk '{a[$2"|"$3]+=$1} END{for(k in a) printf "%d %s\n", a[k], k}' \
    | sort -rn | awk 'NR<=8' > "$_TMP_RSS"

# Userspace process counts (exclude kernel threads: VSZ=0)
ps axo user,vsz --no-headers 2>/dev/null | awk '$2>0{print $1}' | sort | uniq -c | sort -rn | awk 'NR<=5' > "$_TMP_STORM"

# Structured /proc/meminfo for memory classification
cat /proc/meminfo > "$_TMP_MEMINFO"

REPORT=$("$VENV/bin/python3" - "$UUID" "$PKG_VERSION" "$ENV_FILE" "$GIT_ENV" "$SIDECAR_ENV" "$_TMP_RSS" "$_TMP_STORM" "$_TMP_MEMINFO" << 'PYEOF'
import os
import sys
import socket
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from pathlib import Path

uuid = sys.argv[1]
pkg_version = sys.argv[2]
env_local = sys.argv[3]
env_git = sys.argv[4]
env_sidecar = sys.argv[5]
tmp_rss = sys.argv[6] if len(sys.argv) > 6 else ""
tmp_storm = sys.argv[7] if len(sys.argv) > 7 else ""
tmp_meminfo = sys.argv[8] if len(sys.argv) > 8 else ""

top_agg_raw = Path(tmp_rss).read_text() if tmp_rss and Path(tmp_rss).exists() else ""
proc_counts_raw = Path(tmp_storm).read_text() if tmp_storm and Path(tmp_storm).exists() else ""

# Parse /proc/meminfo into a dict (kB values)
meminfo = {}
if tmp_meminfo and Path(tmp_meminfo).exists():
    for line in Path(tmp_meminfo).read_text().splitlines():
        if ":" in line:
            key, val = line.split(":", 1)
            # Strip " kB" suffix and convert to int
            val = val.strip().replace(" kB", "")
            try:
                meminfo[key.strip()] = int(val)
            except ValueError:
                pass

# Friendly server name
_HOST_ALIASES = {"el02": "bigblack", "el01": "littleblack"}
hostname = socket.gethostname()
friendly = _HOST_ALIASES.get(hostname, hostname)
server_label = f"{friendly} ({hostname})" if friendly != hostname else hostname

try:
    with urllib.request.urlopen("http://localhost:2812/_status?format=xml", timeout=10) as resp:
        xml_data = resp.read()
    root = ET.fromstring(xml_data)
except Exception as e:
    print(f"\U0001f534 <b>MONIT</b> \u2014 {server_label}\nAPI error: {e}\n<code>{uuid}</code>")
    sys.exit(0)

server = root.find("server")
monit_uptime = int(server.findtext("uptime", "0"))
days, rem = divmod(monit_uptime, 86400)
hours, rem = divmod(rem, 3600)
mins = rem // 60
monit_up_str = f"{days}d {hours}h {mins}m" if days else f"{hours}h {mins}m"

WARN_DISK_PCT = 70.0
WARN_LOAD = 6.0
WARN_SWAP_PCT = 40.0
WARN_MEM_PCT = 70.0

# Parse all services into categorized buckets for reordered output.
# Order: stable top → dynamic bottom (IM recency = bottom seen first)
services = {}
failed = []
warned = []

for svc in root.findall("service"):
    name = svc.findtext("name", "?")
    status = int(svc.findtext("status", "-1"))
    svc_type = svc.get("type", "?")
    svc_warn = False
    detail = ""

    if svc_type == "3":  # Process
        pid = svc.findtext("pid", "?")
        mem_kb = int(svc.findtext("memory/kilobyte", "0"))
        mem_mb = mem_kb / 1024
        mem_pct = float(svc.findtext("memory/percent", "0"))
        cpu_pct = float(svc.findtext("cpu/percent", "0"))
        threads = svc.findtext("threads", "?")
        fds = svc.findtext("filedescriptors/open", "?")
        uptime_s = int(svc.findtext("uptime", "0"))
        ud, ur = divmod(uptime_s, 86400)
        uh, ur = divmod(ur, 3600)
        um = ur // 60
        up_str = f"{ud}d {uh}h" if ud else f"{uh}h {um}m"
        detail = f"  {mem_mb:.0f}MB | CPU {cpu_pct:.1f}% | {threads}T {fds}fd | up {up_str}"

    elif svc_type == "4":  # Host/port
        port_el = svc.find("port")
        if port_el is not None:
            rt = float(port_el.findtext("responsetime", "0"))
            port = port_el.findtext("portnumber", "?")
            proto = port_el.findtext("protocol", "?").upper()
            if rt > 1.0:
                svc_warn = True
            detail = f"  :{port} {proto} {rt*1000:.0f}ms"

    elif svc_type == "5":  # System
        sys_el = svc.find("system")
        if sys_el is not None:
            load1 = float(sys_el.findtext("load/avg01", "0"))
            load5 = float(sys_el.findtext("load/avg05", "0"))
            load15 = float(sys_el.findtext("load/avg15", "0"))
            mem_pct_f = float(sys_el.findtext("memory/percent", "0"))
            mem_kb = int(sys_el.findtext("memory/kilobyte", "0"))
            mem_gb = mem_kb / 1024 / 1024
            swap_pct_f = float(sys_el.findtext("swap/percent", "0"))
            cpu_usr = sys_el.findtext("cpu/user", "0")
            cpu_sys = sys_el.findtext("cpu/system", "0")
            if load1 > WARN_LOAD or mem_pct_f > WARN_MEM_PCT or swap_pct_f > WARN_SWAP_PCT:
                svc_warn = True
            detail = (
                f"  load [{load1:.1f} {load5:.1f} {load15:.1f}] | "
                f"RAM {mem_gb:.1f}GB ({mem_pct_f:.0f}%) | swap {swap_pct_f:.0f}%\n"
                f"  CPU {cpu_usr}% user / {cpu_sys}% sys"
            )

    elif svc_type == "0":  # Filesystem
        blk = svc.find("block")
        if blk is not None:
            used_pct = float(blk.findtext("percent", "0"))
            total_mb = float(blk.findtext("total", "0"))
            total_gb = total_mb / 1024
            free_gb = total_gb * (1 - used_pct / 100)
            if used_pct > WARN_DISK_PCT:
                svc_warn = True
            bar_len = 10
            filled = int(used_pct / 100 * bar_len)
            bar = "\u2588" * filled + "\u2591" * (bar_len - filled)
            detail = f"  [{bar}] {used_pct:.1f}% | {free_gb:.0f}GB free"

    elif svc_type == "7":  # Program
        prog = svc.find("program")
        if prog is not None:
            exit_code = prog.findtext("status", "?")
            output = prog.findtext("output", "").strip()[:60]
            detail = f"  exit={exit_code}" + (f" {output}" if output else "")

    elif svc_type == "2":  # File
        detail = ""

    if status != 0:
        icon = "\U0001f534"
        failed.append(name)
    elif svc_warn:
        icon = "\U0001f7e0"
        warned.append(name)
    else:
        icon = "\U0001f7e2"

    services[name] = (icon, detail, svc_type)

# --- Environment integrity ---
env_issues = []
def _has_var(var_name, file_path):
    try:
        with open(file_path) as f:
            for line in f:
                line = line.strip()
                if line.startswith("#") or "=" not in line:
                    continue
                k, v = line.split("=", 1)
                if k.strip() == var_name and v.strip():
                    return True
    except FileNotFoundError:
        return False
    return False

for var, src in [
    ("OPENDEVIATIONBAR_TELEGRAM_TOKEN", env_local),
    ("OPENDEVIATIONBAR_TELEGRAM_CHAT_ID", env_git),
    ("OPENDEVIATIONBAR_MODE", env_git),
    ("OPENDEVIATIONBAR_CH_HOSTS", env_git),
]:
    if not _has_var(var, src):
        env_issues.append(f"{var} missing")
if not Path(env_local).is_file():
    env_issues.append("local.env missing")
venv_base = Path(os.environ.get("HOME", "/home/tca")) / "opendeviationbar-py"
if not (venv_base / ".venv/bin/python3").exists():
    env_issues.append(".venv missing")
if not list((venv_base / ".venv").rglob("_core*.so")):
    env_issues.append("_core.so missing")

# --- Memory classification from /proc/meminfo ---
mem_class_lines = []
if meminfo:
    total = meminfo.get("MemTotal", 1)
    G = 1024 * 1024  # kB to GB
    anon = meminfo.get("AnonPages", 0)  # Application heap/stack
    cached = meminfo.get("Cached", 0) + meminfo.get("Buffers", 0)  # Reclaimable file cache
    slab_r = meminfo.get("SReclaimable", 0)  # Reclaimable kernel slab
    slab_u = meminfo.get("SUnreclaim", 0)   # Non-reclaimable kernel slab
    kernel = slab_u + meminfo.get("KernelStack", 0) + meminfo.get("PageTables", 0)
    swap_used = meminfo.get("SwapTotal", 0) - meminfo.get("SwapFree", 0)
    swap_cached = meminfo.get("SwapCached", 0)
    available = meminfo.get("MemAvailable", 0)

    mem_class_lines.append("<b>Memory map</b>")
    mem_class_lines.append(
        f"  Apps (AnonPages): {anon/G:.1f}GB "
        f"({anon*100/total:.0f}%)"
    )
    mem_class_lines.append(
        f"  File cache: {cached/G:.1f}GB "
        f"| kernel overhead: {kernel/G:.2f}GB"
    )
    mem_class_lines.append(
        f"  Swap: {swap_used/G:.1f}GB used "
        f"({swap_cached/G:.1f}GB cached in RAM)"
    )
    mem_class_lines.append(
        f"  Available: {available/G:.1f}GB "
        f"({available*100/total:.0f}% of {total/G:.0f}GB)"
    )

# --- Top RSS aggregated ---
rss_lines = []
if top_agg_raw.strip():
    for line in top_agg_raw.strip().split("\n"):
        parts = line.strip().split(None, 1)
        if len(parts) == 2:
            kb = int(parts[0])
            label = parts[1].replace("|", " ")
            if kb > 100 * 1024:  # >100MB only
                rss_lines.append(f"{label} {kb/1024/1024:.1f}GB")

# --- Process storm detection ---
storm_lines = []
if proc_counts_raw.strip():
    for line in proc_counts_raw.strip().split("\n"):
        parts = line.strip().split(None, 1)
        if len(parts) == 2:
            count = int(parts[0])
            user = parts[1]
            if count > 100:  # 32-core baseline: ~80 root userspace procs
                storm_lines.append(f"{user}: {count} procs")

# --- Ordered output ---
# Top: stable infrastructure (rarely changes)
# Middle: ODB services
# Bottom: system resources, disk, RSS breakdown (frequently monitored)
RENDER_ORDER = [
    # --- Stable infra (top) ---
    "redis",
    "clickhouse",
    "monit-heartbeat",
    "heartbeat-timer",
    # --- Environment ---
    "_env",
    # --- ODB services (middle) ---
    "sidecar",
    "sidecar-health",
    "kintsugi",
    # --- Seeders (Binance + Exness) ---
    "seeder-state-binance",
    "seeder-state-exness",
    # --- System resources (bottom, seen first in IM) ---
    "rootfs",
    hostname,  # el02 system
    "_memmap",
    "_rss",
]

now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
svc_count = len(root.findall("service"))
ok_count = svc_count - len(failed)

if failed or env_issues:
    if len(failed) >= 3 or env_issues:
        level_icon = "\U0001f534"
        level_label = "CRITICAL"
    else:
        level_icon = "\U0001f7e0"
        level_label = "WARNING"
    detail_str = ", ".join(failed + env_issues[:2])
    header = f"{level_icon} <b>MONIT</b> \u2014 {server_label} \u2014 {level_label} ({detail_str})"
elif warned:
    header = f"\U0001f7e1 <b>MONIT</b> \u2014 {server_label} \u2014 ELEVATED ({', '.join(warned[:3])})"
else:
    header = f"\U0001f49a <b>MONIT</b> \u2014 {server_label} \u2014 ALL HEALTHY"

lines = [header, f"<code>{uuid}</code>", ""]
lines.append(f"v{pkg_version} | monit up {monit_up_str} | {ok_count}/{svc_count} ok")

for key in RENDER_ORDER:
    if key == "_env":
        lines.append("")
        if env_issues:
            lines.append(f"\U0001f534 <b>Environment:</b> {', '.join(env_issues)}")
        else:
            lines.append(f"\U0001f7e2 <b>Environment:</b> intact")
        continue

    if key == "_memmap":
        if mem_class_lines:
            lines.append("")
            lines.extend(mem_class_lines)
        continue

    if key == "_rss":
        if rss_lines or storm_lines:
            lines.append("")
            lines.append("<b>Memory breakdown</b>")
            for r in rss_lines:
                lines.append(f"  {r}")
            if storm_lines:
                lines.append("<b>Process storms</b>")
                for s in storm_lines:
                    lines.append(f"  \U0001f534 {s}")
        continue

    if key not in services:
        continue

    icon, detail, svc_type = services[key]
    lines.append("")
    line = f"{icon} <b>{key}</b>"
    if detail:
        line += f"\n{detail}"
    lines.append(line)

# Any services not in RENDER_ORDER (safety net)
for name in services:
    if name not in RENDER_ORDER:
        icon, detail, svc_type = services[name]
        line = f"{icon} <b>{name}</b>"
        if detail:
            line += f"\n{detail}"
        lines.append(line)

lines.append("")
lines.append(f"<i>{now}</i>")

print("\n".join(lines))
PYEOF
)

if [ -z "$REPORT" ]; then
    REPORT="$(printf '\U0001f534') <b>MONIT</b> (report generation failed)\n<code>$UUID</code>"
fi

# Log to local telemetry JSONL before sending
TELEM_DIR="$HOME/.cache/opendeviationbar/telemetry"
mkdir -p "$TELEM_DIR"
PLAIN_REPORT=$(echo "$REPORT" | sed 's/<[^>]*>//g' | head -c 2000)
printf '{"ts":"%s","source":"monit-heartbeat","host":"%s","silent":true,"message":"%s"}\n' \
    "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(hostname)" \
    "$(echo "$PLAIN_REPORT" | sed 's/"/\\"/g' | tr '\n' ' ')" \
    >> "$TELEM_DIR/telegram.jsonl" 2>/dev/null

HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
    "https://api.telegram.org/bot${OPENDEVIATIONBAR_TELEGRAM_TOKEN}/sendMessage" \
    -d chat_id="${OPENDEVIATIONBAR_TELEGRAM_CHAT_ID}" \
    -d parse_mode="HTML" \
    -d disable_notification="true" \
    --data-urlencode "text=${REPORT}")

if [ "$HTTP_CODE" = "200" ]; then
    echo "Heartbeat sent ($UUID)"
    exit 0
else
    echo "Telegram API returned HTTP ${HTTP_CODE}" >&2
    exit 1
fi
