#!/usr/bin/env bash
# Tantalus — Redis Write-Lock Serialization Timeout and Contention Auditor
#
# The kintsugi L4 overlap-prevention layer uses a Redis-based write lock
# (cache_write_lock) per (symbol, threshold) pair. If BLOCKING_TIMEOUT is
# set too high, multiple workers queue behind the same lock, defeating the
# purpose of outer parallelism and causing deterministic serialization.
#
# Thresholds (from deploy config):
#   ≤5s  — safe for any parallelism level
#   6-9s — borderline; may cause minor queuing at MAX_CONCURRENT_REPAIRS ≥8
#   10-29s — warn; workers stall noticeably, throughput degrades
#   ≥30s — fail; serial execution guaranteed under load
#   60s  — the known incident value that caused throughput collapse
#
# Blocked Redis clients are a runtime proxy for lock contention.
set -euo pipefail

# Read configured timeout from the deploy env file (running ON bigblack)
ENV_FILE="$HOME/opendeviationbar-py/deploy/opendeviationbar.env"

configured_timeout=$(grep 'OPENDEVIATIONBAR_REDIS_LOCK_BLOCKING_TIMEOUT' "$ENV_FILE" 2>/dev/null \
  | cut -d= -f2 | tr -d '[:space:]' || echo "")

# Also check systemd drop-ins that might override the env file
dropins=$(grep -r 'REDIS_LOCK_BLOCKING_TIMEOUT' \
  ~/.config/systemd/user/opendeviationbar-kintsugi.service.d/ 2>/dev/null \
  | grep -v '^#' | cut -d= -f2- | tr -d '[:space:]' || echo "")

if [[ -n "$dropins" ]]; then
  effective_timeout="$dropins (drop-in override)"
  timeout_value="$dropins"
elif [[ -n "$configured_timeout" ]]; then
  effective_timeout="$configured_timeout (deploy/opendeviationbar.env)"
  timeout_value="$configured_timeout"
else
  effective_timeout="unknown — could not read from $ENV_FILE"
  timeout_value="unknown"
fi

# Check Redis client stats (direct call, no SSH — we run ON bigblack)
blocked=$(redis-cli INFO clients 2>/dev/null | grep blocked_clients | cut -d: -f2 | tr -d '[:space:]' || echo "0")
connected=$(redis-cli INFO clients 2>/dev/null | grep connected_clients | cut -d: -f2 | tr -d '[:space:]' || echo "0")

echo "Tantalus (Redis write-lock serialization timeout and contention auditor)"
echo "  Effective BLOCKING_TIMEOUT: $effective_timeout"
echo "  Redis blocked clients:      $blocked  (safe: 0, warn: >2, fail: >5)"
echo "  Redis connected clients:    $connected"
echo "  Safe threshold:             ≤5s | Incident value: 60s"

status=0

if [[ "$timeout_value" == "unknown" ]]; then
  echo "  Status: WARN — could not read OPENDEVIATIONBAR_REDIS_LOCK_BLOCKING_TIMEOUT from $ENV_FILE"
  exit 1
elif [[ "$timeout_value" -ge 30 ]]; then
  echo "  Status: FAIL — timeout=${timeout_value}s serializes workers behind lock. Reduce to ≤5s in deploy/opendeviationbar.env."
  exit 2
elif [[ "$timeout_value" -ge 10 ]]; then
  echo "  Status: WARN — timeout=${timeout_value}s higher than safe threshold (≤5s). Workers may queue."
  status=1
fi

if [[ "$blocked" -gt 5 ]]; then
  echo "  Status: FAIL — $blocked blocked Redis clients indicates active lock contention."
  exit 2
elif [[ "$blocked" -gt 2 ]]; then
  echo "  Status: WARN — $blocked blocked Redis clients, possible lock contention."
  [[ $status -lt 1 ]] && status=1
fi

if [[ $status -eq 1 ]]; then
  exit 1
else
  echo "  Status: PASS"
  exit 0
fi
