#!/bin/bash
#SBATCH --job-name=eval-gguf
#SBATCH --partition=normal
#SBATCH --gres=gpu:full:1
#SBATCH --cpus-per-gpu=8
#SBATCH --mem-per-gpu=64G
#SBATCH --time=20:00:00
#SBATCH --output=eval-%j.out
#SBATCH --error=eval-%j.out

set -euo pipefail
# ============================================================================
# SLURM Job: Run llmdebug eval suite against local OpenAI-compatible servers on HaiCORE
# ============================================================================
#
# Documentation:
# - Full runbook: docs/haicore_enroot_eval_runbook.md
# - Quickstart:   slurm/README.md
# - Preflight:    slurm/check_env_gguf_eval.sh
# - Smoke gate:   slurm/run_gguf_smoke.sh
# - Sweep submit: slurm/submit_gguf_sweep.sh
# - Resume chain: slurm/submit_gguf_resume_chain.sh
# - Debug bundle: slurm/collect_gguf_debug_bundle.sh
#
# Supports environment overrides (examples):
#   sbatch --export=ALL,EVAL_MAX_CASES=20 slurm/eval_gguf_eval.sh
#   sbatch --export=ALL,MODEL_FILE=custom.gguf,DRY_RUN=1,SERVER_BACKEND=llama_cpp slurm/eval_gguf_eval.sh
#   sbatch --export=ALL,SERVER_BACKEND=vllm,HF_MODEL_ID=Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8 slurm/eval_gguf_eval.sh
# Note: for comma-containing values (for example EVAL_CONDITIONS),
# set them outside --export to avoid Slurm comma-splitting:
#   EVAL_CONDITIONS='traceback_only,adaptive_llm_discretion' sbatch --export=ALL,... slurm/eval_gguf_eval.sh
# ============================================================================

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
if [[ -f "${SCRIPT_DIR}/_load_local_env.sh" ]]; then
    # shellcheck source=slurm/_load_local_env.sh
    source "${SCRIPT_DIR}/_load_local_env.sh"
    load_slurm_local_env "$SCRIPT_DIR"
fi

command_exists() {
    command -v "$1" >/dev/null 2>&1
}

normalize_flag() {
    local var_name raw
    var_name="$1"
    raw="${!var_name}"

    case "${raw,,}" in
        1|true|yes|on)
            printf -v "$var_name" '1'
            ;;
        0|false|no|off)
            printf -v "$var_name" '0'
            ;;
        *)
            echo "ERROR: ${var_name} must be one of: 1/0/true/false/yes/no/on/off"
            echo "       Got: '${raw}'"
            exit 1
            ;;
    esac
}

normalize_server_backend() {
    local var_name raw
    var_name="$1"
    raw="${!var_name}"

    case "${raw,,}" in
        vllm)
            printf -v "$var_name" 'vllm'
            ;;
        llama_cpp|llama-cpp|llama.cpp|llama)
            printf -v "$var_name" 'llama_cpp'
            ;;
        *)
            echo "ERROR: ${var_name} must be one of: vllm|llama_cpp"
            echo "       Got: '${raw}'"
            exit 1
            ;;
    esac
}

validate_int_ge() {
    local var_name min value
    var_name="$1"
    min="$2"
    value="${!var_name}"

    if [[ -z "$value" ]]; then
        echo "ERROR: ${var_name} must not be empty."
        exit 1
    fi

    if [[ ! "$value" =~ ^[0-9]+$ ]] || (( value < min )); then
        echo "ERROR: ${var_name} must be an integer >= ${min}. Got '${value}'."
        exit 1
    fi
}

validate_float_range() {
    local var_name min max value
    var_name="$1"
    min="$2"
    max="$3"
    value="${!var_name}"

    if [[ -z "$value" ]]; then
        echo "ERROR: ${var_name} must not be empty."
        exit 1
    fi

    if [[ ! "$value" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
        echo "ERROR: ${var_name} must be numeric. Got '${value}'."
        exit 1
    fi

    if ! awk "BEGIN { exit !(${value} >= ${min} && ${value} <= ${max}) }"; then
        echo "ERROR: ${var_name} must be within [${min}, ${max}]. Got '${value}'."
        exit 1
    fi
}

sanitize_runtime_dir() {
    local label candidate fallback resolved
    label="$1"
    candidate="$2"
    fallback="$3"
    resolved="${candidate:-${fallback}}"

    if [[ "$resolved" == /run/user/* ]]; then
        echo "WARN: ${label}='${resolved}' points to /run/user and may be unwritable." >&2
        echo "      Falling back to '${fallback}'." >&2
        resolved="$fallback"
    fi

    if ! mkdir -p "${resolved}" >/dev/null 2>&1; then
        echo "WARN: Unable to create ${label}='${resolved}'." >&2
        if [[ "${resolved}" != "${fallback}" ]]; then
            echo "      Falling back to '${fallback}'." >&2
            resolved="$fallback"
            mkdir -p "${resolved}" >/dev/null 2>&1 || true
        else
            echo "      Enroot may fail if runtime directory is not writable." >&2
        fi
    fi

    chmod 700 "${resolved}" >/dev/null 2>&1 || true
    printf '%s\n' "$resolved"
}

json_escape() {
    printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'
}

ensure_tmp_space() {
    local required_bytes probe_path available_bytes label
    required_bytes="$1"
    label="$2"
    probe_path="${TMPDIR:-/tmp}"

    available_bytes="$(df -PB1 "$probe_path" 2>/dev/null | awk 'NR==2 {print $4}')"
    if [[ -z "$available_bytes" || ! "$available_bytes" =~ ^[0-9]+$ ]]; then
        echo "WARN: Could not determine free space for $probe_path; skipping strict check ($label)."
        return 0
    fi

    if (( available_bytes < required_bytes )); then
        echo "ERROR: Not enough space in ${probe_path} for ${label}."
        echo "       Required bytes: ${required_bytes}"
        echo "       Available bytes: ${available_bytes}"
        echo "       Use a larger TMPDIR or disable fallback (ENABLE_TMPDIR_FALLBACK=0)."
        exit 1
    fi
}

resolve_model_preset() {
    local preset_input preset_key home_dir
    preset_input="$1"
    home_dir="$2"
    preset_key="${preset_input,,}"

    case "$preset_key" in
        glm47flash|glm-4.7-flash|glm47flash-ud-q4_k_xl|glm47flash_ud_q4_k_xl|glm-4.7-flash-ud-q4_k_xl)
            PRESET_CANONICAL="glm47flash-ud-q4_k_xl"
            PRESET_MODEL_DIR="${home_dir}/models/GLM-4.7-Flash-GGUF"
            PRESET_MODEL_FILE="GLM-4.7-Flash-UD-Q4_K_XL.gguf"
            PRESET_MODEL_ALIAS="GLM-4.7-Flash"
            PRESET_CONTEXT_SIZE=202752
            PRESET_MODEL_CTX_TRAIN=202752
            PRESET_SERVER_PARALLEL=3
            PRESET_RUN_ID_PREFIX="glm47flash-udq4kxl"
            PRESET_MODEL_SOURCE="unsloth/GLM-4.7-Flash-GGUF"
            PRESET_SERVER_TEMPERATURE="1.0"
            PRESET_SERVER_TOP_P="0.95"
            PRESET_SERVER_TOP_K=""
            PRESET_SERVER_MIN_P="0.01"
            PRESET_SERVER_REPEAT_PENALTY="1.0"
            PRESET_OPENAI_TEMPERATURE="1.0"
            ;;
        qwen3coder30ba3b-q4_k_m|qwen3-coder-30b-a3b-q4_k_m|qwen3-coder-30b-a3b-instruct-1m-q4_k_m|qwen3coder-q4_k_m|qwen3coder30ba3b)
            PRESET_CANONICAL="qwen3coder30ba3b-q4_k_m"
            PRESET_MODEL_DIR="${home_dir}/models/Qwen3-Coder-30B-A3B-Instruct-1M-GGUF"
            PRESET_MODEL_FILE="Qwen3-Coder-30B-A3B-Instruct-1M-Q4_K_M.gguf"
            PRESET_MODEL_ALIAS="Qwen3-Coder-30B-A3B-Instruct-1M"
            PRESET_CONTEXT_SIZE=196608
            PRESET_MODEL_CTX_TRAIN=262144
            PRESET_SERVER_PARALLEL=2
            PRESET_RUN_ID_PREFIX="qwen3coder30ba3b-q4km"
            PRESET_MODEL_SOURCE="unsloth/Qwen3-Coder-30B-A3B-Instruct-1M-GGUF"
            PRESET_SERVER_TEMPERATURE="0.7"
            PRESET_SERVER_TOP_P="0.8"
            PRESET_SERVER_TOP_K="20"
            PRESET_SERVER_MIN_P="0.00"
            PRESET_SERVER_REPEAT_PENALTY="1.05"
            PRESET_OPENAI_TEMPERATURE="0.7"
            ;;
        qwen3coder30ba3b-q6_k|qwen3-coder-30b-a3b-q6_k|qwen3-coder-30b-a3b-instruct-1m-q6_k|qwen3coder-q6_k)
            PRESET_CANONICAL="qwen3coder30ba3b-q6_k"
            PRESET_MODEL_DIR="${home_dir}/models/Qwen3-Coder-30B-A3B-Instruct-1M-GGUF"
            PRESET_MODEL_FILE="Qwen3-Coder-30B-A3B-Instruct-1M-Q6_K.gguf"
            PRESET_MODEL_ALIAS="Qwen3-Coder-30B-A3B-Instruct-1M"
            PRESET_CONTEXT_SIZE=262144
            PRESET_MODEL_CTX_TRAIN=262144
            PRESET_SERVER_PARALLEL=2
            PRESET_RUN_ID_PREFIX="qwen3coder30ba3b-q6k"
            PRESET_MODEL_SOURCE="unsloth/Qwen3-Coder-30B-A3B-Instruct-1M-GGUF"
            PRESET_SERVER_TEMPERATURE="0.7"
            PRESET_SERVER_TOP_P="0.8"
            PRESET_SERVER_TOP_K="20"
            PRESET_SERVER_MIN_P="0.00"
            PRESET_SERVER_REPEAT_PENALTY="1.05"
            PRESET_OPENAI_TEMPERATURE="0.7"
            ;;
        custom|manual)
            PRESET_CANONICAL="custom"
            PRESET_MODEL_DIR="${home_dir}/models"
            PRESET_MODEL_FILE="model.gguf"
            PRESET_MODEL_ALIAS="custom-model"
            PRESET_CONTEXT_SIZE=8192
            PRESET_MODEL_CTX_TRAIN=""
            PRESET_SERVER_PARALLEL=""
            PRESET_RUN_ID_PREFIX="gguf-custom"
            PRESET_MODEL_SOURCE="custom"
            PRESET_SERVER_TEMPERATURE=""
            PRESET_SERVER_TOP_P=""
            PRESET_SERVER_TOP_K=""
            PRESET_SERVER_MIN_P="0.01"
            PRESET_SERVER_REPEAT_PENALTY=""
            PRESET_OPENAI_TEMPERATURE="0.0"
            ;;
        *)
            if (( HAS_MODEL_DIR_OVERRIDE == 1 && HAS_MODEL_FILE_OVERRIDE == 1 )); then
                if [[ -z "${MODEL_DIR:-}" || -z "${MODEL_FILE:-}" ]]; then
                    echo "ERROR: Unknown MODEL_PRESET='${preset_input}' requires non-empty MODEL_DIR and MODEL_FILE."
                    exit 1
                fi

                PRESET_CANONICAL="custom"
                PRESET_MODEL_DIR="${MODEL_DIR}"
                PRESET_MODEL_FILE="${MODEL_FILE}"
                PRESET_MODEL_ALIAS="${MODEL_ALIAS:-custom-model}"
                PRESET_CONTEXT_SIZE=8192
                PRESET_MODEL_CTX_TRAIN=""
                PRESET_SERVER_PARALLEL=""
                PRESET_RUN_ID_PREFIX="gguf-custom"
                PRESET_MODEL_SOURCE="custom"
                PRESET_SERVER_TEMPERATURE=""
                PRESET_SERVER_TOP_P=""
                PRESET_SERVER_TOP_K=""
                PRESET_SERVER_MIN_P="0.01"
                PRESET_SERVER_REPEAT_PENALTY=""
                PRESET_OPENAI_TEMPERATURE="0.0"
                echo "WARN: Unknown MODEL_PRESET='${preset_input}', falling back to custom explicit model settings."
                return
            fi

            echo "ERROR: Unsupported MODEL_PRESET='${preset_input}'."
            echo "Supported MODEL_PRESET values:"
            echo "  - qwen3coder30ba3b-q4_k_m (default)"
            echo "  - qwen3coder30ba3b-q6_k"
            echo "  - glm47flash-ud-q4_k_xl"
            echo "  - custom (requires MODEL_DIR, MODEL_FILE; MODEL_ALIAS optional)"
            exit 1
            ;;
    esac
}

print_effective_config() {
    echo "========================================"
    echo "Job ID:         $JOB_ID"
    echo "Node:           $(hostname)"
    echo "GPU:            $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo 'N/A')"
    echo "SLURM cpu/gpu:  ${SLURM_CPUS_PER_GPU:-<unset>}"
    echo "SLURM mem/gpu:  ${SLURM_MEM_PER_GPU:-<unset>}"
    echo "Server backend: $SERVER_BACKEND"
    echo "Server image:   $SERVER_CONTAINER"
    echo "Eval:           $EVAL_CONTAINER"
    echo "Model preset:   $MODEL_PRESET"
    echo "Model source:   $MODEL_SOURCE"
    echo "Model:          $MODEL_PATH"
    echo "Model mount:    $MODEL_MOUNT_DIR"
    echo "HF model id:    ${HF_MODEL_ID:-<n/a>}"
    echo "HF model alias: ${HF_MODEL_ALIAS:-<n/a>}"
    echo "vLLM max len:   ${VLLM_MAX_MODEL_LEN:-<n/a>}"
    echo "vLLM gpu util:  ${VLLM_GPU_MEMORY_UTILIZATION:-<n/a>}"
    echo "vLLM tp size:   ${VLLM_TENSOR_PARALLEL_SIZE:-<n/a>}"
    echo "vLLM dtype:     ${VLLM_DTYPE:-<n/a>}"
    echo "vLLM kv dtype:  ${VLLM_KV_CACHE_DTYPE:-<n/a>}"
    echo "vLLM kv scales: ${VLLM_CALCULATE_KV_SCALES:-<n/a>}"
    echo "vLLM max seqs:  ${VLLM_MAX_NUM_SEQS:-<n/a>}"
    echo "vLLM max btok:  ${VLLM_MAX_NUM_BATCHED_TOKENS:-<n/a>}"
    echo "vLLM trust rc:  ${VLLM_TRUST_REMOTE_CODE:-<n/a>}"
    echo "vLLM dl dir:    ${VLLM_DOWNLOAD_DIR:-<n/a>}"
    echo "HF cache dir:   ${HF_CACHE_DIR:-<n/a>}"
    echo "Repo mount:     $REPO_MOUNT_DIR"
    echo "Enroot data:    $ENROOT_DATA_DIR"
    echo "Host XDG runtime: ${HOST_XDG_RUNTIME_DIR:-<unset>}"
    echo "Container XDG runtime: ${CONTAINER_XDG_RUNTIME_DIR:-<unset>}"
    echo "BASH_ENV:       ${BASH_ENV:-<unset>}"
    echo "Server port:    $SERVER_PORT"
    echo "Context:        $CONTEXT_SIZE"
    echo "Model train ctx: ${MODEL_CTX_TRAIN_TOKENS:-unknown}"
    echo "Per-slot ctx:   ${PER_SLOT_CONTEXT_TOKENS}"
    echo "GPU layers:     $GPU_LAYERS"
    echo "Server temp:    ${SERVER_TEMPERATURE:-<default>}"
    echo "Server top-p:   ${SERVER_TOP_P:-<default>}"
    echo "Server top-k:   ${SERVER_TOP_K:-<default>}"
    echo "Server min-p:   ${SERVER_MIN_P:-<default>}"
    echo "Server rep-pen: ${SERVER_REPEAT_PENALTY:-<default>}"
    echo "Cache type K:   ${SERVER_CACHE_TYPE_K:-<default>}"
    echo "Cache type V:   ${SERVER_CACHE_TYPE_V:-<default>}"
    echo "Server flash:   ${SERVER_FLASH_ATTN:-<default>}"
    echo "Server batch:   ${SERVER_BATCH_SIZE}"
    echo "Server ubatch:  ${SERVER_UBATCH_SIZE}"
    echo "Server parallel:${SERVER_PARALLEL}"
    echo "Threads:        ${SERVER_THREADS:-<auto>}"
    echo "Threads batch:  ${SERVER_THREADS_BATCH:-<auto>}"
    echo "Threads http:   ${SERVER_THREADS_HTTP:-<auto>}"
    echo "Metrics:        ${SERVER_METRICS}"
    echo "CUDA queues:    ${CUDA_SCALE_LAUNCH_QUEUES:-<default>}"
    echo "Force cuBLAS:   ${GGML_CUDA_FORCE_CUBLAS:-<default>}"
    echo "Force MMQ:      ${GGML_CUDA_FORCE_MMQ:-<default>}"
    echo "Unified memory: ${GGML_CUDA_ENABLE_UNIFIED_MEMORY:-<default>}"
    echo "OpenAI temp:    ${OPENAI_TEMPERATURE}"
    echo "OpenAI timeout: ${OPENAI_TIMEOUT_SEC}"
    echo "OpenAI max tokens: ${OPENAI_MAX_TOKENS}"
    echo "OpenAI max propose sec: ${OPENAI_MAX_PROPOSE_SEC}"
    echo "Auto ctx budget: ${AUTO_CONTEXT_BUDGETS} (chars/token=${CONTEXT_CHARS_PER_TOKEN}, headroom=${CONTEXT_PROMPT_HEADROOM_PCT}%)"
    echo "Eval max chars: ${EVAL_MAX_CONTEXT_CHARS}"
    echo "Eval prompt cap:${EVAL_PROMPT_BUDGET_TOTAL_CHARS}"
    echo "Eval conditions:${EVAL_CONDITIONS}"
    echo "Run ID:         $EVAL_RUN_ID"
    echo "Smoke mode:     $SMOKE_TEST_MODE"
    echo "Dry run:        $DRY_RUN"
    echo "Deterministic:  $EVAL_DETERMINISTIC"
    echo "Seed:           ${EVAL_SEED:-<unset>}"
    echo "Server srun:    ${SERVER_USE_SRUN} (cpu-bind=${SERVER_SRUN_CPU_BIND}, gpu-bind=${SERVER_SRUN_GPU_BIND}, cpus-per-task=${SERVER_SRUN_CPUS_PER_TASK:-<inherit>}, unbuffered=${SERVER_SRUN_UNBUFFERED})"
    echo "Enroot debug:   $ENABLE_ENROOT_DEBUG"
    echo "TMP fallback:   $ENABLE_TMPDIR_FALLBACK"
    echo "GPU monitor:    $GPU_UTIL_MONITOR"
    echo "GPU fail-fast:  ${GPU_FAILFAST} (warmup=${GPU_FAILFAST_WARMUP_SEC}s, util<th=${GPU_FAILFAST_UTIL_PCT}%, low-frac>=${GPU_FAILFAST_LOW_FRAC_PCT}%, min-samples=${GPU_FAILFAST_MIN_SAMPLES}, check=${GPU_FAILFAST_CHECK_SEC}s)"
    echo "Server fail-fast:${SERVER_FAILFAST} (check=${SERVER_FAILFAST_CHECK_SEC}s, max-consec-health-fails=${SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS})"
    echo "Tmp stage:      ${TMP_STAGE_ROOT:-<unset>}"
    echo "Server log:     $SERVER_LOG"
    echo "Eval log:       $EVAL_LAUNCH_LOG"
    echo "Diag log:       $ENROOT_DIAG_LOG"
    echo "GPU util log:   $GPU_UTIL_LOG"
    echo "Server fail-fast file: $SERVER_FAILFAST_REASON_FILE"
    echo "Config log:     $RUN_CONFIG_LOG"
    echo "Failure dump:   $FAILURE_DUMP_LOG"
    echo "Started:        $(date -Iseconds)"
    echo "========================================"
}

write_run_config_snapshot() {
    local git_hash

    if (( RUN_CONFIG_SNAPSHOT == 0 )); then
        return
    fi

    if git_hash="$(git -C "$REPO_DIR_REAL" rev-parse HEAD 2>/dev/null)"; then
        :
    else
        git_hash="unknown"
    fi

    cat > "$RUN_CONFIG_LOG" <<JSON
{
  "timestamp": "$(date -Iseconds)",
  "job_id": "$(json_escape "$JOB_ID")",
  "run_id": "$(json_escape "$EVAL_RUN_ID")",
  "git_commit": "$(json_escape "$git_hash")",
  "slurm_cpus_per_gpu": "$(json_escape "${SLURM_CPUS_PER_GPU:-}")",
  "slurm_mem_per_gpu": "$(json_escape "${SLURM_MEM_PER_GPU:-}")",
  "smoke_test_mode": ${SMOKE_TEST_MODE},
  "dry_run": ${DRY_RUN},
  "deterministic": ${EVAL_DETERMINISTIC},
  "seed": "$(json_escape "${EVAL_SEED:-}")",
  "server_backend": "$(json_escape "$SERVER_BACKEND")",
  "server_container": "$(json_escape "$SERVER_CONTAINER")",
  "eval_container": "$(json_escape "$EVAL_CONTAINER")",
  "model_preset": "$(json_escape "$MODEL_PRESET")",
  "model_source": "$(json_escape "$MODEL_SOURCE")",
  "model_path": "$(json_escape "$MODEL_PATH")",
  "model_mount_dir": "$(json_escape "$MODEL_MOUNT_DIR")",
  "hf_model_id": "$(json_escape "${HF_MODEL_ID:-}")",
  "hf_model_alias": "$(json_escape "${HF_MODEL_ALIAS:-}")",
  "vllm_max_model_len": "$(json_escape "${VLLM_MAX_MODEL_LEN:-}")",
  "vllm_gpu_memory_utilization": "$(json_escape "${VLLM_GPU_MEMORY_UTILIZATION:-}")",
  "vllm_tensor_parallel_size": "$(json_escape "${VLLM_TENSOR_PARALLEL_SIZE:-}")",
  "vllm_dtype": "$(json_escape "${VLLM_DTYPE:-}")",
  "vllm_kv_cache_dtype": "$(json_escape "${VLLM_KV_CACHE_DTYPE:-}")",
  "vllm_calculate_kv_scales": ${VLLM_CALCULATE_KV_SCALES},
  "vllm_max_num_seqs": ${VLLM_MAX_NUM_SEQS},
  "vllm_max_num_batched_tokens": "$(json_escape "${VLLM_MAX_NUM_BATCHED_TOKENS:-}")",
  "vllm_trust_remote_code": ${VLLM_TRUST_REMOTE_CODE},
  "vllm_download_dir": "$(json_escape "${VLLM_DOWNLOAD_DIR:-}")",
  "hf_cache_dir": "$(json_escape "${HF_CACHE_DIR:-}")",
  "repo_mount_dir": "$(json_escape "$REPO_MOUNT_DIR")",
  "server_port": ${SERVER_PORT},
  "context_size": ${CONTEXT_SIZE},
  "model_ctx_train_tokens": "$(json_escape "${MODEL_CTX_TRAIN_TOKENS:-}")",
  "per_slot_context_tokens": ${PER_SLOT_CONTEXT_TOKENS},
  "gpu_layers": ${GPU_LAYERS},
  "server_temperature": "$(json_escape "${SERVER_TEMPERATURE:-}")",
  "server_top_p": "$(json_escape "${SERVER_TOP_P:-}")",
  "server_top_k": "$(json_escape "${SERVER_TOP_K:-}")",
  "server_min_p": "$(json_escape "${SERVER_MIN_P:-}")",
  "server_repeat_penalty": "$(json_escape "${SERVER_REPEAT_PENALTY:-}")",
  "server_cache_type_k": "$(json_escape "${SERVER_CACHE_TYPE_K:-}")",
  "server_cache_type_v": "$(json_escape "${SERVER_CACHE_TYPE_V:-}")",
  "server_flash_attn": "$(json_escape "${SERVER_FLASH_ATTN}")",
  "server_batch_size": ${SERVER_BATCH_SIZE},
  "server_ubatch_size": ${SERVER_UBATCH_SIZE},
  "server_parallel": ${SERVER_PARALLEL},
  "server_threads": "$(json_escape "${SERVER_THREADS:-}")",
  "server_threads_batch": "$(json_escape "${SERVER_THREADS_BATCH:-}")",
  "server_threads_http": "$(json_escape "${SERVER_THREADS_HTTP:-}")",
  "server_metrics": ${SERVER_METRICS},
  "cuda_scale_launch_queues": "$(json_escape "${CUDA_SCALE_LAUNCH_QUEUES:-}")",
  "ggml_cuda_force_cublas": "$(json_escape "${GGML_CUDA_FORCE_CUBLAS:-}")",
  "ggml_cuda_force_mmq": "$(json_escape "${GGML_CUDA_FORCE_MMQ:-}")",
  "ggml_cuda_enable_unified_memory": "$(json_escape "${GGML_CUDA_ENABLE_UNIFIED_MEMORY:-}")",
  "openai_temperature": "$(json_escape "${OPENAI_TEMPERATURE}")",
  "openai_timeout_sec": "$(json_escape "${OPENAI_TIMEOUT_SEC}")",
  "openai_max_tokens": "$(json_escape "${OPENAI_MAX_TOKENS}")",
  "openai_max_propose_sec": "$(json_escape "${OPENAI_MAX_PROPOSE_SEC}")",
  "auto_context_budgets": ${AUTO_CONTEXT_BUDGETS},
  "context_chars_per_token": ${CONTEXT_CHARS_PER_TOKEN},
  "context_prompt_headroom_pct": ${CONTEXT_PROMPT_HEADROOM_PCT},
  "eval_max_context_chars": ${EVAL_MAX_CONTEXT_CHARS},
  "eval_prompt_budget_total_chars": ${EVAL_PROMPT_BUDGET_TOTAL_CHARS},
  "host_xdg_runtime_dir": "$(json_escape "${HOST_XDG_RUNTIME_DIR:-}")",
  "container_xdg_runtime_dir": "$(json_escape "${CONTAINER_XDG_RUNTIME_DIR:-}")",
  "bash_env": "$(json_escape "${BASH_ENV:-}")",
  "eval_k": ${EVAL_K},
  "eval_jobs": ${EVAL_JOBS},
  "eval_max_cases": ${EVAL_MAX_CASES},
  "eval_category": "$(json_escape "$EVAL_CATEGORY")",
  "eval_conditions": "$(json_escape "${EVAL_CONDITIONS}")",
  "enroot_data_dir": "$(json_escape "$ENROOT_DATA_DIR")",
  "enable_enroot_debug": ${ENABLE_ENROOT_DEBUG},
  "enable_tmpdir_fallback": ${ENABLE_TMPDIR_FALLBACK},
  "server_use_srun": ${SERVER_USE_SRUN},
  "server_srun_cpu_bind": "$(json_escape "${SERVER_SRUN_CPU_BIND}")",
  "server_srun_gpu_bind": "$(json_escape "${SERVER_SRUN_GPU_BIND}")",
  "server_srun_cpus_per_task": "$(json_escape "${SERVER_SRUN_CPUS_PER_TASK:-}")",
  "server_srun_unbuffered": ${SERVER_SRUN_UNBUFFERED},
  "gpu_util_monitor": ${GPU_UTIL_MONITOR},
  "gpu_util_poll_sec": ${GPU_UTIL_POLL_SEC},
  "gpu_failfast": ${GPU_FAILFAST},
  "gpu_failfast_warmup_sec": ${GPU_FAILFAST_WARMUP_SEC},
  "gpu_failfast_util_pct": ${GPU_FAILFAST_UTIL_PCT},
  "gpu_failfast_low_frac_pct": ${GPU_FAILFAST_LOW_FRAC_PCT},
  "gpu_failfast_min_samples": ${GPU_FAILFAST_MIN_SAMPLES},
  "gpu_failfast_check_sec": ${GPU_FAILFAST_CHECK_SEC},
  "gpu_failfast_triggered": ${GPU_FAILFAST_TRIGGERED},
  "gpu_failfast_reason": "$(json_escape "${GPU_FAILFAST_REASON:-}")",
  "server_failfast": ${SERVER_FAILFAST},
  "server_failfast_check_sec": ${SERVER_FAILFAST_CHECK_SEC},
  "server_failfast_max_consec_health_fails": ${SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS},
  "server_failfast_triggered": ${SERVER_FAILFAST_TRIGGERED},
  "server_failfast_reason": "$(json_escape "${SERVER_FAILFAST_REASON:-}")"
}
JSON
}

write_failure_dump() {
    local reason exit_code
    reason="$1"
    exit_code="$2"

    if (( FAILURE_DUMP_MODE == 0 || FAILURE_DUMP_WRITTEN == 1 )); then
        return
    fi

    cat > "$FAILURE_DUMP_LOG" <<JSON
{
  "timestamp": "$(date -Iseconds)",
  "job_id": "$(json_escape "$JOB_ID")",
  "run_id": "$(json_escape "$EVAL_RUN_ID")",
  "model_preset": "$(json_escape "$MODEL_PRESET")",
  "server_backend": "$(json_escape "$SERVER_BACKEND")",
  "model_path": "$(json_escape "$MODEL_PATH")",
  "exit_code": ${exit_code},
  "reason": "$(json_escape "$reason")",
  "smoke_test_mode": ${SMOKE_TEST_MODE},
  "tmpdir_fallback_used": ${USING_TMPDIR_STAGING},
  "model_staged": ${MODEL_STAGED},
  "repo_staged": ${REPO_STAGED},
  "tmp_stage_root": "$(json_escape "${TMP_STAGE_ROOT:-}")",
  "gpu_failfast_triggered": ${GPU_FAILFAST_TRIGGERED},
  "gpu_failfast_reason": "$(json_escape "${GPU_FAILFAST_REASON:-}")",
  "server_failfast_triggered": ${SERVER_FAILFAST_TRIGGERED},
  "server_failfast_reason": "$(json_escape "${SERVER_FAILFAST_REASON:-}")",
  "server_log": "$(json_escape "$SERVER_LOG")",
  "eval_log": "$(json_escape "$EVAL_LAUNCH_LOG")",
  "diag_log": "$(json_escape "$ENROOT_DIAG_LOG")",
  "gpu_util_log": "$(json_escape "$GPU_UTIL_LOG")",
  "run_config_log": "$(json_escape "$RUN_CONFIG_LOG")",
  "server_failfast_reason_file": "$(json_escape "$SERVER_FAILFAST_REASON_FILE")"
}
JSON

    FAILURE_DUMP_WRITTEN=1
}

start_gpu_monitor() {
    if (( GPU_UTIL_MONITOR == 0 )); then
        return
    fi

    if ! command_exists nvidia-smi; then
        echo "GPU utilization monitor disabled: nvidia-smi not available."
        return
    fi

    echo "timestamp,index,utilization_gpu_pct,utilization_mem_pct,memory_used_mb,memory_total_mb,power_draw_w" \
        > "$GPU_UTIL_LOG"

    (
        while true; do
            nvidia-smi \
                --query-gpu=timestamp,index,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw \
                --format=csv,noheader,nounits >> "$GPU_UTIL_LOG" 2>/dev/null || true
            sleep "$GPU_UTIL_POLL_SEC"
        done
    ) &

    GPU_MONITOR_PID=$!
}

stop_gpu_monitor() {
    if [[ -n "${GPU_MONITOR_PID:-}" ]]; then
        kill "$GPU_MONITOR_PID" 2>/dev/null || true
        wait "$GPU_MONITOR_PID" 2>/dev/null || true
    fi
}

summarize_gpu_monitor() {
    local summary

    if (( GPU_UTIL_MONITOR == 0 )); then
        return
    fi
    if [[ ! -f "$GPU_UTIL_LOG" ]]; then
        return
    fi

    summary="$(awk -F',' '
        NR==1 {next}
        {
            gsub(/^[ \t]+|[ \t]+$/, "", $3)
            if ($3 ~ /^[0-9]+(\.[0-9]+)?$/) {
                util=$3+0
                sum+=util
                n+=1
                if (util < 40) low+=1
            }
        }
        END {
            if (n == 0) {
                print "samples=0 avg=0.0 low_frac=0.0"
            } else {
                avg=sum/n
                low_frac=low/n
                printf "samples=%d avg=%.1f low_frac=%.3f", n, avg, low_frac
            }
        }
    ' "$GPU_UTIL_LOG")"

    echo "GPU utilization summary: ${summary}"

    if awk -F'[ =]' '/low_frac=/{ if ($6+0 > 0.70) exit 0; exit 1 }' <<< "$summary"; then
        echo "WARN: GPU utilization appears low for much of the run."
        echo "      Check llama.cpp batching/parallel settings, CPU thread limits, and storage throughput."
    fi
}

summarize_gpu_window_since_line() {
    local start_line
    start_line="$1"

    if [[ ! -f "$GPU_UTIL_LOG" ]]; then
        echo "0 0.0 0.0"
        return
    fi

    awk -F',' -v start_line="$start_line" -v util_threshold="$GPU_FAILFAST_UTIL_PCT" '
        NR <= start_line {next}
        {
            gsub(/^[ \t]+|[ \t]+$/, "", $3)
            if ($3 ~ /^[0-9]+(\.[0-9]+)?$/) {
                util=$3+0
                sum+=util
                n+=1
                if (util < util_threshold) low+=1
            }
        }
        END {
            if (n == 0) {
                print "0 0.0 0.0"
            } else {
                avg=sum/n
                low_pct=(low*100.0)/n
                printf "%d %.1f %.1f\n", n, avg, low_pct
            }
        }
    ' "$GPU_UTIL_LOG"
}

terminate_eval_process_tree() {
    local eval_pid
    eval_pid="$1"
    kill "$eval_pid" 2>/dev/null || true
    pkill -TERM -P "$eval_pid" 2>/dev/null || true
    sleep 5
    pkill -KILL -P "$eval_pid" 2>/dev/null || true
    kill -9 "$eval_pid" 2>/dev/null || true
}

start_gpu_failfast_watchdog() {
    local eval_pid eval_start_epoch
    eval_pid="$1"
    eval_start_epoch="$2"

    GPU_FAILFAST_TRIGGERED=0
    GPU_FAILFAST_REASON=""
    rm -f "$GPU_FAILFAST_REASON_FILE"

    if (( GPU_FAILFAST == 0 )); then
        return
    fi
    if (( GPU_UTIL_MONITOR == 0 )); then
        echo "GPU fail-fast disabled because GPU_UTIL_MONITOR=0."
        return
    fi
    if [[ ! -f "$GPU_UTIL_LOG" ]]; then
        echo "GPU fail-fast disabled because utilization log is unavailable: $GPU_UTIL_LOG"
        return
    fi

    (
        local now_epoch elapsed samples avg_util low_pct reason
        while kill -0 "$eval_pid" 2>/dev/null; do
            sleep "$GPU_FAILFAST_CHECK_SEC"
            if ! kill -0 "$eval_pid" 2>/dev/null; then
                break
            fi

            now_epoch="$(date +%s)"
            elapsed=$(( now_epoch - eval_start_epoch ))
            if (( elapsed < GPU_FAILFAST_WARMUP_SEC )); then
                continue
            fi

            read -r samples avg_util low_pct <<< "$(summarize_gpu_window_since_line "$GPU_FAILFAST_START_LINE")"
            if (( samples < GPU_FAILFAST_MIN_SAMPLES )); then
                continue
            fi

            if awk "BEGIN { exit !(${low_pct} >= ${GPU_FAILFAST_LOW_FRAC_PCT}) }"; then
                reason="GPU fail-fast triggered: ${low_pct}% of ${samples} samples were below ${GPU_FAILFAST_UTIL_PCT}% utilization (avg=${avg_util}%) after ${elapsed}s."
                printf '%s\n' "$reason" > "$GPU_FAILFAST_REASON_FILE"
                echo "$reason"

                terminate_eval_process_tree "$eval_pid"
                break
            fi
        done
    ) &

    GPU_FAILFAST_WATCHDOG_PID=$!
}

stop_gpu_failfast_watchdog() {
    if [[ -n "${GPU_FAILFAST_WATCHDOG_PID:-}" ]]; then
        kill "$GPU_FAILFAST_WATCHDOG_PID" 2>/dev/null || true
        wait "$GPU_FAILFAST_WATCHDOG_PID" 2>/dev/null || true
        GPU_FAILFAST_WATCHDOG_PID=""
    fi

    if [[ -f "$GPU_FAILFAST_REASON_FILE" ]]; then
        GPU_FAILFAST_TRIGGERED=1
        GPU_FAILFAST_REASON="$(cat "$GPU_FAILFAST_REASON_FILE")"
    fi
}

server_healthcheck() {
    if curl -sf "http://localhost:${SERVER_PORT}/health" > /dev/null 2>&1; then
        return 0
    fi
    if curl -sf "http://localhost:${SERVER_PORT}/v1/models" > /dev/null 2>&1; then
        return 0
    fi
    return 1
}

start_server_failfast_watchdog() {
    local eval_pid
    eval_pid="$1"

    SERVER_FAILFAST_TRIGGERED=0
    SERVER_FAILFAST_REASON=""
    rm -f "$SERVER_FAILFAST_REASON_FILE"

    if (( SERVER_FAILFAST == 0 )); then
        return
    fi

    if [[ -z "${SERVER_PID:-}" ]]; then
        echo "Server fail-fast disabled because SERVER_PID is not set."
        return
    fi

    (
        local consecutive_health_fails reason
        consecutive_health_fails=0
        while kill -0 "$eval_pid" 2>/dev/null; do
            sleep "$SERVER_FAILFAST_CHECK_SEC"
            if ! kill -0 "$eval_pid" 2>/dev/null; then
                break
            fi

            if ! kill -0 "$SERVER_PID" 2>/dev/null; then
                reason="Server fail-fast triggered: LLM server (PID=${SERVER_PID}) is no longer running during eval."
                printf '%s\n' "$reason" > "$SERVER_FAILFAST_REASON_FILE"
                echo "$reason"
                terminate_eval_process_tree "$eval_pid"
                break
            fi

            if server_healthcheck; then
                consecutive_health_fails=0
                continue
            fi

            consecutive_health_fails=$(( consecutive_health_fails + 1 ))
            if (( consecutive_health_fails >= SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS )); then
                reason="Server fail-fast triggered: health checks failed ${consecutive_health_fails} consecutive times (interval=${SERVER_FAILFAST_CHECK_SEC}s) during eval."
                printf '%s\n' "$reason" > "$SERVER_FAILFAST_REASON_FILE"
                echo "$reason"
                terminate_eval_process_tree "$eval_pid"
                break
            fi
        done
    ) &

    SERVER_FAILFAST_WATCHDOG_PID=$!
}

stop_server_failfast_watchdog() {
    if [[ -n "${SERVER_FAILFAST_WATCHDOG_PID:-}" ]]; then
        kill "$SERVER_FAILFAST_WATCHDOG_PID" 2>/dev/null || true
        wait "$SERVER_FAILFAST_WATCHDOG_PID" 2>/dev/null || true
        SERVER_FAILFAST_WATCHDOG_PID=""
    fi

    if [[ -f "$SERVER_FAILFAST_REASON_FILE" ]]; then
        SERVER_FAILFAST_TRIGGERED=1
        SERVER_FAILFAST_REASON="$(cat "$SERVER_FAILFAST_REASON_FILE")"
    fi
}

append_path_diagnostics() {
    local label path
    label="$1"
    path="$2"
    {
        echo "[$(date -Iseconds)] ${label}"
        if [[ -z "$path" ]]; then
            echo "  path: <n/a>"
        else
            echo "  path: ${path}"
            ls -ld "${path}" 2>&1 | sed 's/^/  ls: /'
            stat -c "  stat: mode=%a uid=%u gid=%g dev=%d inode=%i type=%F" "${path}" 2>&1
            stat -f -c "  fstype: %T" "${path}" 2>&1
            if command_exists findmnt; then
                findmnt -T "${path}" -o TARGET,SOURCE,FSTYPE,OPTIONS 2>&1 | sed 's/^/  findmnt: /'
            fi
            if command_exists df; then
                df -T "${path}" 2>&1 | sed 's/^/  df: /'
            fi
        fi
        echo ""
    } >> "$ENROOT_DIAG_LOG"
}

log_mount_diagnostics() {
    : > "$ENROOT_DIAG_LOG"
    {
        echo "=== enroot mount diagnostics ==="
        echo "timestamp:        $(date -Iseconds)"
        echo "logical HOME_DIR: ${HOME_DIR}"
        echo "shell HOME:       ${HOME}"
        echo "pwd:              $(pwd)"
        echo "pwd -P:           $(pwd -P)"
        echo "MODEL_DIR config: ${MODEL_DIR}"
        echo "MODEL_DIR real:   ${MODEL_DIR_REAL}"
        echo "MODEL_MOUNT_DIR:  ${MODEL_MOUNT_DIR}"
        echo "REPO_DIR config:  ${REPO_DIR}"
        echo "REPO_DIR real:    ${REPO_DIR_REAL}"
        echo "REPO_MOUNT_DIR:   ${REPO_MOUNT_DIR}"
        echo "ENROOT_DATA_PATH: ${ENROOT_DATA_PATH:-<unset>}"
        echo "ENROOT_DATA_DIR:  ${ENROOT_DATA_DIR}"
        echo "SERVER_ROOTFS:    ${SERVER_ROOTFS}"
        echo "EVAL_ROOTFS:      ${EVAL_ROOTFS}"
        echo "TMPDIR:           ${TMPDIR:-<unset>}"
        echo ""
    } >> "$ENROOT_DIAG_LOG"

    append_path_diagnostics "model mount source" "$MODEL_MOUNT_DIR"
    append_path_diagnostics "repo mount source" "$REPO_MOUNT_DIR"
    append_path_diagnostics "server rootfs" "$SERVER_ROOTFS"
    append_path_diagnostics "server mount target (/models)" "${SERVER_ROOTFS}/models"
    append_path_diagnostics "eval rootfs" "$EVAL_ROOTFS"
    append_path_diagnostics "eval mount target (/workspace)" "${EVAL_ROOTFS}/workspace"
}

prepare_bind_mount_targets() {
    local lock_file
    lock_file="${ENROOT_DATA_DIR}/.llmdebug-mount-targets.lock"

    echo "Preparing enroot bind-mount targets..."
    if command_exists flock; then
        exec 9>"$lock_file"
        if ! flock -w 30 9; then
            echo "ERROR: Timed out waiting for mount-target lock: $lock_file"
            exit 1
        fi
    fi

    if ! mkdir -p "${SERVER_ROOTFS}/models" "${EVAL_ROOTFS}/workspace"; then
        echo "ERROR: Failed to create bind-mount targets inside enroot rootfs."
        echo "       Check permissions for: ${SERVER_ROOTFS} and ${EVAL_ROOTFS}"
        exit 1
    fi

    if command_exists flock; then
        flock -u 9 || true
        exec 9>&-
    fi
}

ensure_tmp_stage_root() {
    if [[ -n "$TMP_STAGE_ROOT" ]]; then
        return
    fi

    TMP_STAGE_ROOT="${TMPDIR:-/tmp}/llmdebug-${JOB_ID}"
    TMP_STAGE_MODEL_DIR="${TMP_STAGE_ROOT}/models"
    TMP_STAGE_REPO_DIR="${TMP_STAGE_ROOT}/repo"

    mkdir -p "$TMP_STAGE_MODEL_DIR" "$TMP_STAGE_REPO_DIR"
}

stage_model_to_tmpdir() {
    local src_size dst_size required_bytes

    ensure_tmp_stage_root

    src_size="$(stat -c %s "$MODEL_PATH")"
    required_bytes=$(( src_size + TMPDIR_HEADROOM_BYTES ))
    ensure_tmp_space "$required_bytes" "model staging"

    dst_size="$(stat -c %s "${TMP_STAGE_MODEL_DIR}/${MODEL_FILE}" 2>/dev/null || echo 0)"
    if [[ "$src_size" != "$dst_size" ]]; then
        echo "Staging model to TMPDIR..."
        cp -f "$MODEL_PATH" "${TMP_STAGE_MODEL_DIR}/${MODEL_FILE}"
    fi

    MODEL_MOUNT_DIR="$TMP_STAGE_MODEL_DIR"
    MODEL_STAGED=1
    USING_TMPDIR_STAGING=1
}

stage_repo_to_tmpdir() {
    local repo_size required_bytes

    ensure_tmp_stage_root

    repo_size="$(du -sb "$REPO_DIR_REAL" | awk '{print $1}')"
    required_bytes=$(( repo_size + TMPDIR_HEADROOM_BYTES ))
    ensure_tmp_space "$required_bytes" "repository staging"

    echo "Staging repo to TMPDIR..."
    if command_exists rsync; then
        rsync -a --delete \
            --exclude '.git' \
            --exclude '.llmdebug' \
            --exclude '.venv' \
            --exclude '.venv312' \
            --exclude '.pytest_cache' \
            --exclude '.ruff_cache' \
            --exclude '.uv-cache' \
            --exclude 'evals/results' \
            --exclude 'evals/artifacts' \
            "${REPO_DIR_REAL}/" "${TMP_STAGE_REPO_DIR}/"
    else
        rm -rf "$TMP_STAGE_REPO_DIR"
        mkdir -p "$TMP_STAGE_REPO_DIR"
        cp -a "${REPO_DIR_REAL}/." "$TMP_STAGE_REPO_DIR/"
    fi

    # Preserve run-resume state if present when staging to TMPDIR.
    if [[ -f "${REPO_DIR_REAL}/evals/results/${EVAL_RUN_ID}.jsonl" ]]; then
        mkdir -p "${TMP_STAGE_REPO_DIR}/evals/results"
        cp -f "${REPO_DIR_REAL}/evals/results/${EVAL_RUN_ID}.jsonl" "${TMP_STAGE_REPO_DIR}/evals/results/"
    fi

    REPO_MOUNT_DIR="$TMP_STAGE_REPO_DIR"
    REPO_STAGED=1
    USING_TMPDIR_STAGING=1
}

sync_staged_results_back() {
    local staged_results staged_artifacts

    if (( REPO_STAGED == 0 )); then
        return
    fi

    staged_results="${REPO_MOUNT_DIR}/evals/results/${EVAL_RUN_ID}.jsonl"
    staged_artifacts="${REPO_MOUNT_DIR}/evals/artifacts"

    if [[ -f "$staged_results" ]]; then
        mkdir -p "${REPO_DIR_REAL}/evals/results"
        cp -f "$staged_results" "${REPO_DIR_REAL}/evals/results/"
    fi

    if [[ -d "$staged_artifacts" ]]; then
        mkdir -p "${REPO_DIR_REAL}/evals/artifacts"
        if command_exists rsync; then
            rsync -a "${staged_artifacts}/" "${REPO_DIR_REAL}/evals/artifacts/"
        else
            cp -a "${staged_artifacts}/." "${REPO_DIR_REAL}/evals/artifacts/"
        fi
    fi
}

cleanup_staged_tmpdir() {
    if [[ -z "$TMP_STAGE_ROOT" || ! -d "$TMP_STAGE_ROOT" ]]; then
        return
    fi

    if (( JOB_EXIT_CODE == 0 && KEEP_TMPDIR_ON_SUCCESS == 0 )); then
        rm -rf "$TMP_STAGE_ROOT"
    fi

    if (( JOB_EXIT_CODE != 0 && KEEP_TMPDIR_ON_FAILURE == 0 )); then
        rm -rf "$TMP_STAGE_ROOT"
    fi
}

print_failure_diagnostics() {
    local reason
    reason="$1"

    echo "ERROR: ${reason}"
    echo "Related logs:"
    echo "  server log: ${SERVER_LOG}"
    echo "  eval log:   ${EVAL_LAUNCH_LOG}"
    echo "  diag log:   ${ENROOT_DIAG_LOG}"

    if [[ -f "$SERVER_LOG" ]]; then
        echo "Last 30 lines of server log:"
        tail -30 "$SERVER_LOG"
    fi

    if [[ -f "$EVAL_LAUNCH_LOG" ]]; then
        echo "Last 30 lines of eval log:"
        tail -30 "$EVAL_LAUNCH_LOG"
    fi

    if [[ -f "$ENROOT_DIAG_LOG" ]]; then
        echo "Last 30 lines of diagnostics log:"
        tail -30 "$ENROOT_DIAG_LOG"
    fi

    write_failure_dump "$reason" "${JOB_EXIT_CODE:-1}"
}

start_llama_cpp_server() {
    local mount_dir
    local -a enroot_cmd launch_cmd

    mount_dir="$1"
    MODEL_MOUNT_DIR="$mount_dir"

    enroot_cmd=(
        enroot start --rw
        -m "${MODEL_MOUNT_DIR}:/models"
        -e "XDG_RUNTIME_DIR=${CONTAINER_XDG_RUNTIME_DIR:-${HOST_XDG_RUNTIME_DIR:-/tmp/xdg-runtime-${UID:-0}}}"
        -e BASH_ENV=/dev/null
        -e ENV=/dev/null
    )

    if [[ -n "$CUDA_SCALE_LAUNCH_QUEUES" ]]; then
        enroot_cmd+=(-e "CUDA_SCALE_LAUNCH_QUEUES=${CUDA_SCALE_LAUNCH_QUEUES}")
    fi
    if [[ -n "$GGML_CUDA_FORCE_CUBLAS" ]]; then
        enroot_cmd+=(-e "GGML_CUDA_FORCE_CUBLAS=${GGML_CUDA_FORCE_CUBLAS}")
    fi
    if [[ -n "$GGML_CUDA_FORCE_MMQ" ]]; then
        enroot_cmd+=(-e "GGML_CUDA_FORCE_MMQ=${GGML_CUDA_FORCE_MMQ}")
    fi
    if [[ -n "$GGML_CUDA_ENABLE_UNIFIED_MEMORY" ]]; then
        enroot_cmd+=(-e "GGML_CUDA_ENABLE_UNIFIED_MEMORY=${GGML_CUDA_ENABLE_UNIFIED_MEMORY}")
    fi

    enroot_cmd+=(
        -- "$SERVER_CONTAINER"
        --model "/models/${MODEL_FILE}"
        --alias "$MODEL_ALIAS"
        --port "$SERVER_PORT"
        --host 127.0.0.1
        --ctx-size "$CONTEXT_SIZE"
        --n-gpu-layers "$GPU_LAYERS"
        --parallel "$SERVER_PARALLEL"
        --batch-size "$SERVER_BATCH_SIZE"
        --ubatch-size "$SERVER_UBATCH_SIZE"
    )

    if [[ -n "$SERVER_FLASH_ATTN" ]]; then
        enroot_cmd+=(--flash-attn "$SERVER_FLASH_ATTN")
    fi
    if [[ -n "$SERVER_THREADS" ]]; then
        enroot_cmd+=(--threads "$SERVER_THREADS")
    fi
    if [[ -n "$SERVER_THREADS_BATCH" ]]; then
        enroot_cmd+=(--threads-batch "$SERVER_THREADS_BATCH")
    fi
    if [[ -n "$SERVER_THREADS_HTTP" ]]; then
        enroot_cmd+=(--threads-http "$SERVER_THREADS_HTTP")
    fi
    if (( SERVER_METRICS == 1 )); then
        enroot_cmd+=(--metrics)
    fi

    if [[ -n "$SERVER_TEMPERATURE" ]]; then
        enroot_cmd+=(--temp "$SERVER_TEMPERATURE")
    fi
    if [[ -n "$SERVER_TOP_P" ]]; then
        enroot_cmd+=(--top-p "$SERVER_TOP_P")
    fi
    if [[ -n "$SERVER_TOP_K" ]]; then
        enroot_cmd+=(--top-k "$SERVER_TOP_K")
    fi
    if [[ -n "$SERVER_MIN_P" ]]; then
        enroot_cmd+=(--min-p "$SERVER_MIN_P")
    fi
    if [[ -n "$SERVER_REPEAT_PENALTY" ]]; then
        enroot_cmd+=(--repeat-penalty "$SERVER_REPEAT_PENALTY")
    fi
    if [[ -n "$SERVER_CACHE_TYPE_K" ]]; then
        enroot_cmd+=(--cache-type-k "$SERVER_CACHE_TYPE_K")
    fi
    if [[ -n "$SERVER_CACHE_TYPE_V" ]]; then
        enroot_cmd+=(--cache-type-v "$SERVER_CACHE_TYPE_V")
    fi

    launch_cmd=()
    if (( SERVER_USE_SRUN == 1 )); then
        launch_cmd+=(
            srun
            --ntasks=1
            --nodes=1
            --cpu-bind="${SERVER_SRUN_CPU_BIND}"
            --gpu-bind="${SERVER_SRUN_GPU_BIND}"
        )
        if [[ -n "${SERVER_SRUN_CPUS_PER_TASK:-}" ]]; then
            launch_cmd+=(--cpus-per-task "${SERVER_SRUN_CPUS_PER_TASK}")
        fi
        if (( SERVER_SRUN_UNBUFFERED == 1 )); then
            launch_cmd+=(--unbuffered)
        fi
    fi
    launch_cmd+=("${enroot_cmd[@]}")

    {
        echo "[$(date -Iseconds)] launching llama.cpp server"
        echo "backend: llama_cpp"
        echo "model source mount: ${MODEL_MOUNT_DIR} -> /models"
        printf 'command: %q ' "${launch_cmd[@]}"
        echo ""
        echo "ENROOT_DEBUG enabled: ${ENABLE_ENROOT_DEBUG}"
        echo "diagnostics log: ${ENROOT_DIAG_LOG}"
        echo ""

        if (( ENABLE_ENROOT_DEBUG == 1 )); then
            ENROOT_DEBUG=1 "${launch_cmd[@]}"
        else
            "${launch_cmd[@]}"
        fi
    } > "$SERVER_LOG" 2>&1 &

    SERVER_PID=$!
    SERVER_PROCESS_PATTERN="llama-server.*${SERVER_PORT}"
}

start_vllm_server() {
    local -a enroot_cmd launch_cmd
    local vllm_download_dir_in_container

    MODEL_MOUNT_DIR=""
    vllm_download_dir_in_container=""

    enroot_cmd=(
        enroot start --rw
        -e "XDG_RUNTIME_DIR=${CONTAINER_XDG_RUNTIME_DIR:-${HOST_XDG_RUNTIME_DIR:-/tmp/xdg-runtime-${UID:-0}}}"
        -e BASH_ENV=/dev/null
        -e ENV=/dev/null
    )

    if [[ -n "${HF_CACHE_DIR:-}" ]]; then
        enroot_cmd+=(
            -m "${HF_CACHE_DIR}:/cache"
            -e HF_HOME=/cache
            -e HUGGINGFACE_HUB_CACHE=/cache/hub
        )
    fi
    if [[ -n "${VLLM_DOWNLOAD_DIR:-}" ]]; then
        vllm_download_dir_in_container="/download"
        enroot_cmd+=(-m "${VLLM_DOWNLOAD_DIR}:${vllm_download_dir_in_container}")
    fi

    # vllm/vllm-openai image entrypoint already launches the OpenAI server.
    # Pass engine/server args directly (docker-style), and keep model positional
    # for forward-compatibility with vLLM >= 0.13 deprecating --model in serve mode.
    enroot_cmd+=(
        -- "$SERVER_CONTAINER"
        "$HF_MODEL_ID"
        --host 127.0.0.1
        --port "$SERVER_PORT"
        --served-model-name "$MODEL_ALIAS"
        --max-model-len "$VLLM_MAX_MODEL_LEN"
        --gpu-memory-utilization "$VLLM_GPU_MEMORY_UTILIZATION"
        --tensor-parallel-size "$VLLM_TENSOR_PARALLEL_SIZE"
        --dtype "$VLLM_DTYPE"
        --kv-cache-dtype "$VLLM_KV_CACHE_DTYPE"
        --max-num-seqs "$VLLM_MAX_NUM_SEQS"
    )

    if (( VLLM_TRUST_REMOTE_CODE == 1 )); then
        enroot_cmd+=(--trust-remote-code)
    fi
    if (( VLLM_CALCULATE_KV_SCALES == 1 )); then
        enroot_cmd+=(--calculate-kv-scales)
    fi
    if [[ -n "${VLLM_MAX_NUM_BATCHED_TOKENS:-}" ]]; then
        enroot_cmd+=(--max-num-batched-tokens "$VLLM_MAX_NUM_BATCHED_TOKENS")
    fi
    if [[ -n "${vllm_download_dir_in_container}" ]]; then
        enroot_cmd+=(--download-dir "$vllm_download_dir_in_container")
    fi

    launch_cmd=()
    if (( SERVER_USE_SRUN == 1 )); then
        launch_cmd+=(
            srun
            --ntasks=1
            --nodes=1
            --cpu-bind="${SERVER_SRUN_CPU_BIND}"
            --gpu-bind="${SERVER_SRUN_GPU_BIND}"
        )
        if [[ -n "${SERVER_SRUN_CPUS_PER_TASK:-}" ]]; then
            launch_cmd+=(--cpus-per-task "${SERVER_SRUN_CPUS_PER_TASK}")
        fi
        if (( SERVER_SRUN_UNBUFFERED == 1 )); then
            launch_cmd+=(--unbuffered)
        fi
    fi
    launch_cmd+=("${enroot_cmd[@]}")

    {
        echo "[$(date -Iseconds)] launching vLLM server"
        echo "backend: vllm"
        echo "hf model: ${HF_MODEL_ID}"
        if [[ -n "${HF_CACHE_DIR:-}" ]]; then
            echo "hf cache mount: ${HF_CACHE_DIR} -> /cache"
        fi
        if [[ -n "${vllm_download_dir_in_container}" ]]; then
            echo "vllm download mount: ${VLLM_DOWNLOAD_DIR} -> ${vllm_download_dir_in_container}"
        fi
        printf 'command: %q ' "${launch_cmd[@]}"
        echo ""
        echo "ENROOT_DEBUG enabled: ${ENABLE_ENROOT_DEBUG}"
        echo "diagnostics log: ${ENROOT_DIAG_LOG}"
        echo ""

        if (( ENABLE_ENROOT_DEBUG == 1 )); then
            ENROOT_DEBUG=1 "${launch_cmd[@]}"
        else
            "${launch_cmd[@]}"
        fi
    } > "$SERVER_LOG" 2>&1 &

    SERVER_PID=$!
    SERVER_PROCESS_PATTERN="vllm.*${SERVER_PORT}"
}

start_server() {
    case "$SERVER_BACKEND" in
        llama_cpp)
            start_llama_cpp_server "$MODEL_MOUNT_DIR"
            ;;
        vllm)
            start_vllm_server
            ;;
        *)
            echo "ERROR: Unsupported SERVER_BACKEND='$SERVER_BACKEND'"
            exit 1
            ;;
    esac
}

run_eval_once() {
    local container_xdg_runtime_dir eval_start_epoch rc
    local -a eval_cmd
    container_xdg_runtime_dir="$(sanitize_runtime_dir \
        "CONTAINER_XDG_RUNTIME_DIR" \
        "${CONTAINER_XDG_RUNTIME_DIR:-${HOST_XDG_RUNTIME_DIR:-${DEFAULT_XDG_RUNTIME_DIR:-/tmp/xdg-runtime-${UID:-0}}}}" \
        "${DEFAULT_XDG_RUNTIME_DIR:-/tmp/xdg-runtime-${UID:-0}}")"

    eval_cmd=(
        enroot start --rw
        -m "${REPO_MOUNT_DIR}:/workspace"
        -e PYTHONPATH=/workspace/src
        -e XDG_RUNTIME_DIR="${container_xdg_runtime_dir}"
        -e BASH_ENV=/dev/null
        -e ENV=/dev/null
        -- "$EVAL_CONTAINER"
        bash -c '
            mkdir -p "${XDG_RUNTIME_DIR:-/tmp}" >/dev/null 2>&1 || true
            chmod 700 "${XDG_RUNTIME_DIR:-/tmp}" >/dev/null 2>&1 || true
            cd /workspace && exec "$@"
        ' _
        python -m evals.run_eval "${EVAL_ARGS[@]}"
    )

    set +e
    # Some site shell hooks target /run/user/<uid> (not writable in container sessions).
    # Pin a writable runtime dir and neutralize BASH_ENV startup hooks.
    (
        "${eval_cmd[@]}" 2>&1 | tee "$EVAL_LAUNCH_LOG"
    ) &
    EVAL_LAUNCH_PID=$!

    GPU_FAILFAST_START_LINE="$(wc -l < "$GPU_UTIL_LOG" 2>/dev/null || echo 1)"
    eval_start_epoch="$(date +%s)"
    start_gpu_failfast_watchdog "$EVAL_LAUNCH_PID" "$eval_start_epoch"
    start_server_failfast_watchdog "$EVAL_LAUNCH_PID"

    wait "$EVAL_LAUNCH_PID"
    rc=$?
    EVAL_LAUNCH_PID=""
    stop_server_failfast_watchdog
    stop_gpu_failfast_watchdog

    set -e
    return "$rc"
}

cleanup() {
    JOB_EXIT_CODE=$?

    stop_server_failfast_watchdog
    stop_gpu_failfast_watchdog

    if [[ -n "${SERVER_PID:-}" ]]; then
        echo "Cleaning up LLM server (backend=${SERVER_BACKEND}, PID=$SERVER_PID)..."
        kill "$SERVER_PID" 2>/dev/null || true
        if [[ -n "${SERVER_PROCESS_PATTERN:-}" ]]; then
            pkill -f "${SERVER_PROCESS_PATTERN}" 2>/dev/null || true
        fi
        wait "$SERVER_PID" 2>/dev/null || true
    fi

    stop_gpu_monitor
    summarize_gpu_monitor

    sync_staged_results_back

    if (( JOB_EXIT_CODE != 0 )); then
        write_failure_dump "job exited with failure" "$JOB_EXIT_CODE"
    fi

    cleanup_staged_tmpdir

    echo "Finished: $(date -Iseconds) (exit=${JOB_EXIT_CODE})"
    exit "$JOB_EXIT_CODE"
}

# Detect explicit overrides (needed for smoke mode without clobbering user intent).
HAS_EVAL_MAX_CASES_OVERRIDE=0
[[ -n "${EVAL_MAX_CASES+x}" ]] && HAS_EVAL_MAX_CASES_OVERRIDE=1
HAS_EVAL_K_OVERRIDE=0
[[ -n "${EVAL_K+x}" ]] && HAS_EVAL_K_OVERRIDE=1
HAS_EVAL_MAX_CONTEXT_CHARS_OVERRIDE=0
[[ -n "${EVAL_MAX_CONTEXT_CHARS+x}" ]] && HAS_EVAL_MAX_CONTEXT_CHARS_OVERRIDE=1
HAS_EVAL_PROMPT_BUDGET_TOTAL_CHARS_OVERRIDE=0
[[ -n "${EVAL_PROMPT_BUDGET_TOTAL_CHARS+x}" ]] && HAS_EVAL_PROMPT_BUDGET_TOTAL_CHARS_OVERRIDE=1
HAS_CONTEXT_SIZE_OVERRIDE=0
[[ -n "${CONTEXT_SIZE+x}" ]] && HAS_CONTEXT_SIZE_OVERRIDE=1
HAS_VLLM_MAX_MODEL_LEN_OVERRIDE=0
[[ -n "${VLLM_MAX_MODEL_LEN+x}" ]] && HAS_VLLM_MAX_MODEL_LEN_OVERRIDE=1
HAS_MAX_WAIT_OVERRIDE=0
[[ -n "${MAX_WAIT+x}" ]] && HAS_MAX_WAIT_OVERRIDE=1
HAS_SERVER_PORT_OVERRIDE=0
[[ -n "${SERVER_PORT+x}" ]] && HAS_SERVER_PORT_OVERRIDE=1
HAS_EVAL_RUN_ID_OVERRIDE=0
[[ -n "${EVAL_RUN_ID+x}" ]] && HAS_EVAL_RUN_ID_OVERRIDE=1
HAS_MODEL_DIR_OVERRIDE=0
[[ -n "${MODEL_DIR+x}" ]] && HAS_MODEL_DIR_OVERRIDE=1
HAS_MODEL_FILE_OVERRIDE=0
[[ -n "${MODEL_FILE+x}" ]] && HAS_MODEL_FILE_OVERRIDE=1

# ── User-configurable defaults (overridable via environment) ───────────────
DEFAULT_HOME_DIR="${HOME:-}"
if [[ -z "$DEFAULT_HOME_DIR" ]]; then
    DEFAULT_HOME_DIR="/home/${USER:-user}"
fi
DEFAULT_REPO_BASENAME="llmdebug"
DEFAULT_MODEL_PRESET="qwen3coder30ba3b-q4_k_m"
DEFAULT_SERVER_BACKEND="vllm"
DEFAULT_SERVER_CONTAINER_VLLM="vllm-server-cuda"
DEFAULT_SERVER_CONTAINER_LLAMA_CPP="llama-server-cuda"
DEFAULT_EVAL_CONTAINER="python-eval"
DEFAULT_HF_MODEL_ID="Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
DEFAULT_HF_MODEL_ALIAS="qwen3coder30ba3b"
DEFAULT_HF_RUN_ID_PREFIX="vllm-qwen3coder30ba3b"
DEFAULT_VLLM_MAX_MODEL_LEN=32768
DEFAULT_VLLM_GPU_MEMORY_UTILIZATION="0.92"
DEFAULT_VLLM_TENSOR_PARALLEL_SIZE=1
DEFAULT_VLLM_DTYPE="auto"
DEFAULT_VLLM_KV_CACHE_DTYPE="auto"
DEFAULT_VLLM_CALCULATE_KV_SCALES=0
DEFAULT_VLLM_MAX_NUM_SEQS=1
DEFAULT_VLLM_MAX_NUM_BATCHED_TOKENS=""
DEFAULT_VLLM_TRUST_REMOTE_CODE=0
DEFAULT_VLLM_DOWNLOAD_DIR=""
DEFAULT_HF_CACHE_DIR=""

DEFAULT_GPU_LAYERS=99
DEFAULT_EVAL_K=3
DEFAULT_EVAL_JOBS=4
DEFAULT_EVAL_MAX_CASES=0
DEFAULT_EVAL_CATEGORY=""
DEFAULT_EVAL_CONDITIONS="traceback_only,adaptive_llm_discretion"
DEFAULT_MAX_WAIT=600
DEFAULT_POLL_INTERVAL=5
DEFAULT_SERVER_PORT=8000
DEFAULT_OPENAI_TEMPERATURE=0.0
DEFAULT_OPENAI_TIMEOUT_SEC=180
DEFAULT_OPENAI_MAX_TOKENS=2048
DEFAULT_OPENAI_MAX_PROPOSE_SEC=300
DEFAULT_SERVER_FLASH_ATTN="off"
DEFAULT_SERVER_BATCH_SIZE=512
DEFAULT_SERVER_UBATCH_SIZE=128
DEFAULT_SERVER_THREADS=""
DEFAULT_SERVER_THREADS_HTTP=""
DEFAULT_SERVER_METRICS=0
DEFAULT_SERVER_CACHE_TYPE_K=""
DEFAULT_SERVER_CACHE_TYPE_V=""
DEFAULT_SERVER_USE_SRUN=1
DEFAULT_SERVER_SRUN_CPU_BIND="cores"
DEFAULT_SERVER_SRUN_GPU_BIND="single:1"
DEFAULT_SERVER_SRUN_UNBUFFERED=1
DEFAULT_AUTO_CONTEXT_BUDGETS=1
DEFAULT_CONTEXT_CHARS_PER_TOKEN=1
DEFAULT_CONTEXT_PROMPT_HEADROOM_PCT=90
DEFAULT_LOW_PER_SLOT_CTX_WARN=4096
DEFAULT_EVAL_MAX_CONTEXT_CHARS=200000
DEFAULT_EVAL_PROMPT_BUDGET_TOTAL_CHARS=180000
DEFAULT_GPU_FAILFAST=1
DEFAULT_GPU_FAILFAST_WARMUP_SEC=1800
DEFAULT_GPU_FAILFAST_UTIL_PCT=25
DEFAULT_GPU_FAILFAST_LOW_FRAC_PCT=95
DEFAULT_GPU_FAILFAST_MIN_SAMPLES=30
DEFAULT_GPU_FAILFAST_CHECK_SEC=60
DEFAULT_SERVER_FAILFAST=1
DEFAULT_SERVER_FAILFAST_CHECK_SEC=20
DEFAULT_SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS=12

# Smoke defaults (small/fast pilot).
DEFAULT_SMOKE_TEST_MAX_CASES=3
DEFAULT_SMOKE_TEST_EVAL_K=1
DEFAULT_SMOKE_TEST_CONTEXT_SIZE=2048
DEFAULT_SMOKE_TEST_MAX_WAIT_SEC=240

HOME_DIR="${HOME_DIR:-$DEFAULT_HOME_DIR}"
REPO_DIR="${REPO_DIR:-${HOME_DIR}/${DEFAULT_REPO_BASENAME}}"
SERVER_BACKEND="${SERVER_BACKEND:-$DEFAULT_SERVER_BACKEND}"
normalize_server_backend SERVER_BACKEND

HF_MODEL_ID="${HF_MODEL_ID:-$DEFAULT_HF_MODEL_ID}"
HF_MODEL_ALIAS="${HF_MODEL_ALIAS:-$DEFAULT_HF_MODEL_ALIAS}"
HF_RUN_ID_PREFIX="${HF_RUN_ID_PREFIX:-$DEFAULT_HF_RUN_ID_PREFIX}"
VLLM_MAX_MODEL_LEN="${VLLM_MAX_MODEL_LEN:-$DEFAULT_VLLM_MAX_MODEL_LEN}"
VLLM_GPU_MEMORY_UTILIZATION="${VLLM_GPU_MEMORY_UTILIZATION:-$DEFAULT_VLLM_GPU_MEMORY_UTILIZATION}"
VLLM_TENSOR_PARALLEL_SIZE="${VLLM_TENSOR_PARALLEL_SIZE:-$DEFAULT_VLLM_TENSOR_PARALLEL_SIZE}"
VLLM_DTYPE="${VLLM_DTYPE:-$DEFAULT_VLLM_DTYPE}"
VLLM_KV_CACHE_DTYPE="${VLLM_KV_CACHE_DTYPE:-$DEFAULT_VLLM_KV_CACHE_DTYPE}"
VLLM_CALCULATE_KV_SCALES="${VLLM_CALCULATE_KV_SCALES:-$DEFAULT_VLLM_CALCULATE_KV_SCALES}"
VLLM_MAX_NUM_SEQS="${VLLM_MAX_NUM_SEQS:-$DEFAULT_VLLM_MAX_NUM_SEQS}"
VLLM_MAX_NUM_BATCHED_TOKENS="${VLLM_MAX_NUM_BATCHED_TOKENS:-$DEFAULT_VLLM_MAX_NUM_BATCHED_TOKENS}"
VLLM_TRUST_REMOTE_CODE="${VLLM_TRUST_REMOTE_CODE:-$DEFAULT_VLLM_TRUST_REMOTE_CODE}"
VLLM_DOWNLOAD_DIR="${VLLM_DOWNLOAD_DIR:-$DEFAULT_VLLM_DOWNLOAD_DIR}"
HF_CACHE_DIR="${HF_CACHE_DIR:-$DEFAULT_HF_CACHE_DIR}"

if [[ "$SERVER_BACKEND" == "llama_cpp" ]]; then
    MODEL_PRESET="${MODEL_PRESET:-$DEFAULT_MODEL_PRESET}"
    resolve_model_preset "$MODEL_PRESET" "$HOME_DIR"
    SERVER_CONTAINER="${SERVER_CONTAINER:-$DEFAULT_SERVER_CONTAINER_LLAMA_CPP}"
else
    MODEL_PRESET="${MODEL_PRESET:-hf-qwen3coder30ba3b-fp8}"
    PRESET_CANONICAL="$MODEL_PRESET"
    PRESET_MODEL_DIR=""
    PRESET_MODEL_FILE=""
    PRESET_MODEL_ALIAS="$HF_MODEL_ALIAS"
    PRESET_CONTEXT_SIZE="$VLLM_MAX_MODEL_LEN"
    PRESET_MODEL_CTX_TRAIN=""
    PRESET_SERVER_PARALLEL=1
    PRESET_RUN_ID_PREFIX="$HF_RUN_ID_PREFIX"
    PRESET_MODEL_SOURCE="$HF_MODEL_ID"
    PRESET_SERVER_TEMPERATURE=""
    PRESET_SERVER_TOP_P=""
    PRESET_SERVER_TOP_K=""
    PRESET_SERVER_MIN_P=""
    PRESET_SERVER_REPEAT_PENALTY=""
    PRESET_OPENAI_TEMPERATURE="$DEFAULT_OPENAI_TEMPERATURE"
    SERVER_CONTAINER="${SERVER_CONTAINER:-$DEFAULT_SERVER_CONTAINER_VLLM}"
fi

MODEL_PRESET="$PRESET_CANONICAL"
MODEL_SOURCE="$PRESET_MODEL_SOURCE"
MODEL_CTX_TRAIN_TOKENS="${PRESET_MODEL_CTX_TRAIN:-}"
MODEL_DIR="${MODEL_DIR:-$PRESET_MODEL_DIR}"
MODEL_FILE="${MODEL_FILE:-$PRESET_MODEL_FILE}"

EVAL_CONTAINER="${EVAL_CONTAINER:-$DEFAULT_EVAL_CONTAINER}"
MODEL_ALIAS="${MODEL_ALIAS:-$PRESET_MODEL_ALIAS}"
if [[ "$SERVER_BACKEND" == "vllm" ]]; then
    HF_MODEL_ALIAS="$MODEL_ALIAS"
fi
CONTEXT_SIZE="${CONTEXT_SIZE:-$PRESET_CONTEXT_SIZE}"
GPU_LAYERS="${GPU_LAYERS:-$DEFAULT_GPU_LAYERS}"
EVAL_K="${EVAL_K:-$DEFAULT_EVAL_K}"
EVAL_JOBS="${EVAL_JOBS:-$DEFAULT_EVAL_JOBS}"
EVAL_MAX_CASES="${EVAL_MAX_CASES:-$DEFAULT_EVAL_MAX_CASES}"
EVAL_CATEGORY="${EVAL_CATEGORY:-$DEFAULT_EVAL_CATEGORY}"
EVAL_CONDITIONS="${EVAL_CONDITIONS:-$DEFAULT_EVAL_CONDITIONS}"
MAX_WAIT="${MAX_WAIT:-$DEFAULT_MAX_WAIT}"
POLL_INTERVAL="${POLL_INTERVAL:-$DEFAULT_POLL_INTERVAL}"
OPENAI_TEMPERATURE="${OPENAI_TEMPERATURE:-${PRESET_OPENAI_TEMPERATURE:-$DEFAULT_OPENAI_TEMPERATURE}}"
OPENAI_TIMEOUT_SEC="${OPENAI_TIMEOUT_SEC:-$DEFAULT_OPENAI_TIMEOUT_SEC}"
OPENAI_MAX_TOKENS="${OPENAI_MAX_TOKENS:-$DEFAULT_OPENAI_MAX_TOKENS}"
OPENAI_MAX_PROPOSE_SEC="${OPENAI_MAX_PROPOSE_SEC:-$DEFAULT_OPENAI_MAX_PROPOSE_SEC}"
SERVER_PARALLEL="${SERVER_PARALLEL:-${PRESET_SERVER_PARALLEL:-$EVAL_JOBS}}"
SERVER_FLASH_ATTN="${SERVER_FLASH_ATTN:-$DEFAULT_SERVER_FLASH_ATTN}"
SERVER_BATCH_SIZE="${SERVER_BATCH_SIZE:-$DEFAULT_SERVER_BATCH_SIZE}"
SERVER_UBATCH_SIZE="${SERVER_UBATCH_SIZE:-$DEFAULT_SERVER_UBATCH_SIZE}"
SERVER_THREADS="${SERVER_THREADS:-${SLURM_CPUS_PER_TASK:-${SLURM_CPUS_PER_GPU:-$DEFAULT_SERVER_THREADS}}}"
SERVER_THREADS_BATCH="${SERVER_THREADS_BATCH:-$SERVER_THREADS}"
SERVER_THREADS_HTTP="${SERVER_THREADS_HTTP:-$DEFAULT_SERVER_THREADS_HTTP}"
SERVER_METRICS="${SERVER_METRICS:-$DEFAULT_SERVER_METRICS}"
SERVER_CACHE_TYPE_K="${SERVER_CACHE_TYPE_K:-$DEFAULT_SERVER_CACHE_TYPE_K}"
SERVER_CACHE_TYPE_V="${SERVER_CACHE_TYPE_V:-$DEFAULT_SERVER_CACHE_TYPE_V}"
SERVER_USE_SRUN="${SERVER_USE_SRUN:-$DEFAULT_SERVER_USE_SRUN}"
SERVER_SRUN_CPU_BIND="${SERVER_SRUN_CPU_BIND:-$DEFAULT_SERVER_SRUN_CPU_BIND}"
SERVER_SRUN_GPU_BIND="${SERVER_SRUN_GPU_BIND:-$DEFAULT_SERVER_SRUN_GPU_BIND}"
SERVER_SRUN_CPUS_PER_TASK="${SERVER_SRUN_CPUS_PER_TASK:-${SLURM_CPUS_PER_TASK:-${SLURM_CPUS_PER_GPU:-}}}"
SERVER_SRUN_UNBUFFERED="${SERVER_SRUN_UNBUFFERED:-$DEFAULT_SERVER_SRUN_UNBUFFERED}"
CUDA_SCALE_LAUNCH_QUEUES="${CUDA_SCALE_LAUNCH_QUEUES:-}"
GGML_CUDA_FORCE_CUBLAS="${GGML_CUDA_FORCE_CUBLAS:-}"
GGML_CUDA_FORCE_MMQ="${GGML_CUDA_FORCE_MMQ:-}"
GGML_CUDA_ENABLE_UNIFIED_MEMORY="${GGML_CUDA_ENABLE_UNIFIED_MEMORY:-}"

SERVER_TEMPERATURE="${SERVER_TEMPERATURE:-$PRESET_SERVER_TEMPERATURE}"
SERVER_TOP_P="${SERVER_TOP_P:-$PRESET_SERVER_TOP_P}"
SERVER_TOP_K="${SERVER_TOP_K:-$PRESET_SERVER_TOP_K}"
SERVER_MIN_P="${SERVER_MIN_P:-$PRESET_SERVER_MIN_P}"
SERVER_REPEAT_PENALTY="${SERVER_REPEAT_PENALTY:-$PRESET_SERVER_REPEAT_PENALTY}"

DRY_RUN="${DRY_RUN:-0}"
SMOKE_TEST_MODE="${SMOKE_TEST_MODE:-0}"
SMOKE_TEST_MAX_CASES="${SMOKE_TEST_MAX_CASES:-$DEFAULT_SMOKE_TEST_MAX_CASES}"
SMOKE_TEST_EVAL_K="${SMOKE_TEST_EVAL_K:-$DEFAULT_SMOKE_TEST_EVAL_K}"
SMOKE_TEST_CONTEXT_SIZE="${SMOKE_TEST_CONTEXT_SIZE:-$DEFAULT_SMOKE_TEST_CONTEXT_SIZE}"
SMOKE_TEST_MAX_WAIT_SEC="${SMOKE_TEST_MAX_WAIT_SEC:-$DEFAULT_SMOKE_TEST_MAX_WAIT_SEC}"

EVAL_DETERMINISTIC="${EVAL_DETERMINISTIC:-0}"
EVAL_SEED="${EVAL_SEED:-}"

ENABLE_ENROOT_DEBUG="${ENABLE_ENROOT_DEBUG:-1}"
ENABLE_TMPDIR_FALLBACK="${ENABLE_TMPDIR_FALLBACK:-1}"
KEEP_TMPDIR_ON_SUCCESS="${KEEP_TMPDIR_ON_SUCCESS:-0}"
KEEP_TMPDIR_ON_FAILURE="${KEEP_TMPDIR_ON_FAILURE:-1}"
TMPDIR_HEADROOM_BYTES="${TMPDIR_HEADROOM_BYTES:-2147483648}"

GPU_UTIL_MONITOR="${GPU_UTIL_MONITOR:-1}"
GPU_UTIL_POLL_SEC="${GPU_UTIL_POLL_SEC:-10}"
GPU_FAILFAST="${GPU_FAILFAST:-$DEFAULT_GPU_FAILFAST}"
GPU_FAILFAST_WARMUP_SEC="${GPU_FAILFAST_WARMUP_SEC:-$DEFAULT_GPU_FAILFAST_WARMUP_SEC}"
GPU_FAILFAST_UTIL_PCT="${GPU_FAILFAST_UTIL_PCT:-$DEFAULT_GPU_FAILFAST_UTIL_PCT}"
GPU_FAILFAST_LOW_FRAC_PCT="${GPU_FAILFAST_LOW_FRAC_PCT:-$DEFAULT_GPU_FAILFAST_LOW_FRAC_PCT}"
GPU_FAILFAST_MIN_SAMPLES="${GPU_FAILFAST_MIN_SAMPLES:-$DEFAULT_GPU_FAILFAST_MIN_SAMPLES}"
GPU_FAILFAST_CHECK_SEC="${GPU_FAILFAST_CHECK_SEC:-$DEFAULT_GPU_FAILFAST_CHECK_SEC}"
SERVER_FAILFAST="${SERVER_FAILFAST:-$DEFAULT_SERVER_FAILFAST}"
SERVER_FAILFAST_CHECK_SEC="${SERVER_FAILFAST_CHECK_SEC:-$DEFAULT_SERVER_FAILFAST_CHECK_SEC}"
SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS="${SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS:-$DEFAULT_SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS}"
RUN_CONFIG_SNAPSHOT="${RUN_CONFIG_SNAPSHOT:-1}"
FAILURE_DUMP_MODE="${FAILURE_DUMP_MODE:-1}"
AUTO_CONTEXT_BUDGETS="${AUTO_CONTEXT_BUDGETS:-$DEFAULT_AUTO_CONTEXT_BUDGETS}"
CONTEXT_CHARS_PER_TOKEN="${CONTEXT_CHARS_PER_TOKEN:-$DEFAULT_CONTEXT_CHARS_PER_TOKEN}"
CONTEXT_PROMPT_HEADROOM_PCT="${CONTEXT_PROMPT_HEADROOM_PCT:-$DEFAULT_CONTEXT_PROMPT_HEADROOM_PCT}"
LOW_PER_SLOT_CTX_WARN="${LOW_PER_SLOT_CTX_WARN:-$DEFAULT_LOW_PER_SLOT_CTX_WARN}"
EVAL_MAX_CONTEXT_CHARS="${EVAL_MAX_CONTEXT_CHARS:-$DEFAULT_EVAL_MAX_CONTEXT_CHARS}"
EVAL_PROMPT_BUDGET_TOTAL_CHARS="${EVAL_PROMPT_BUDGET_TOTAL_CHARS:-$DEFAULT_EVAL_PROMPT_BUDGET_TOTAL_CHARS}"

normalize_flag DRY_RUN
normalize_flag SMOKE_TEST_MODE
normalize_flag EVAL_DETERMINISTIC
normalize_flag ENABLE_ENROOT_DEBUG
normalize_flag ENABLE_TMPDIR_FALLBACK
normalize_flag KEEP_TMPDIR_ON_SUCCESS
normalize_flag KEEP_TMPDIR_ON_FAILURE
normalize_flag GPU_UTIL_MONITOR
normalize_flag GPU_FAILFAST
normalize_flag SERVER_FAILFAST
normalize_flag RUN_CONFIG_SNAPSHOT
normalize_flag FAILURE_DUMP_MODE
normalize_flag SERVER_METRICS
normalize_flag AUTO_CONTEXT_BUDGETS
normalize_flag SERVER_USE_SRUN
normalize_flag SERVER_SRUN_UNBUFFERED
normalize_flag VLLM_TRUST_REMOTE_CODE
normalize_flag VLLM_CALCULATE_KV_SCALES

# Some HAICore environments provide /run/user/<uid> paths that are not writable in
# batch/container contexts. Pin a writable runtime dir for enroot and child processes.
DEFAULT_XDG_RUNTIME_DIR="/tmp/xdg-runtime-${UID:-0}"
HOST_XDG_RUNTIME_DIR="$(sanitize_runtime_dir \
    "HOST_XDG_RUNTIME_DIR" \
    "${HOST_XDG_RUNTIME_DIR:-${XDG_RUNTIME_DIR:-${DEFAULT_XDG_RUNTIME_DIR}}}" \
    "${DEFAULT_XDG_RUNTIME_DIR}")"
CONTAINER_XDG_RUNTIME_DIR="$(sanitize_runtime_dir \
    "CONTAINER_XDG_RUNTIME_DIR" \
    "${CONTAINER_XDG_RUNTIME_DIR:-${HOST_XDG_RUNTIME_DIR}}" \
    "${DEFAULT_XDG_RUNTIME_DIR}")"
XDG_RUNTIME_DIR="${HOST_XDG_RUNTIME_DIR}"
export XDG_RUNTIME_DIR HOST_XDG_RUNTIME_DIR CONTAINER_XDG_RUNTIME_DIR

# Neutralize host/site BASH_ENV hooks that can trigger permission errors in non-interactive shells.
BASH_ENV=/dev/null
ENV=/dev/null
export BASH_ENV ENV

if [[ "$SERVER_BACKEND" == "llama_cpp" && -n "$SERVER_FLASH_ATTN" ]]; then
    case "${SERVER_FLASH_ATTN,,}" in
        on|off|auto)
            SERVER_FLASH_ATTN="${SERVER_FLASH_ATTN,,}"
            ;;
        *)
            echo "ERROR: SERVER_FLASH_ATTN must be one of: on/off/auto"
            echo "       Got: '${SERVER_FLASH_ATTN}'"
            exit 1
            ;;
    esac
fi

validate_int_ge SERVER_PARALLEL 1
validate_int_ge SERVER_BATCH_SIZE 1
validate_int_ge SERVER_UBATCH_SIZE 1
if [[ "$SERVER_BACKEND" == "vllm" ]]; then
    validate_int_ge VLLM_MAX_MODEL_LEN 1
    validate_int_ge VLLM_TENSOR_PARALLEL_SIZE 1
    validate_int_ge VLLM_MAX_NUM_SEQS 1
    validate_float_range VLLM_GPU_MEMORY_UTILIZATION 0.1 1.0
    if [[ -z "${HF_MODEL_ID//[[:space:]]/}" ]]; then
        echo "ERROR: HF_MODEL_ID must not be empty when SERVER_BACKEND=vllm."
        exit 1
    fi
    if [[ -z "${HF_MODEL_ALIAS//[[:space:]]/}" ]]; then
        echo "ERROR: HF_MODEL_ALIAS must not be empty when SERVER_BACKEND=vllm."
        exit 1
    fi
    if [[ -z "${VLLM_DTYPE//[[:space:]]/}" ]]; then
        echo "ERROR: VLLM_DTYPE must not be empty when SERVER_BACKEND=vllm."
        exit 1
    fi
    if [[ -z "${VLLM_KV_CACHE_DTYPE//[[:space:]]/}" ]]; then
        echo "ERROR: VLLM_KV_CACHE_DTYPE must not be empty when SERVER_BACKEND=vllm."
        exit 1
    fi
    if [[ -n "${VLLM_MAX_NUM_BATCHED_TOKENS:-}" ]]; then
        validate_int_ge VLLM_MAX_NUM_BATCHED_TOKENS 1
    fi
fi
if [[ -n "${SERVER_THREADS}" ]]; then
    validate_int_ge SERVER_THREADS 1
fi
if [[ -n "${SERVER_THREADS_BATCH}" ]]; then
    validate_int_ge SERVER_THREADS_BATCH 1
fi
if [[ -n "${SERVER_THREADS_HTTP}" ]]; then
    validate_int_ge SERVER_THREADS_HTTP 1
fi
if [[ -n "${SERVER_SRUN_CPUS_PER_TASK}" ]]; then
    validate_int_ge SERVER_SRUN_CPUS_PER_TASK 1
fi
validate_int_ge GPU_UTIL_POLL_SEC 1
validate_int_ge GPU_FAILFAST_WARMUP_SEC 0
validate_int_ge GPU_FAILFAST_UTIL_PCT 0
validate_int_ge GPU_FAILFAST_LOW_FRAC_PCT 0
validate_int_ge GPU_FAILFAST_MIN_SAMPLES 1
validate_int_ge GPU_FAILFAST_CHECK_SEC 1
validate_int_ge SERVER_FAILFAST_CHECK_SEC 1
validate_int_ge SERVER_FAILFAST_MAX_CONSEC_HEALTH_FAILS 1

if (( GPU_FAILFAST_UTIL_PCT > 100 )); then
    echo "ERROR: GPU_FAILFAST_UTIL_PCT must be <= 100. Got '${GPU_FAILFAST_UTIL_PCT}'."
    exit 1
fi

if (( GPU_FAILFAST_LOW_FRAC_PCT > 100 )); then
    echo "ERROR: GPU_FAILFAST_LOW_FRAC_PCT must be <= 100. Got '${GPU_FAILFAST_LOW_FRAC_PCT}'."
    exit 1
fi

if (( SERVER_USE_SRUN == 1 )); then
    if [[ -z "${SERVER_SRUN_CPU_BIND}" ]]; then
        echo "ERROR: SERVER_SRUN_CPU_BIND must not be empty when SERVER_USE_SRUN=1."
        exit 1
    fi
    if [[ -z "${SERVER_SRUN_GPU_BIND}" ]]; then
        echo "ERROR: SERVER_SRUN_GPU_BIND must not be empty when SERVER_USE_SRUN=1."
        exit 1
    fi
fi

validate_cache_type() {
    local var_name value
    var_name="$1"
    value="${!var_name}"
    if [[ -z "$value" ]]; then
        return
    fi
    case "${value,,}" in
        f32|f16|bf16|q8_0|q4_0|q4_1|iq4_nl|q5_0|q5_1)
            printf -v "$var_name" '%s' "${value,,}"
            ;;
        *)
            echo "ERROR: ${var_name} must be one of: f32,f16,bf16,q8_0,q4_0,q4_1,iq4_nl,q5_0,q5_1"
            echo "       Got: '${value}'"
            exit 1
            ;;
    esac
}

if [[ "$SERVER_BACKEND" == "llama_cpp" ]]; then
    validate_cache_type SERVER_CACHE_TYPE_K
    validate_cache_type SERVER_CACHE_TYPE_V
fi

if [[ -n "${SLURM_JOB_ID:-}" ]]; then
    JOB_ID="$SLURM_JOB_ID"
else
    JOB_ID="manual-$(date +%s)"
fi

if (( SERVER_USE_SRUN == 1 )) && [[ -z "${SLURM_JOB_ID:-}" ]]; then
    echo "WARN: SERVER_USE_SRUN=1 requested outside a Slurm allocation; disabling srun launch."
    SERVER_USE_SRUN=0
fi

if (( HAS_SERVER_PORT_OVERRIDE == 0 )); then
    if [[ "$JOB_ID" =~ ^[0-9]+$ ]]; then
        SERVER_PORT="$((18000 + (JOB_ID % 1000)))"
    else
        SERVER_PORT="$DEFAULT_SERVER_PORT"
    fi
fi

if (( HAS_EVAL_RUN_ID_OVERRIDE == 0 )); then
    EVAL_RUN_ID="${PRESET_RUN_ID_PREFIX}-${JOB_ID}"
fi

if (( SMOKE_TEST_MODE == 1 )); then
    if (( HAS_EVAL_RUN_ID_OVERRIDE == 0 )); then
        EVAL_RUN_ID="smoke-${PRESET_RUN_ID_PREFIX}-${JOB_ID}"
    fi
    if (( HAS_EVAL_MAX_CASES_OVERRIDE == 0 )); then
        EVAL_MAX_CASES="$SMOKE_TEST_MAX_CASES"
    fi
    if (( HAS_EVAL_K_OVERRIDE == 0 )); then
        EVAL_K="$SMOKE_TEST_EVAL_K"
    fi
    if (( HAS_CONTEXT_SIZE_OVERRIDE == 0 )); then
        if [[ "$SERVER_BACKEND" == "vllm" ]] && (( HAS_VLLM_MAX_MODEL_LEN_OVERRIDE == 1 )); then
            :
        else
            CONTEXT_SIZE="$SMOKE_TEST_CONTEXT_SIZE"
            HAS_CONTEXT_SIZE_OVERRIDE=1
        fi
    fi
    if (( HAS_MAX_WAIT_OVERRIDE == 0 )); then
        MAX_WAIT="$SMOKE_TEST_MAX_WAIT_SEC"
    fi
fi

if [[ "$SERVER_BACKEND" == "vllm" ]]; then
    if (( HAS_VLLM_MAX_MODEL_LEN_OVERRIDE == 1 )); then
        CONTEXT_SIZE="$VLLM_MAX_MODEL_LEN"
    elif (( HAS_CONTEXT_SIZE_OVERRIDE == 1 )); then
        VLLM_MAX_MODEL_LEN="$CONTEXT_SIZE"
    else
        CONTEXT_SIZE="$VLLM_MAX_MODEL_LEN"
    fi
    validate_int_ge VLLM_MAX_MODEL_LEN 1
    if (( SERVER_PARALLEL != 1 )); then
        echo "INFO: SERVER_PARALLEL=${SERVER_PARALLEL} is ignored for SERVER_BACKEND=vllm; forcing SERVER_PARALLEL=1 for budgeting."
    fi
    SERVER_PARALLEL=1
fi

validate_int_ge CONTEXT_SIZE 1
validate_int_ge OPENAI_MAX_TOKENS 1
validate_int_ge EVAL_MAX_CONTEXT_CHARS 1
validate_int_ge EVAL_PROMPT_BUDGET_TOTAL_CHARS 0
validate_int_ge CONTEXT_CHARS_PER_TOKEN 1
validate_int_ge CONTEXT_PROMPT_HEADROOM_PCT 1
validate_int_ge LOW_PER_SLOT_CTX_WARN 1

if (( CONTEXT_PROMPT_HEADROOM_PCT > 100 )); then
    echo "ERROR: CONTEXT_PROMPT_HEADROOM_PCT must be <= 100. Got '${CONTEXT_PROMPT_HEADROOM_PCT}'."
    exit 1
fi

if (( SERVER_PARALLEL > CONTEXT_SIZE )); then
    echo "ERROR: CONTEXT_SIZE (${CONTEXT_SIZE}) must be >= SERVER_PARALLEL (${SERVER_PARALLEL})."
    exit 1
fi

PER_SLOT_CONTEXT_TOKENS=$(( CONTEXT_SIZE / SERVER_PARALLEL ))
if (( PER_SLOT_CONTEXT_TOKENS < 1 )); then
    echo "ERROR: Computed per-slot context is < 1 token."
    echo "       CONTEXT_SIZE=${CONTEXT_SIZE}, SERVER_PARALLEL=${SERVER_PARALLEL}"
    exit 1
fi

if (( CONTEXT_SIZE % SERVER_PARALLEL != 0 )); then
    echo "WARN: CONTEXT_SIZE (${CONTEXT_SIZE}) is not divisible by SERVER_PARALLEL (${SERVER_PARALLEL})."
    echo "      Effective per-slot context uses floor division: ${PER_SLOT_CONTEXT_TOKENS} tokens."
fi

if (( AUTO_CONTEXT_BUDGETS == 1 )); then
    AUTO_MAX_CONTEXT_CHARS=$(( PER_SLOT_CONTEXT_TOKENS * CONTEXT_CHARS_PER_TOKEN ))
    AUTO_PROMPT_BUDGET_TOTAL_CHARS=$(( AUTO_MAX_CONTEXT_CHARS * CONTEXT_PROMPT_HEADROOM_PCT / 100 ))

    if (( AUTO_PROMPT_BUDGET_TOTAL_CHARS < 1 )); then
        AUTO_PROMPT_BUDGET_TOTAL_CHARS=1
    fi

    if (( HAS_EVAL_MAX_CONTEXT_CHARS_OVERRIDE == 0 && AUTO_MAX_CONTEXT_CHARS < EVAL_MAX_CONTEXT_CHARS )); then
        EVAL_MAX_CONTEXT_CHARS="$AUTO_MAX_CONTEXT_CHARS"
    fi
    if (( HAS_EVAL_PROMPT_BUDGET_TOTAL_CHARS_OVERRIDE == 0 && AUTO_PROMPT_BUDGET_TOTAL_CHARS < EVAL_PROMPT_BUDGET_TOTAL_CHARS )); then
        EVAL_PROMPT_BUDGET_TOTAL_CHARS="$AUTO_PROMPT_BUDGET_TOTAL_CHARS"
    fi
fi

if (( EVAL_PROMPT_BUDGET_TOTAL_CHARS > EVAL_MAX_CONTEXT_CHARS )); then
    EVAL_PROMPT_BUDGET_TOTAL_CHARS="$EVAL_MAX_CONTEXT_CHARS"
fi

if (( PER_SLOT_CONTEXT_TOKENS < LOW_PER_SLOT_CTX_WARN )); then
    echo "WARN: Low per-slot context (${PER_SLOT_CONTEXT_TOKENS} tokens)."
    echo "      Consider increasing CONTEXT_SIZE or reducing SERVER_PARALLEL."
fi

if [[ -n "${MODEL_CTX_TRAIN_TOKENS}" ]] && (( CONTEXT_SIZE < MODEL_CTX_TRAIN_TOKENS )); then
    echo "INFO: CONTEXT_SIZE (${CONTEXT_SIZE}) is below model train context (${MODEL_CTX_TRAIN_TOKENS})."
fi

if (( EVAL_DETERMINISTIC == 1 )) && [[ -z "$EVAL_SEED" ]]; then
    echo "ERROR: EVAL_DETERMINISTIC=1 requires EVAL_SEED to be set."
    exit 1
fi

if [[ -z "${EVAL_CONDITIONS//[[:space:]]/}" ]]; then
    echo "ERROR: EVAL_CONDITIONS must not be empty."
    exit 1
fi

if ! REPO_DIR_REAL="$(cd "$REPO_DIR" 2>/dev/null && pwd -P)"; then
    echo "ERROR: Repo not found at $REPO_DIR"
    exit 1
fi

MODEL_DIR_REAL=""
MODEL_PATH=""
if [[ "$SERVER_BACKEND" == "llama_cpp" ]]; then
    if ! MODEL_DIR_REAL="$(cd "$MODEL_DIR" 2>/dev/null && pwd -P)"; then
        echo "ERROR: Model directory not found: $MODEL_DIR"
        exit 1
    fi
    MODEL_PATH="${MODEL_DIR_REAL}/${MODEL_FILE}"
else
    MODEL_PATH="$HF_MODEL_ID"
fi

if [[ -n "${HF_CACHE_DIR:-}" ]]; then
    if ! mkdir -p "$HF_CACHE_DIR" >/dev/null 2>&1; then
        echo "ERROR: Could not create HF_CACHE_DIR at '$HF_CACHE_DIR'."
        exit 1
    fi
    if ! HF_CACHE_DIR="$(cd "$HF_CACHE_DIR" 2>/dev/null && pwd -P)"; then
        echo "ERROR: HF_CACHE_DIR does not resolve to a valid directory: '$HF_CACHE_DIR'"
        exit 1
    fi
fi
if [[ "$SERVER_BACKEND" == "vllm" ]] && [[ -n "${VLLM_DOWNLOAD_DIR:-}" ]]; then
    if ! mkdir -p "$VLLM_DOWNLOAD_DIR" >/dev/null 2>&1; then
        echo "ERROR: Could not create VLLM_DOWNLOAD_DIR at '$VLLM_DOWNLOAD_DIR'."
        exit 1
    fi
    if ! VLLM_DOWNLOAD_DIR="$(cd "$VLLM_DOWNLOAD_DIR" 2>/dev/null && pwd -P)"; then
        echo "ERROR: VLLM_DOWNLOAD_DIR does not resolve to a valid directory: '$VLLM_DOWNLOAD_DIR'"
        exit 1
    fi
fi

SERVER_LOG="${REPO_DIR_REAL}/llm-server-${JOB_ID}.log"
EVAL_LAUNCH_LOG="${REPO_DIR_REAL}/eval-launch-${JOB_ID}.log"
ENROOT_DIAG_LOG="${REPO_DIR_REAL}/enroot-diag-${JOB_ID}.log"
GPU_UTIL_LOG="${REPO_DIR_REAL}/gpu-util-${JOB_ID}.csv"
RUN_CONFIG_LOG="${REPO_DIR_REAL}/run-config-${JOB_ID}.json"
FAILURE_DUMP_LOG="${REPO_DIR_REAL}/failure-dump-${JOB_ID}.json"

MODEL_MOUNT_DIR="${MODEL_DIR_REAL}"
REPO_MOUNT_DIR="$REPO_DIR_REAL"
USING_TMPDIR_STAGING=0
MODEL_STAGED=0
REPO_STAGED=0
TMP_STAGE_ROOT=""
TMP_STAGE_MODEL_DIR=""
TMP_STAGE_REPO_DIR=""
SERVER_PID=""
SERVER_PROCESS_PATTERN=""
EVAL_LAUNCH_PID=""
GPU_MONITOR_PID=""
GPU_FAILFAST_WATCHDOG_PID=""
GPU_FAILFAST_START_LINE=1
GPU_FAILFAST_TRIGGERED=0
GPU_FAILFAST_REASON=""
GPU_FAILFAST_REASON_FILE="${REPO_DIR_REAL}/gpu-failfast-${JOB_ID}.txt"
SERVER_FAILFAST_WATCHDOG_PID=""
SERVER_FAILFAST_TRIGGERED=0
SERVER_FAILFAST_REASON=""
SERVER_FAILFAST_REASON_FILE="${REPO_DIR_REAL}/server-failfast-${JOB_ID}.txt"
JOB_EXIT_CODE=0
FAILURE_DUMP_WRITTEN=0

if [[ "$SERVER_BACKEND" == "llama_cpp" ]] && [[ ! -f "$MODEL_PATH" ]]; then
    echo "ERROR: Model not found: $MODEL_PATH"
    echo "       Preset source: ${MODEL_SOURCE}"
    echo "       Run: huggingface-cli download ${MODEL_SOURCE} --include \"${MODEL_FILE}\" ..."
    exit 1
fi

if [[ ! -d "$REPO_DIR_REAL/evals" ]]; then
    echo "ERROR: Repo not found at $REPO_DIR_REAL (missing evals/ directory)"
    exit 1
fi

for required_cmd in enroot curl stat awk; do
    if ! command_exists "$required_cmd"; then
        echo "ERROR: Required command not found: $required_cmd"
        exit 1
    fi
done

if (( SERVER_USE_SRUN == 1 )) && ! command_exists srun; then
    echo "ERROR: SERVER_USE_SRUN=1 but srun is not available."
    echo "       Either install/access Slurm client commands or set SERVER_USE_SRUN=0."
    exit 1
fi

for ctr in "$SERVER_CONTAINER" "$EVAL_CONTAINER"; do
    if ! enroot list | grep -q "^${ctr}$"; then
        echo "ERROR: Enroot container '$ctr' not found."
        echo "       Run: enroot create --name $ctr <image>.sqsh"
        exit 1
    fi
done

ENROOT_DATA_DIR="${ENROOT_DATA_PATH:-$HOME/.local/share/enroot}"
if ENROOT_DATA_DIR_REAL="$(cd "$ENROOT_DATA_DIR" 2>/dev/null && pwd -P)"; then
    ENROOT_DATA_DIR="$ENROOT_DATA_DIR_REAL"
fi

SERVER_ROOTFS="${ENROOT_DATA_DIR}/${SERVER_CONTAINER}"
EVAL_ROOTFS="${ENROOT_DATA_DIR}/${EVAL_CONTAINER}"

for rootfs in "$SERVER_ROOTFS" "$EVAL_ROOTFS"; do
    if [[ ! -d "$rootfs" ]]; then
        echo "ERROR: Enroot rootfs not found: $rootfs"
        echo "       ENROOT_DATA_PATH resolves to: $ENROOT_DATA_DIR"
        echo "       If containers were created elsewhere, export ENROOT_DATA_PATH"
        echo "       before submitting the job and recreate containers there."
        echo "       Debug: enroot list --fancy"
        exit 1
    fi
done

print_effective_config
write_run_config_snapshot

if (( DRY_RUN == 1 )); then
    echo "DRY_RUN=1 set, stopping before container launch."
    exit 0
fi

prepare_bind_mount_targets

cd "$REPO_DIR_REAL"
trap cleanup EXIT

start_gpu_monitor

# ── Step 1: Launch LLM server via Enroot ───────────────────────────────────
echo "Starting LLM server (backend=${SERVER_BACKEND})..."
log_mount_diagnostics
start_server
echo "LLM server launched (PID=$SERVER_PID), logging to $SERVER_LOG"

# ── Step 2: Wait for server to become ready ────────────────────────────────
echo "Waiting for LLM server to become ready..."
ELAPSED=0
SERVER_RETRIES=0

while ! server_healthcheck; do
    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
        if [[ "$SERVER_BACKEND" == "llama_cpp" ]] \
            && (( ENABLE_TMPDIR_FALLBACK == 1 && SERVER_RETRIES == 0 )) \
            && grep -q "enroot-mount: failed to mount: .* No such device" "$SERVER_LOG"; then
            echo "Detected enroot bind-mount failure (No such device)."
            echo "Retrying once by staging model + repo to TMPDIR..."
            stage_model_to_tmpdir
            stage_repo_to_tmpdir
            log_mount_diagnostics
            write_run_config_snapshot
            start_server
            SERVER_RETRIES=1
            ELAPSED=0
            echo "LLM server relaunched (PID=$SERVER_PID), logging to $SERVER_LOG"
            continue
        fi

        JOB_EXIT_CODE=1
        print_failure_diagnostics "LLM server process died before health endpoint became ready"
        exit 1
    fi

    if (( ELAPSED >= MAX_WAIT )); then
        JOB_EXIT_CODE=1
        print_failure_diagnostics "LLM server did not start within ${MAX_WAIT}s"
        exit 1
    fi

    sleep "$POLL_INTERVAL"
    (( ELAPSED += POLL_INTERVAL )) || true
    echo "  ... waiting (${ELAPSED}s / ${MAX_WAIT}s)"
done

echo "LLM server ready after ${ELAPSED}s"

# ── Step 3: Run the eval harness ──────────────────────────────────────────
EVAL_ARGS=(
    --executor local
    --patcher openai
    --openai-base-url "http://localhost:${SERVER_PORT}"
    --openai-model "$MODEL_ALIAS"
    --openai-api-key "local"
    --openai-timeout-sec "$OPENAI_TIMEOUT_SEC"
    --openai-max-tokens "$OPENAI_MAX_TOKENS"
    --openai-max-propose-sec "$OPENAI_MAX_PROPOSE_SEC"
    --openai-temperature "$OPENAI_TEMPERATURE"
    --run-id "$EVAL_RUN_ID"
    --k "$EVAL_K"
    --jobs "$EVAL_JOBS"
    --conditions "$EVAL_CONDITIONS"
    --max-context-chars "$EVAL_MAX_CONTEXT_CHARS"
    --prompt-budget-total-chars "$EVAL_PROMPT_BUDGET_TOTAL_CHARS"
)

if [[ -n "$EVAL_CATEGORY" ]]; then
    EVAL_ARGS+=(--category "$EVAL_CATEGORY")
fi

if (( EVAL_MAX_CASES > 0 )); then
    EVAL_ARGS+=(--max-cases "$EVAL_MAX_CASES")
fi

if (( EVAL_DETERMINISTIC == 1 )); then
    EVAL_ARGS+=(--deterministic --seed "$EVAL_SEED")
fi

echo ""
echo "Running eval: python -m evals.run_eval ${EVAL_ARGS[*]}"
echo "────────────────────────────────────────"

if run_eval_once; then
    EVAL_EXIT=0
else
    EVAL_EXIT=$?
fi

if (( SERVER_FAILFAST_TRIGGERED == 1 )) && (( EVAL_EXIT == 0 )); then
    EVAL_EXIT=71
fi

if (( GPU_FAILFAST_TRIGGERED == 1 )) && (( EVAL_EXIT == 0 )); then
    EVAL_EXIT=70
fi

if (( EVAL_EXIT != 0 )) \
    && (( GPU_FAILFAST_TRIGGERED == 0 )) \
    && (( ENABLE_TMPDIR_FALLBACK == 1 )) \
    && (( REPO_STAGED == 0 )) \
    && grep -q "enroot-mount: failed to mount: .* No such device" "$EVAL_LAUNCH_LOG"; then
    echo "Detected enroot bind-mount failure during eval mount."
    echo "Retrying once by staging repo to TMPDIR..."

    stage_repo_to_tmpdir
    log_mount_diagnostics
    write_run_config_snapshot

    if run_eval_once; then
        EVAL_EXIT=0
    else
        EVAL_EXIT=$?
    fi
fi

write_run_config_snapshot
sync_staged_results_back

if (( EVAL_EXIT != 0 )); then
    JOB_EXIT_CODE="$EVAL_EXIT"
    if (( SERVER_FAILFAST_TRIGGERED == 1 )); then
        if [[ -z "${SERVER_FAILFAST_REASON}" ]]; then
            SERVER_FAILFAST_REASON="Server fail-fast triggered: LLM server became unavailable during eval."
        fi
        print_failure_diagnostics "$SERVER_FAILFAST_REASON"
    elif (( GPU_FAILFAST_TRIGGERED == 1 )); then
        if [[ -z "${GPU_FAILFAST_REASON}" ]]; then
            GPU_FAILFAST_REASON="GPU fail-fast triggered due to sustained low utilization."
        fi
        print_failure_diagnostics "$GPU_FAILFAST_REASON"
    else
        print_failure_diagnostics "eval finished with non-zero exit code: ${EVAL_EXIT}"
    fi
fi

echo "────────────────────────────────────────"
echo "Eval finished with exit code: $EVAL_EXIT"
echo "Results: evals/results/${EVAL_RUN_ID}.jsonl"

exit "$EVAL_EXIT"
