{% set user = ssh_user if image_id is none else 'root' %}
{% set has_image = image_id is not none %}
cluster_name: {{cluster_name_on_cloud}}

# The maximum number of workers nodes to launch in addition to the head node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60

provider:
  type: external
  module: sky.provision.slurm

  cluster: {{slurm_cluster}}
  partition: {{slurm_partition}}
  provision_timeout: {{provision_timeout}}

  ssh:
    hostname: {{ssh_hostname}}
    port: {{ssh_port}}
    user: {{ssh_user}}
{% if slurm_private_key is not none %}
    private_key: {{slurm_private_key}}
{% endif %}
{% if slurm_proxy_command is not none %}
    proxycommand: {{slurm_proxy_command | tojson }}
{% endif %}
{% if slurm_proxy_jump is not none %}
    proxyjump: {{slurm_proxy_jump | tojson }}
{% endif %}
    identities_only: {{slurm_identities_only}}

auth:
  ssh_user: {{user}}
  ssh_private_key: {{ssh_private_key}}

available_node_types:
  ray_head_default:
    resources: {}
    node_config:
      # From clouds/slurm.py::Slurm.make_deploy_resources_variables.
      instance_type: {{instance_type}}
      disk_size: {{disk_size}}
      cpus: {{cpus}}
      memory: {{memory}}
      accelerator_type: {{accelerator_type}}
      accelerator_count: {{accelerator_count}}
{% if image_id is not none %}
      image_id: {{image_id}}
{% endif %}

      # TODO: more configs that is required by the provisioner to create new
      # instances on the FluffyCloud:
      # sky/provision/fluffycloud/instance.py::run_instances

head_node_type: ray_head_default

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
{%- endfor %}
}

rsync_exclude: []

initialization_commands: []

# List of shell commands to run to set up nodes.
# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
# connection, which is expensive. Try your best to co-locate commands into fewer
# items!
#
# Increment the following for catching performance bugs easier:
#   current num items (num SSH connections): 1
setup_commands:
  - |
    {%- for initial_setup_command in initial_setup_commands %}
    {{ initial_setup_command }}
    {%- endfor %}
    # SSH setup: authorized_keys, host key, and bashrc for interactive sessions.
    # - Non-container: always run (for the normal user)
    # - Container: only run inside container (when root), not on the host
    if [ {{ 'true' if not has_image else 'false' }} ] || [ "$(id -u)" = "0" ]; then
    # Generate host key for sshd -i if not exists
    mkdir -p ~{{user}}/.ssh && chmod 700 ~{{user}}/.ssh
    [ -f ~{{user}}/.ssh/{{slurm_sshd_host_key_filename}} ] || ssh-keygen -t ed25519 -f ~{{user}}/.ssh/{{slurm_sshd_host_key_filename}} -N "" -q
    # Add public key to user's authorized_keys if not already present
    grep -qF 'skypilot:ssh_public_key_content' ~{{user}}/.ssh/authorized_keys 2>/dev/null || cat >> ~{{user}}/.ssh/authorized_keys <<'SKYPILOT_SSH_KEY_EOF'
    skypilot:ssh_public_key_content
    SKYPILOT_SSH_KEY_EOF
    chmod 600 ~{{user}}/.ssh/authorized_keys

    mkdir -p ~{{user}}/.sky
    cat > ~{{user}}/.sky_ssh_rc <<'SKYPILOT_SSH_RC'
    # Added by SkyPilot: override HOME for Slurm interactive sessions
    if [ -n "${{slurm_cluster_name_env_var}}" ]; then
        CLUSTER_DIR=~/.sky_clusters/${{slurm_cluster_name_env_var}}
        if [ -d "$CLUSTER_DIR" ]; then
            cd "$CLUSTER_DIR"
            export HOME=$(pwd)
        fi
    fi
    SKYPILOT_SSH_RC
    grep -q "source ~/.sky_ssh_rc" ~{{user}}/.bashrc 2>/dev/null || (echo "" >> ~{{user}}/.bashrc && echo "source ~/.sky_ssh_rc" >> ~{{user}}/.bashrc)
    fi
    {{ setup_sky_dirs_commands }}
    {{ conda_installation_commands }}
    {{ uv_installation_commands }}
    {{ skypilot_wheel_installation_commands }}
    {{ copy_skypilot_templates_commands }}
{% if image_id is not none %}
    # Install dropbear and socat for container SSH.
    # OpenSSH fails in enroot containers created with --container-remap-root with:
    #   "permanently_set_uid: was able to restore old [e]gid [preauth]"
    # sshd verifies it can actually drop privileges, which doesn't work with
    # --container-remap-root because of the uid remapping.
    # See:
    # - https://github.com/NVIDIA/pyxis/issues/85#issuecomment-1201913582
    # - https://github.com/NVIDIA/enroot/issues/92
    #
    # So we use Dropbear (https://matt.ucc.asn.au/dropbear/dropbear.html), a
    # lightweight SSH server popular in embedded and resource-constrained
    # environments. It's actively maintained (https://github.com/mkj/dropbear/releases)
    # and doesn't have the same issue with OpenSSH.
    #
    # This section runs on both host and container; only install when root (container).
    # TODO(kevin): Host the binary somewhere, so we don't have to build it every time.
    if [ "$(id -u)" = "0" ]; then
      set -e
      DROPBEAR_VERSION=2025.89
      PACKAGES="socat iproute2 ccache"
      export CCACHE_DIR=/var/cache/ccache
      # Only Debian-based images are supported for now.
      # See: https://docs.skypilot.co/en/latest/examples/docker-containers.html
      export DEBIAN_FRONTEND=noninteractive
      # Helper: apt with retries and exponential backoff (adapted from kubernetes-ray.yml.j2)
      # TODO(kevin): Consolidate with the one in kubernetes-ray.yml.j2
      apt_install_with_retries() {
        local packages="$@"
        [ -z "$packages" ] && return 0
        set +e
        local log=/tmp/apt-install.log
        local tries=3
        local delay=5
        for i in $(seq 1 $tries); do
          apt-get update >> "$log" 2>&1
          apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $packages >> "$log" 2>&1 && { set -e; return 0; }
          echo "apt-get install failed (attempt $i/$tries). Retrying in ${delay}s..." >> "$log"
          apt-get -f install -y >> "$log" 2>&1 || true
          apt-get clean >> "$log" 2>&1 || true
          sleep $delay
          delay=$((delay * 2))
        done
        echo "apt-get install failed after $tries attempts. Log:" >&2
        cat "$log" >&2
        set -e
        return 1
      }
      if ! /usr/local/bin/dropbear -V 2>&1 | grep -q "$DROPBEAR_VERSION"; then
        cd /tmp
        apt_install_with_retries build-essential $PACKAGES
        curl -sL https://matt.ucc.asn.au/dropbear/releases/dropbear-$DROPBEAR_VERSION.tar.bz2 | tar xj
        cd dropbear-$DROPBEAR_VERSION
        ./configure --disable-zlib --disable-syslog --disable-wtmp --disable-lastlog >/dev/null
        BUILD_START=$SECONDS
        make CC="ccache gcc" -j$(nproc) PROGRAMS="dropbear" >/dev/null
        echo "[dropbear] Build took $((SECONDS - BUILD_START))s"
        cp dropbear /usr/local/bin/
        cd /; rm -rf /tmp/dropbear-$DROPBEAR_VERSION
      fi
      mkdir -p /etc/dropbear
    fi
{% endif %}

head_node: {}
worker_nodes: {}

# These fields are required for external cloud providers.
head_setup_commands: []
worker_setup_commands: []
cluster_synced_files: []
file_mounts_sync_continuously: False
