# Ray cluster config template for Verda Cloud
cluster_name: {{cluster_name_on_cloud}}

# The maximum number of workers nodes to launch in addition to the head node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60

provider:
  type: external
  module: sky.provision.verda
  region: "{{region}}"
  # Disable Ray's autoscaler config check, similar to Mithril.
  disable_launch_config_check: true

auth:
  ssh_user: root
  ssh_private_key: {{ssh_private_key}}

{%- if docker_image is not none %}
docker:
  image: {{docker_image}}
  container_name: {{docker_container_name}}
  run_options:
    - --ulimit nofile=1048576:1048576
    {%- for run_option in docker_run_options %}
    - {{run_option}}
    {%- endfor %}
  {%- if docker_login_config is not none %}
  docker_login_config:
    username: |-
      {{docker_login_config.username}}
    password: |-
      {{docker_login_config.password | indent(6) }}
    server: |-
      {{docker_login_config.server}}
  {%- endif %}
{%- endif %}

available_node_types:
  ray_head_default:
    resources: {}
    node_config:
      InstanceType: {{instance_type}}
      DiskSize: {{disk_size}}
      ImageId: {{image_id}}
      Preemptible: {{use_spot}}
      PublicKey: |-
        skypilot:ssh_public_key_content
      labels: {{labels}}

head_node_type: ray_head_default

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
  "~/.ssh/sky-cluster-key": "{{ssh_private_key}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
{%- endfor %}
}

rsync_exclude: []

initialization_commands: []

# List of shell commands to run to set up nodes.
# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
# connection, which is expensive. Try your best to co-locate commands into fewer
# items!
#
# Increment the following for catching performance bugs easier:
#   current num items (num SSH connections): 1
setup_commands:
  # Disable unattended-upgrades and handle apt-get locks
  # Install patch utility for Ray
  # Install conda and Ray
  # Set system limits for Ray performance (nofile and TasksMax)
  - {%- for initial_setup_command in initial_setup_commands %}
    {{ initial_setup_command }}
    {%- endfor %}
    systemctl stop unattended-upgrades || true;
    systemctl disable unattended-upgrades || true;
    sed -i 's/Unattended-Upgrade "1"/Unattended-Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades || true;
    LOCK_PID=$(lsof /var/lib/dpkg/lock-frontend 2>/dev/null | awk 'NR>1{print $2}' | tail -n 1); [ -n "$LOCK_PID" ] && kill -9 "$LOCK_PID" || true;
    pkill -9 apt-get || true;
    pkill -9 dpkg || true;
    dpkg --configure -a;
    which patch > /dev/null || apt install -y patch;
    [ -f /etc/fuse.conf ] && sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sh -c 'echo "user_allow_other" > /etc/fuse.conf');
    mkdir -p ~/.ssh; touch ~/.ssh/config; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no\n  IdentityFile ~/.ssh/id_ed25519\n  IdentityFile ~/.ssh/id_ecdsa\n  IdentityFile ~/.ssh/id_rsa\n  IdentityFile ~/.ssh/sky-cluster-key" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n  IdentityFile ~/.ssh/id_ed25519\n  IdentityFile ~/.ssh/id_ecdsa\n  IdentityFile ~/.ssh/id_rsa\n  IdentityFile ~/.ssh/sky-cluster-key\n" >> ~/.ssh/config;
    {{ conda_installation_commands }}
    {{ uv_installation_commands }}
    {{ ray_skypilot_installation_commands }}
    {{ copy_skypilot_templates_commands }}
    bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
    grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; systemctl daemon-reload;
    {{ ssh_max_sessions_config }}
