cluster_name: {{cluster_name_on_cloud}}

# The maximum number of workers nodes to launch in addition to the head node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60

provider:
  type: external
  module: sky.provision.nebius
  region: "{{region}}"
  use_internal_ips: {{use_internal_ips}}
  use_static_ip_address: {{ use_static_ip_address }}

{%- if docker_image is not none %}
docker:
  image: {{docker_image}}
  container_name: {{docker_container_name}}
  run_options:
    - --ulimit nofile=1048576:1048576
    {%- for run_option in docker_run_options %}
    - {{run_option}}
    {%- endfor %}
  {%- if docker_login_config is not none %}
  docker_login_config:
    username: |-
      {{docker_login_config.username}}
    password: |-
      {{docker_login_config.password | indent(6) }}
    server: |-
      {{docker_login_config.server}}
  {%- endif %}
{%- endif %}


auth:
  ssh_user: ubuntu
  ssh_private_key: {{ssh_private_key}}
{% if ssh_proxy_command is not none %}
  ssh_proxy_command: {{ssh_proxy_command}}
{% endif %}

available_node_types:
  ray_head_default:
    resources: {}
    node_config:
      InstanceType: {{instance_type}}
      ImageId: {{image_id}}
      DiskSize: {{disk_size}}
      use_spot: {{ use_spot }}
      network_tier: {{network_tier}}
      filesystems:
        {%- for fs in filesystems %}
        - filesystem_id: {{ fs.filesystem_id }}
          filesystem_mount_tag: {{ fs.filesystem_mount_tag }}
          filesystem_attach_mode: {{ fs.filesystem_attach_mode }}
          filesystem_mount_path: {{ fs.filesystem_mount_path }}
        {%- endfor %}
      UserData: |
        {# Two available OS images:
               1. ubuntu24.04-driverless - requires Docker installation
               2. ubuntu24.04-cuda12 - comes with Docker pre-installed
            To optimize deployment speed, Docker is only installed when using ubuntu24.04-driverless #}
        {%- if docker_image is not none and image_id.endswith('-driverless') %}
        apt:
          sources:
            docker.list:
              source: deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable
              keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88

        packages:
          - apt-transport-https
          - ca-certificates
          - curl
          - gnupg-agent
          - software-properties-common
          - docker-ce
          - docker-ce-cli
          - containerd.io

        # Enable ipv4 forwarding, required on CIS hardened machines
        write_files:
          - path: /etc/sysctl.d/enabled_ipv4_forwarding.conf
            content: |
              net.ipv4.conf.all.forwarding=1

        # create the docker group
        groups:
          - docker

        # Add default auto created user to docker group
        system_info:
          default_user:
            groups: [docker]
        {%- endif %}
        users:
          - name: skypilot:ssh_user
            shell: /bin/bash
            sudo: ALL=(ALL) NOPASSWD:ALL
            ssh_authorized_keys:
              - |-
                skypilot:ssh_public_key_content

head_node_type: ray_head_default

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
  "~/.ssh/sky-cluster-key": "{{ssh_private_key}}",
{%- endfor %}
}

rsync_exclude: []

initialization_commands: []

# List of shell commands to run to set up nodes.
# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
# connection, which is expensive. Try your best to co-locate commands into fewer
# items!
#
# Increment the following for catching performance bugs easier:
#   current num items (num SSH connections): 1
setup_commands:
  # Add ~/.ssh/sky-cluster-key to SSH config to allow nodes within a cluster to connect to each other
  # Disable `unattended-upgrades` to prevent apt-get from hanging. It should be called at the beginning before the process started to avoid being blocked. (This is a temporary fix.)
  # Create ~/.ssh/config file in case the file does not exist in the image.
  # Line 'rm ..': there is another installation of pip.
  # Line 'sudo bash ..': set the ulimit as suggested by ray docs for performance. https://docs.ray.io/en/latest/cluster/vms/user-guides/large-cluster-best-practices.html#system-configuration
  # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
  # Line 'mkdir -p ..': disable host key check
  # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
  - {%- for initial_setup_command in initial_setup_commands %}
    {{ initial_setup_command }}
    {%- endfor %}
    {%- for fs in filesystems %}
    sudo mkdir {{ fs.filesystem_mount_path }};
    sudo mount -t virtiofs {{ fs.filesystem_mount_tag }} {{ fs.filesystem_mount_path }};
    sudo chmod a+w {{ fs.filesystem_mount_path }};
    {%- endfor %}
    sudo systemctl stop unattended-upgrades || true;
    sudo systemctl disable unattended-upgrades || true;
    sudo sed -i 's/Unattended-Upgrade "1"/Unattended-Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades || true;
    sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
    sudo pkill -9 apt-get;
    sudo pkill -9 dpkg;
    sudo dpkg --configure -a;
    mkdir -p ~/.ssh; touch ~/.ssh/config;
    {{ conda_installation_commands }}
    sudo apt install -y ufw;
    sudo ufw default allow incoming;
    sudo ufw default allow outgoing;
    sudo ufw allow from 10.0.0.0/8;
    sudo ufw allow from 172.16.0.0/12;
    sudo ufw deny 6379/tcp;
    sudo ufw deny 8265/tcp;
    sudo ufw deny 10001/tcp;
    sudo ufw deny 11000:20000/tcp;
    sudo ufw deny 6379:6390/tcp;
    sudo ufw deny 8265:8270/tcp;
    sudo ufw --force enable;
    {{ uv_installation_commands }}
    {{ ray_skypilot_installation_commands }}
    {{ copy_skypilot_templates_commands }}
  {%- if env_vars is defined %}
    {%- for env_var, env_value in env_vars.items() %}
    echo '{{env_var}}={{env_value}}' | sudo tee -a /etc/environment;
    {%- endfor %}
  {%- endif %}
    IP=$(hostname -I | awk '{print $1}'); echo "$IP $(hostname)" | sudo tee -a /etc/hosts;
    sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
    mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no\n  IdentityFile ~/.ssh/sky-cluster-key\n  IdentityFile ~/.ssh/id_rsa" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n  IdentityFile ~/.ssh/sky-cluster-key\n  IdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config;
    [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf');
    {{ ssh_max_sessions_config }}
