cluster_name: {{cluster_name}}

# The maximum number of workers nodes to launch in addition to the head node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60

provider:
    type: external
    module: sky.skylet.providers.azure.AzureNodeProvider
    location: {{region}}
    # Ref: https://github.com/ray-project/ray/blob/2367a2cb9033913b68b1230316496ae273c25b54/python/ray/autoscaler/_private/_azure/node_provider.py#L87
    # For Azure, ray distinguishes different instances by the resource_group,
    # instead of the cluster_name. This ensures that ray creates new instances
    # for different cluster_name.
    resource_group: {{resource_group}}
    # Keep (otherwise cannot reuse when re-provisioning).
    # teardown(terminate=True) will override this.
    cache_stopped_nodes: True
    # subscription id of the azure user
    subscription_id: {{azure_subscription_id}}

auth:
    ssh_user: azureuser

available_node_types:
    ray.head.default:
      resources: {}
      node_config:
        azure_arm_parameters:
            vmSize: {{instance_type}}
            # List images https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage
            imagePublisher: {{image_publisher}}
            imageOffer: {{image_offer}}
            imageSku: "{{image_sku}}"
            imageVersion: {{image_version}}
            osDiskSizeGB: {{disk_size}}
            # optionally set priority to use Spot instances
            {%- if use_spot %}
            priority: Spot
            # set a maximum price for spot instances if desired
            # billingProfile:
            #     maxPrice: -1
            {%- endif %}
        # TODO: attach disk
{% if num_nodes > 1 %}
    ray.worker.default:
      min_workers: {{num_nodes - 1}}
      max_workers: {{num_nodes - 1}}
      resources: {}
      node_config:
        azure_arm_parameters:
            vmSize: {{instance_type}}
            # List images https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage
            imagePublisher: {{image_publisher}}
            imageOffer: {{image_offer}}
            imageSku: "{{image_sku}}"
            imageVersion: {{image_version}}
            osDiskSizeGB: {{disk_size}}
          {%- if use_spot %}
            priority: Spot
            # set a maximum price for spot instances if desired
            # billingProfile:
            #     maxPrice: -1
          {%- endif %}
{%- endif %}

head_node_type: ray.head.default

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}": "{{sky_local_path}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
{%- endfor %}
}

rsync_exclude: []

initialization_commands: []

# List of shell commands to run to set up nodes.
setup_commands:
  # Create ~/.ssh/config file in case the file does not exist in the image.
  # Make sure python3 & pip3 are available on this image.
  # This also kills the service that is holding the lock on dpkg (problem only exists on aws/azure, not gcp)
  - mkdir -p ~/.ssh; touch ~/.ssh/config;
    pip3 --version > /dev/null 2>&1 || (curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py && echo "PATH=$HOME/.local/bin:$PATH" >> ~/.bashrc);
    (type -a python | grep -q python3) || echo 'alias python=python3' >> ~/.bashrc;
  - which conda > /dev/null 2>&1 || (wget -nc https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b && eval "$(/home/azureuser/miniconda3/bin/conda shell.bash hook)" && conda init && conda config --set auto_activate_base true);
  # patch the buggy ray files and enable `-o allow_other` option for `goofys`
  # This also kills the service that is holding the lock on dpkg (problem only exists on aws/azure, not gcp)
  - (type -a pip | grep -q pip3) || echo 'alias pip=pip3' >> ~/.bashrc;
    (pip3 list | grep ray | grep {{ray_version}} 2>&1 > /dev/null || pip3 install -U ray[default]=={{ray_version}}) && mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app && touch ~/.sudo_as_admin_successful;
    pip3 uninstall skypilot -y &> /dev/null;
    pip3 install "$(echo {{sky_remote_path}}/skypilot-{{sky_version}}*.whl)[azure]";
    python3 -c "from sky.skylet.ray_patches import patch; patch()";
    sudo systemctl stop unattended-upgrades;
    sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
    sudo pkill -9 apt-get;
    sudo pkill -9 dpkg;
    sudo dpkg --configure -a;
    sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf;

# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
  # Set the ulimit as suggested by ray docs for performance. https://docs.ray.io/en/latest/cluster/guide.html?highlight=ulimit#system-configuration
  # Solution from https://discuss.ray.io/t/setting-ulimits-on-ec2-instances/590
  # This line is intentionally separated from the next line to reload the session after the ulimit is set.
  - sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 65535" >> /etc/security/limits.conf; echo "* hard nofile 65535" >> /etc/security/limits.conf;';
    (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
  - (ps aux | grep "-m sky.skylet.skylet" | grep -q python3) || nohup python3 -m sky.skylet.skylet >> ~/.sky/skylet.log 2>&1 & # Start skylet daemon. (Should not place it in the head_setup_commands, otherwise it will run before sky is installed.)
  - ray stop; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}}

{%- if num_nodes > 1 %}
worker_start_ray_commands:
  - sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 65535" >> /etc/security/limits.conf; echo "* hard nofile 65535" >> /etc/security/limits.conf;';
    (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
  - ray stop; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}}
{%- else %}
worker_start_ray_commands: []
{%- endif %}

head_node: {}
worker_nodes: {}

# These fields are required for external cloud providers.
head_setup_commands: []
worker_setup_commands: []
cluster_synced_files: []
file_mounts_sync_continuously: False
