
cluster_name: {{cluster_name}}

# The maximum number of workers nodes to launch in addition to the head node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60

provider:
  # We use a custom node provider for GCP to support instance stop and reuse.
  type: external  # type: gcp
  module: sky.skylet.providers.gcp.GCPNodeProvider
  region: {{region}}
  availability_zone: {{zones}}
  # Keep (otherwise cannot reuse when re-provisioning).
  # teardown(terminate=True) will override this.
  cache_stopped_nodes: True
  # The GCP project ID.
  project_id: {{gcp_project_id}}
{%- if tpu_vm %}
  _has_tpus: True
{%- endif %}

auth:
  ssh_user: gcpuser

available_node_types:
  ray_head_default:
    resources: {}
    node_config:
{%- if tpu_vm %}
      acceleratorType: {{tpu_type}}
      runtimeVersion: {{runtime_version}}
  {%- if use_spot %}
      schedulingConfig:
        preemptible: true
  {%- endif %}
{%- else %}
      machineType: {{instance_type}}
      disks:
        - boot: true
          autoDelete: true
          type: PERSISTENT
          initializeParams:
            diskSizeGb: {{disk_size}}
            # See https://cloud.google.com/deep-learning-vm/docs/images
            sourceImage: {{image_id}}
  {%- if gpu is not none %}
      guestAccelerators:
        - acceleratorType: projects/{{gcp_project_id}}/zones/{{zones}}/acceleratorTypes/{{gpu}}
          acceleratorCount: {{gpu_count}}
      metadata:
        items:
          - key: install-nvidia-driver
            value: "True"
  {%- endif %}
      scheduling:
  {%- if use_spot %}
        - preemptible: true
  {%- endif %}
        - onHostMaintenance: TERMINATE  # Required for GPU-attached VMs.
{%- endif %}
{% if num_nodes > 1 %}
  ray_worker_default:
    min_workers: {{num_nodes - 1}}
    max_workers: {{num_nodes - 1}}
    resources: {}
    node_config:
  {%- if tpu_vm %}
      acceleratorType: {{tpu_type}}
      runtimeVersion: {{runtime_version}}
    {%- if use_spot %}
      schedulingConfig:
        preemptible: true
    {%- endif %}
  {%- else %}
      machineType: {{instance_type}}
      disks:
        - boot: true
          autoDelete: true
          type: PERSISTENT
          initializeParams:
            diskSizeGb: {{disk_size}}
            # See https://cloud.google.com/deep-learning-vm/docs/images
            sourceImage: {{image_id}}
    {%- if gpu is not none %}
      guestAccelerators:
        - acceleratorType: projects/{{gcp_project_id}}/zones/{{zones}}/acceleratorTypes/{{gpu}}
          acceleratorCount: {{gpu_count}}
      metadata:
        items:
          - key: install-nvidia-driver
            value: "True"
    {%- endif %}
      scheduling:
    {%- if use_spot %}
        - preemptible: true
    {%- endif %}
        - onHostMaintenance: TERMINATE  # Required for GPU-attached VMs.
  {%- endif %}
{%- endif %}

head_node_type: ray_head_default

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}": "{{sky_local_path}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
{%- endfor %}
}

rsync_exclude: []

initialization_commands: []

# List of shell commands to run to set up nodes.
setup_commands:
  # Create ~/.ssh/config file in case the file does not exist in the custom image.
  # Make sure python3 & pip3 are available on this image.
  - mkdir -p ~/.ssh; touch ~/.ssh/config;
    pip3 --version > /dev/null 2>&1 || (curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py && echo "PATH=$HOME/.local/bin:$PATH" >> ~/.bashrc);
    (type -a python | grep -q python3) || echo 'alias python=python3' >> ~/.bashrc;
    (type -a pip | grep -q pip3) || echo 'alias pip=pip3' >> ~/.bashrc;
  # GCP TPU VM image does not install conda by default.
  - which conda > /dev/null 2>&1 || (wget -nc https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b && eval "$(/home/gcpuser/miniconda3/bin/conda shell.bash hook)" && conda init && conda config --set auto_activate_base true);
  # patch the buggy ray files and enable `-o allow_other` option for `goofys`
  - (pip3 list | grep ray | grep {{ray_version}} 2>&1 > /dev/null || pip3 install -U ray[default]=={{ray_version}}) && mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app;
    pip3 uninstall skypilot -y &> /dev/null;
    pip3 install "$(echo {{sky_remote_path}}/skypilot-{{sky_version}}*.whl)[gcp]";
    python3 -c "from sky.skylet.ray_patches import patch; patch()";
    [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf'); # This is needed for `-o allow_other` option for `gcsfuse`;
  # For TPU VM
  {%- if tpu_vm %}
  - pip3 install --upgrade google-api-python-client
  {%- endif %}

# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
  # Set the ulimit as suggested by ray docs for performance. https://docs.ray.io/en/latest/cluster/guide.html?highlight=ulimit#system-configuration
  # Solution from https://discuss.ray.io/t/setting-ulimits-on-ec2-instances/590
  # This line is intentionally separated from the next line to reload the session after the ulimit is set.
  - sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 65535" >> /etc/security/limits.conf; echo "* hard nofile 65535" >> /etc/security/limits.conf;';
    (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
  - (ps aux | grep "-m sky.skylet.skylet" | grep -q python3) || nohup python3 -m sky.skylet.skylet >> ~/.sky/skylet.log 2>&1 & # Start skylet daemon. (Should not place it in the head_setup_commands, otherwise it will run before sky is installed.)
  - SKY_NUM_GPUS=0 && which nvidia-smi > /dev/null && SKY_NUM_GPUS=$(nvidia-smi --query-gpu=index,name --format=csv,noheader | wc -l);
    grep "export SKY_NUM_GPUS" ~/.bashrc > /dev/null || echo "export SKY_NUM_GPUS=$SKY_NUM_GPUS" >> ~/.bashrc
  - ray stop; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --num-gpus=$SKY_NUM_GPUS

{%- if num_nodes > 1 %}
worker_start_ray_commands:
  - sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 65535" >> /etc/security/limits.conf; echo "* hard nofile 65535" >> /etc/security/limits.conf;';
    (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
  - SKY_NUM_GPUS=0 && which nvidia-smi > /dev/null && SKY_NUM_GPUS=$(nvidia-smi --query-gpu=index,name --format=csv,noheader | wc -l);
    grep "export SKY_NUM_GPUS" ~/.bashrc > /dev/null || echo "export SKY_NUM_GPUS=$SKY_NUM_GPUS" >> ~/.bashrc
  - ray stop; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --num-gpus=$SKY_NUM_GPUS
{%- else %}
worker_start_ray_commands: []
{%- endif %}

head_node: {}
worker_nodes: {}

# These fields are required for external cloud providers.
head_setup_commands: []
worker_setup_commands: []
cluster_synced_files: []
file_mounts_sync_continuously: False
