cluster_name: {{cluster_name_on_cloud}}

# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: {{num_nodes - 1}}
upscaling_speed: {{num_nodes - 1}}
idle_timeout_minutes: 60


# Kubernetes resources that need to be configured for the autoscaler to be
# able to manage the Ray cluster. If any of the provided resources don't
# exist, the autoscaler will attempt to create them. If this fails, you may
# not have the required permissions and will have to request them to be
# created by your cluster administrator.
provider:
    type: external
    module: sky.provision.kubernetes

    region: kubernetes


    namespace: {{k8s_namespace}}

    # The kubecontext used to connect to the Kubernetes cluster.
    {% if k8s_context is not none %}
    context: {{k8s_context}}
    {% endif %}

    # This should be one of KubernetesPortMode
    port_mode: {{k8s_port_mode}}

    # The networking mode used to ssh to pods. One of KubernetesNetworkingMode.
    networking_mode: {{k8s_networking_mode}}

    # We use internal IPs since we set up a port-forward between the kubernetes
    # cluster and the local machine, or directly use NodePort to reach the
    # head node.
    use_internal_ips: true

    timeout: {{timeout}}

    ssh_jump_image: {{k8s_ssh_jump_image}}

    # Namespace used to host SkyPilot system components, such as fuse device
    # manager.
    skypilot_system_namespace: {{k8s_skypilot_system_namespace}}

    # Boolean flag to indicate if the cluster requires FUSE mounting.
    # Used to set up the necessary permissions and sidecars.
    fuse_device_required: {{k8s_fuse_device_required}}

    # ServiceAccount created by the autoscaler for the head node pod that it
    # runs in. If this field isn't provided, the head pod config below must
    # contain a user-created service account with the proper permissions.
    autoscaler_service_account:
        apiVersion: v1
        kind: ServiceAccount
        metadata:
            labels:
                parent: skypilot
            name: skypilot-service-account

    # Role created by the autoscaler for the head node pod that it runs in.
    # If this field isn't provided, the role referenced in
    # autoscaler_role_binding must exist and have at least these permissions.
    autoscaler_role:
        kind: Role
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            labels:
                parent: skypilot
            name: skypilot-service-account-role
        # TODO(romilb): This is a very permissive role - gives all access in the
        #  namespace. We should restrict this. For reference, this is required
        #  for autodown and creating more SkyPilot clusters from within the pod.
        rules:
        - apiGroups: ["*"]
          resources: ["*"]
          verbs: ["*"]

    # RoleBinding created by the autoscaler for the head node pod that it runs
    # in. If this field isn't provided, the head pod config below must contain
    # a user-created service account with the proper permissions.
    autoscaler_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: RoleBinding
        metadata:
            labels:
                parent: skypilot
            name: skypilot-service-account-role-binding
        subjects:
        - kind: ServiceAccount
          name: skypilot-service-account
        roleRef:
            kind: Role
            name: skypilot-service-account-role
            apiGroup: rbac.authorization.k8s.io

    # Role for the skypilot-system namespace to create FUSE device manager and
    # any other system components.
    autoscaler_skypilot_system_role:
        kind: Role
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            namespace: {{k8s_skypilot_system_namespace}}
            labels:
                parent: skypilot
            name: skypilot-system-service-account-role
        rules:
        - apiGroups: ["*"]
          resources: ["*"]
          verbs: ["*"]

    # RoleBinding for skypilot-system namespace role
    autoscaler_skypilot_system_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: RoleBinding
        metadata:
            namespace: {{k8s_skypilot_system_namespace}}
            labels:
                parent: skypilot
            name: skypilot-system-service-account-role-binding
        subjects:
        - kind: ServiceAccount
          name: skypilot-service-account
        roleRef:
            kind: Role
            name: skypilot-system-service-account-role
            apiGroup: rbac.authorization.k8s.io

    # Role to access ingress services for fetching IP
    autoscaler_ingress_role:
        kind: Role
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            namespace: ingress-nginx
            name: skypilot-service-account-ingress-role
            labels:
                parent: skypilot
        rules:
            - apiGroups: [ "" ]
              resources: [ "services" ]
              verbs: [ "list", "get", "watch" ]
            - apiGroups: [ "rbac.authorization.k8s.io" ]
              resources: [ "roles", "rolebindings" ]
              verbs: [ "get", "list", "watch", "patch" ]

    # RoleBinding to access ingress services for fetching IP
    autoscaler_ingress_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: RoleBinding
        metadata:
            namespace: ingress-nginx
            name: skypilot-service-account-ingress-role-binding
            labels:
                parent: skypilot
        subjects:
            - kind: ServiceAccount
              name: skypilot-service-account
        roleRef:
            kind: Role
            name: skypilot-service-account-ingress-role
            apiGroup: rbac.authorization.k8s.io

    # In addition to a role binding, we also need a cluster role binding to give
    # the SkyPilot access to the cluster-wide resources such as nodes to get
    # node resources.
    autoscaler_cluster_role:
        kind: ClusterRole
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            labels:
                parent: skypilot
            name: skypilot-service-account-cluster-role
        rules:
        - apiGroups: [ "" ]
          resources: [ "nodes" ]  # Required for getting node resources.
          verbs: [ "get", "list", "watch" ]
        - apiGroups: [ "" ]
          resources: [ "namespaces" ]  # Required for creating skypilot-system namespace, which hosts fuse device manager.
          verbs: [ "get", "list", "watch", "create" ]
        - apiGroups: [ "rbac.authorization.k8s.io" ]
          resources: [ "clusterroles", "clusterrolebindings" ]  # Required for launching more SkyPilot clusters from within the pod.
          verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ]
        - apiGroups: [ "node.k8s.io" ]
          resources: [ "runtimeclasses" ]   # Required for autodetecting the runtime class of the nodes.
          verbs: [ "get", "list", "watch" ]
        - apiGroups: [ "networking.k8s.io" ]   # Required for exposing services.
          resources: [ "ingressclasses" ]
          verbs: [ "get", "list", "watch" ]

    # Bind cluster role to the service account
    autoscaler_cluster_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: ClusterRoleBinding
        metadata:
            labels:
                parent: skypilot
            name: skypilot-service-account-cluster-role-binding
        subjects:
        - kind: ServiceAccount
          name: skypilot-service-account
        roleRef:
            kind: ClusterRole
            name: skypilot-service-account-cluster-role
            apiGroup: rbac.authorization.k8s.io

    services:
      # Service to expose the head node pod's SSH port.
      - apiVersion: v1
        kind: Service
        metadata:
          labels:
            parent: skypilot
            skypilot-cluster: {{cluster_name_on_cloud}}
            skypilot-user: {{ user }}
          name: {{cluster_name_on_cloud}}-head-ssh
        spec:
          selector:
            component: {{cluster_name_on_cloud}}-head
          ports:
            - protocol: TCP
              port: 22
              targetPort: 22
      # Service that maps to the head node of the Ray cluster, so that the
      # worker nodes can find the head node using
      # {{cluster_name_on_cloud}}-head.{{k8s_namespace}}.svc.cluster.local
      - apiVersion: v1
        kind: Service
        metadata:
            labels:
              parent: skypilot
              skypilot-cluster: {{cluster_name_on_cloud}}
              skypilot-user: {{ user }}
            # NOTE: If you're running multiple Ray clusters with services
            # on one Kubernetes cluster, they must have unique service
            # names.
            name: {{cluster_name_on_cloud}}-head
        spec:
            # Create a headless service so that the head node can be reached by
            # the worker nodes with any port number.
            clusterIP: None
            # This selector must match the head node pod's selector below.
            selector:
                component: {{cluster_name_on_cloud}}-head

# Specify the pod type for the ray head node (as configured below).
head_node_type: ray_head_default
# Specify the allowed pod types for this ray cluster and the resources they provide.
available_node_types:
  ray_head_default:
    node_config:
      apiVersion: v1
      kind: Pod
      metadata:
        # name will be filled in the provisioner
        # head node name will be {{cluster_name_on_cloud}}-head, which will match the head node service selector above if a head node
        # service is required.
        labels:
            parent: skypilot
            # component will be set for the head node pod to be the same as the head node service selector above if a 
            skypilot-cluster: {{cluster_name_on_cloud}}
            # Identifies the SSH jump pod used by this pod. Used in life cycle management of the ssh jump pod.
            skypilot-ssh-jump: {{k8s_ssh_jump_name}}
            skypilot-user: {{ user }}
            # Custom tags for the pods
            {%- for label_key, label_value in labels.items() %}
            {{ label_key }}: {{ label_value|tojson }}
            {%- endfor %}
        {% if k8s_fuse_device_required %}
        annotations:
            # Required for FUSE mounting to access /dev/fuse
            container.apparmor.security.beta.kubernetes.io/ray-node: unconfined
        {% endif %}
      spec:
        # serviceAccountName: skypilot-service-account
        serviceAccountName: {{k8s_service_account_name}}
        automountServiceAccountToken: {{k8s_automount_sa_token}}
        restartPolicy: Never

        # Add node selector if GPU/TPUs are requested:
        {% if (k8s_acc_label_key is not none and k8s_acc_label_value is not none) or (k8s_spot_label_key is not none) %}
        nodeSelector:
            {% if k8s_acc_label_key is not none and k8s_acc_label_value is not none %}
            {{k8s_acc_label_key}}: {{k8s_acc_label_value}}
            {% endif %}
            {% if k8s_topology_label_key is not none and k8s_topology_label_value is not none %}
            {{k8s_topology_label_key}}: {{k8s_topology_label_value}}
            {% endif %}
            {% if k8s_spot_label_key is not none %}
            {{k8s_spot_label_key}}: {{k8s_spot_label_value|tojson}}
            {% endif %}
        {% endif %}

        {% if k8s_spot_label_key is not none %}
        tolerations:
          - key: {{k8s_spot_label_key}}
            operator: Equal
            value: {{k8s_spot_label_value|tojson}}
            effect: NoSchedule
        {% endif %}

        # This volume allocates shared memory for Ray to use for its plasma
        # object store. If you do not provide this, Ray will fall back to
        # /tmp which cause slowdowns if is not a shared memory volume.
        volumes:
        - name: secret-volume
          secret:
            secretName: {{k8s_ssh_key_secret_name}}
        - name: dshm
          emptyDir:
            medium: Memory
        - name: dev-fuse    # Required for fuse mounting
          hostPath:
            path: /dev/fuse
        containers:
        - name: ray-node
          imagePullPolicy: IfNotPresent
          image: {{image_id}}
          env:
            - name: SKYPILOT_POD_NODE_TYPE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.labels['ray-node-type']
            {% for key, value in k8s_env_vars.items() if k8s_env_vars is not none %}
            - name: {{ key }}
              value: {{ value }}
            {% endfor %}
          # Do not change this command - it keeps the pod alive until it is
          # explicitly killed.
          command: ["/bin/bash", "-c", "--"]
          args: 
            - |
              # For backwards compatibility, we put a marker file in the pod
              # to indicate that the pod is running with the changes introduced 
              # in project nimbus: https://github.com/skypilot-org/skypilot/pull/4393
              # TODO: Remove this marker file and it's usage in setup_commands
              # after v0.10.0 release.
              touch /tmp/skypilot_is_nimbus
              
              # Helper function to conditionally use sudo
              # TODO(zhwu): consolidate the two prefix_cmd and sudo replacements
              prefix_cmd() { if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; }
              [ $(id -u) -eq 0 ] && function sudo() { "$@"; } || true;

              STEPS=("apt-ssh-setup" "runtime-setup" "env-setup")

              # STEP 1: Run apt update, install missing packages, and set up ssh.
              (
                (
                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update > /tmp/apt-update.log 2>&1 || \
                echo "Warning: apt-get update failed. Continuing anyway..." >> /tmp/apt-update.log
                PACKAGES="rsync curl netcat gcc patch pciutils fuse openssh-server";

                # Separate packages into two groups: packages that are installed first
                # so that curl and rsync are available sooner to unblock the following
                # conda installation and rsync.
                set -e
                INSTALL_FIRST="";
                MISSING_PACKAGES="";
                for pkg in $PACKAGES; do
                  if [ "$pkg" == "netcat" ]; then
                    if ! dpkg -l | grep -q "^ii  \(netcat\|netcat-openbsd\|netcat-traditional\) "; then
                      INSTALL_FIRST="$INSTALL_FIRST netcat-openbsd";
                    fi
                  elif ! dpkg -l | grep -q "^ii  $pkg "; then
                    if [ "$pkg" == "curl" ] || [ "$pkg" == "rsync" ]; then
                      INSTALL_FIRST="$INSTALL_FIRST $pkg";
                    else
                      MISSING_PACKAGES="$MISSING_PACKAGES $pkg";
                    fi
                  fi
                done;
                if [ ! -z "$INSTALL_FIRST" ]; then
                  echo "Installing core packages: $INSTALL_FIRST";
                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $INSTALL_FIRST;
                fi;
                # SSH and other packages are not necessary, so we disable set -e
                set +e
                
                if [ ! -z "$MISSING_PACKAGES" ]; then
                  echo "Installing missing packages: $MISSING_PACKAGES";
                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES;
                fi;
                $(prefix_cmd) mkdir -p /var/run/sshd;
                $(prefix_cmd) sed -i "s/PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config;
                $(prefix_cmd) sed "s@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g" -i /etc/pam.d/sshd;
                cd /etc/ssh/ && $(prefix_cmd) ssh-keygen -A;
                $(prefix_cmd) mkdir -p ~/.ssh;
                $(prefix_cmd) chown -R $(whoami) ~/.ssh;
                $(prefix_cmd) chmod 700 ~/.ssh;
                $(prefix_cmd) cat /etc/secret-volume/ssh-publickey* > ~/.ssh/authorized_keys;
                $(prefix_cmd) chmod 644 ~/.ssh/authorized_keys;
                $(prefix_cmd) service ssh restart;
                $(prefix_cmd) sed -i "s/mesg n/tty -s \&\& mesg n/" ~/.profile;
                ) > /tmp/${STEPS[0]}.log 2>&1 || {
                  echo "Error: ${STEPS[0]} failed. Continuing anyway..." > /tmp/${STEPS[0]}.failed
                  cat /tmp/${STEPS[0]}.log
                  exit 1
                }
              ) &

              # STEP 2: Install conda, ray and skypilot (for dependencies); start
              # ray cluster.
              (
                (
                set -e
                mkdir -p ~/.sky
                # Wait for `curl` package to be installed before installing conda
                # and ray.
                until dpkg -l | grep -q "^ii  curl "; do
                  sleep 0.1
                  echo "Waiting for curl package to be installed..."
                done
                {{ conda_installation_commands }}
                {{ ray_installation_commands }}
                VIRTUAL_ENV=~/skypilot-runtime ~/.local/bin/uv pip install skypilot[kubernetes,remote]
                touch /tmp/ray_skypilot_installation_complete
                echo "=== Ray and skypilot installation completed ==="

                # Disable set -e, as we have some commands that are ok to fail
                # after the ray start.
                # TODO(zhwu): this is a hack, we should fix the commands that are
                # ok to fail.
                if [ "$SKYPILOT_POD_NODE_TYPE" == "head" ]; then
                  set +e
                  {{ ray_head_start_command }}
                else
                  # Start ray worker on the worker pod.
                  # Wait until the head pod is available with an IP address
                  export SKYPILOT_RAY_HEAD_IP="{{cluster_name_on_cloud}}-head.{{k8s_namespace}}.svc.cluster.local"
                  export SKYPILOT_RAY_PORT={{skypilot_ray_port}}
                  # Wait until the ray cluster is started on the head pod
                  until dpkg -l | grep -q "^ii  \(netcat\|netcat-openbsd\|netcat-traditional\) "; do
                    sleep 0.1
                    echo "Waiting for netcat package to be installed..."
                  done
                  until nc -z -w 1 ${SKYPILOT_RAY_HEAD_IP} ${SKYPILOT_RAY_PORT}; do
                    sleep 0.1
                  done

                  set +e
                  {{ ray_worker_start_command }}
                fi
                ) > /tmp/${STEPS[1]}.log 2>&1 || {
                  echo "Error: ${STEPS[1]} failed. Continuing anyway..." > /tmp/${STEPS[1]}.failed
                  cat /tmp/${STEPS[1]}.log
                  exit 1
                }
              ) &


              # STEP 3: Set up environment variables; this should be relatively fast.
              (
                (
                set -e
                if [ $(id -u) -eq 0 ]; then
                  echo 'alias sudo=""' >> ~/.bashrc; echo succeed;
                else
                  if command -v sudo >/dev/null 2>&1; then
                    timeout 2 sudo -l >/dev/null 2>&1 && echo succeed || { echo 52; exit 52; };
                  else
                    { echo 52; exit 52; };
                  fi;
                fi;
                printenv | while IFS='=' read -r key value; do echo "export $key=\"$value\""; done > ~/container_env_var.sh && $(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh
                ) > /tmp/${STEPS[2]}.log 2>&1 || {
                  echo "Error: ${STEPS[2]} failed. Continuing anyway..." > /tmp/${STEPS[2]}.failed
                  cat /tmp/${STEPS[2]}.log
                  exit 1
                }
              ) &

              function mylsof { p=$(for pid in /proc/{0..9}*; do i=$(basename "$pid"); for file in "$pid"/fd/*; do link=$(readlink -e "$file"); if [ "$link" = "$1" ]; then echo "$i"; fi; done; done); echo "$p"; };

              # Tails file and checks every 5 sec for
              # open file handlers with write access
              # closes if none exist
              monitor_file() {
                tail -f $file &
                TAIL_PID=$!
                while kill -0 $TAIL_PID 2> /dev/null; do
                  # only two PIDs should be accessing the file
                  # the log appender and log tailer
                  if [ $(mylsof $file | wc -l) -lt 2 ]; then
                    kill $TAIL_PID
                    break
                  fi
                  # Sleep for 5 seconds before checking again. Do not make this
                  # too short as it will consume CPU, and too long will cause
                  # the file to be closed too late keeping the pod alive.
                  sleep 5
                done
              }

              log_tail() {
                FILE_PATTERN="~/sky_logs/*/tasks/*.log"
                while ! ls $(eval echo $FILE_PATTERN) 1> /dev/null 2>&1; do
                  sleep 1
                done

                # Keep track of already monitored files
                already_monitored=""

                # Infinite loop to continuously check for new files
                while true; do
                  for file in $(eval echo $FILE_PATTERN); do
                    if echo $already_monitored | grep -q $file; then
                      # File is already being monitored
                      continue
                    fi
                    
                    # Monitor the new file
                    monitor_file $file &
                    already_monitored="${already_monitored} ${file}"
                  done
                  sleep 0.1
                done
              }
              trap : TERM INT; log_tail || sleep infinity & wait

          ports:
          - containerPort: 22  # Used for SSH
          - containerPort: {{ray_port}}  # Redis port
          - containerPort: 10001  # Used by Ray Client
          - containerPort: {{ray_dashboard_port}}  # Used by Ray Dashboard

          # This volume allocates shared memory for Ray to use for its plasma
          # object store. If you do not provide this, Ray will fall back to
          # /tmp which cause slowdowns if is not a shared memory volume.
          volumeMounts:
          - name: secret-volume
            readOnly: true
            mountPath: "/etc/secret-volume"
          # This volume allocates shared memory for Ray to use for its plasma
          # object store. If you do not provide this, Ray will fall back to
          # /tmp which cause slowdowns if is not a shared memory volume.
          - mountPath: /dev/shm
            name: dshm
          {% if k8s_fuse_device_required %}
          securityContext:
            capabilities:
              add:
                - "SYS_ADMIN"
          {% endif %}
          resources:
            requests:
              cpu: {{cpus}}
              memory: {{memory}}G
              {% if k8s_resource_key is not none %}
              # Number of requested google.com/tpu must be equal to the total
              # number of available TPU chips on the TPU slice node either it
              # being a node from multi-host TPU slice or single-host TPU
              # slice. Example reference:
              # https://cloud.google.com/kubernetes-engine/docs/concepts/tpus#how_tpus_work
              {{k8s_resource_key}}: {{accelerator_count}}
              {% endif %}
              {% if k8s_fuse_device_required %}
              # Kubernetes resource exposed by the fuse device manager
              # https://gitlab.com/arm-research/smarter/smarter-device-manager
              smarter-devices/fuse: "1"
              {% endif %}
            {% if k8s_resource_key is not none or k8s_fuse_device_required %}
            limits:
              # Limits need to be defined for GPU/TPU requests
              {% if k8s_resource_key is not none %}
              {{k8s_resource_key}}: {{accelerator_count}}
              {% endif %}
              {% if k8s_fuse_device_required %}
              smarter-devices/fuse: "1"
              {% endif %}
            {% endif %}
            
setup_commands:
  # Disable `unattended-upgrades` to prevent apt-get from hanging. It should be called at the beginning before the process started to avoid being blocked. (This is a temporary fix.)
  # Create ~/.ssh/config file in case the file does not exist in the image.
  # Line 'sudo bash ..': set the ulimit as suggested by ray docs for performance. https://docs.ray.io/en/latest/cluster/vms/user-guides/large-cluster-best-practices.html#system-configuration
  # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
  # Line 'mkdir -p ..': disable host key check
  # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
  # Line 'for step in ..': check if any failure indicator exists for the setup done in pod args and print the error message. This is only a best effort, as the
  # commands in pod args are asynchronous and we cannot guarantee the failure indicators are created before the setup commands finish.
  - |
    mkdir -p ~/.ssh; touch ~/.ssh/config;
    {%- for initial_setup_command in initial_setup_commands %}
    {{ initial_setup_command }}
    {%- endfor %}
    STEPS=("apt-ssh-setup" "runtime-setup" "env-setup")
    start_epoch=$(date +%s);
    echo "=== Logs for asynchronous ray and skypilot installation ===";
    if [ -f /tmp/skypilot_is_nimbus ]; then
      echo "=== Logs for asynchronous ray and skypilot installation ===";
      [ -f /tmp/ray_skypilot_installation_complete ] && cat /tmp/${STEPS[1]}.log || 
      { tail -f -n +1 /tmp/${STEPS[1]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/ray_skypilot_installation_complete ]; do sleep 0.5; done; kill $TAIL_PID || true; };
      [ -f /tmp/${STEPS[1]}.failed ] && { echo "Error: ${STEPS[1]} failed. Exiting."; exit 1; } || true;
    fi
    end_epoch=$(date +%s);
    echo "=== Ray and skypilot dependencies installation completed in $(($end_epoch - $start_epoch)) secs ===";
    start_epoch=$(date +%s);
    {{ skypilot_wheel_installation_commands }}
    end_epoch=$(date +%s);
    echo "=== Skypilot wheel installation completed in $(($end_epoch - $start_epoch)) secs ===";
    start_epoch=$(date +%s);
    sudo touch ~/.sudo_as_admin_successful;
    sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf');
    ulimit -n 1048576;
    mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
    [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf'); # This is needed for `-o allow_other` option for `goofys`;
    end_epoch=$(date +%s);
    echo "=== Setup system configs and fuse completed in $(($end_epoch - $start_epoch)) secs ===";
    for step in $STEPS; do [ -f "/tmp/${step}.failed" ] && { echo "Error: /tmp/${step}.failed found:"; cat /tmp/${step}.log; exit 1; } || true; done;
    {% if tpu_requested %}
    # The /tmp/tpu_logs directory is where TPU-related logs, such as logs from
    # the TPU runtime, are written. These capture runtime information about the
    # TPU execution, including any warnings, errors, or general activity of
    # the TPU driver. By default, the /tmp/tpu_logs directory is created with
    # 755 permissions, and the user of the provisioned pod is not necessarily
    # a root. Hence, we need to update the write permission so the logs can be
    # properly written.
    # TODO(Doyoung): Investigate to see why TPU workload fails to run without
    # execution permission, such as granting 766 to log file. Check if it's a
    # must and see if there's a workaround to grant minimum permission. 
    sudo chmod 777 /tmp/tpu_logs;
    {% endif %}

# Format: `REMOTE_PATH : LOCAL_PATH`
file_mounts: {
  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
{%- for remote_path, local_path in credentials.items() %}
  "{{remote_path}}": "{{local_path}}",
{%- endfor %}
}

auth:
  ssh_user: sky
  ssh_private_key: {{ssh_private_key}}
