apiVersion: kops/v1alpha2
kind: Cluster
metadata:
  name: {{ cluster_name }}
spec:
  {%- if additional_policies %}
  additionalPolicies:
   {{ additional_policies|indent(4) }}
  {%- endif %}
  api:
    loadBalancer:
      type: Public
  authorization:
    {{ authorization|indent(2) }}
  channel: stable
  cloudProvider: aws
  configBase: s3://{{ kops_state_store_bucket }}/{{ cluster_name }}
  dnsZone: {{ cluster_dns }}
  etcdClusters:
  - name: main
    enableEtcdTLS: true
    version: 3.2.24
    etcdMembers:
    {%- for az in master_availability_zones %}
    - instanceGroup: master-{{ az }}
      name: {{ az }}
      encryptedVolume: true
    {%- endfor %}
  - name: events
    enableEtcdTLS: true
    version: 3.2.24
    etcdMembers:
    {%- for az in master_availability_zones %}
    - instanceGroup: master-{{ az }}
      name: {{ az }}
      encryptedVolume: true
    {%- endfor %}
  kubeAPIServer:
    auditLogPath: /var/log/kube-apiserver-audit.log
    auditLogMaxAge: 10
    auditLogMaxBackups: 1
    auditLogMaxSize: 100
    auditPolicyFile: /srv/kubernetes/audit.yaml
  kubernetesApiAccess:
  - 0.0.0.0/0
  kubernetesVersion: {{ kubernetes_version }}
  masterPublicName: api.{{ cluster_name }}
  networkCIDR: {{ network_cidr }}
  {%- if vpc_id %}
  networkID: {{ vpc_id }}
  {%- endif %}
  networking:
    {{ networking|indent(2) }}
  nonMasqueradeCIDR: 100.64.0.0/10
  sshAccess:
  - 0.0.0.0/0
  subnets:
  {%- for az in availability_zones %}
  - cidr: {{ network_cidr_base|string + "." + ((third_octet|int) + (loop.index - 1) * third_octet_increment)|string + ".0/" + network_mask|string }}
    name: {{ az }}
    type: Private
    zone: {{ az }}
    {%- if nat_gateways %}
    egress: {{ nat_gateways[loop.index-1] }}
    {%- endif %}
  {%- endfor -%}
  {%- for az in availability_zones %}
  - cidr: {{ network_cidr_base|string + "." + ((third_octet|int + 4 * third_octet_increment) + (loop.index - 1) * third_octet_increment)|string + ".0/" + network_mask|string }}
    name: utility-{{ az }}
    type: Utility
    zone: {{ az }}
  {%- endfor %}
  topology:
    dns:
      type: Public
    masters: private
    nodes: private
  fileAssets:
  - name: auditPolicyFile
    path: /srv/kubernetes/audit.yaml
    roles: [Master]
    content: |
      apiVersion: audit.k8s.io/v1beta1
      kind: Policy
      rules:
        # The following requests were manually identified as high-volume and low-risk,
        # so drop them.
        - level: None
          users: ["system:kube-proxy"]
          verbs: ["watch"]
          resources:
            - group: "" # core
              resources: ["endpoints", "services", "services/status"]
        - level: None
          # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
          # TODO(#46983): Change this to the ingress controller service account.
          users: ["system:unsecured"]
          namespaces: ["kube-system"]
          verbs: ["get"]
          resources:
            - group: "" # core
              resources: ["configmaps"]
        - level: None
          users: ["kubelet"] # legacy kubelet identity
          verbs: ["get"]
          resources:
            - group: "" # core
              resources: ["nodes", "nodes/status"]
        - level: None
          userGroups: ["system:nodes"]
          verbs: ["get"]
          resources:
            - group: "" # core
              resources: ["nodes", "nodes/status"]
        - level: None
          users:
            - system:kube-controller-manager
            - system:kube-scheduler
            - system:serviceaccount:kube-system:endpoint-controller
          verbs: ["get", "update"]
          namespaces: ["kube-system"]
          resources:
            - group: "" # core
              resources: ["endpoints"]
        - level: None
          users: ["system:apiserver"]
          verbs: ["get"]
          resources:
            - group: "" # core
              resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
        # Don't log HPA fetching metrics.
        - level: None
          users:
            - system:kube-controller-manager
          verbs: ["get", "list"]
          resources:
            - group: "metrics.k8s.io"
        # Don't log these read-only URLs.
        - level: None
          nonResourceURLs:
            - /healthz*
            - /version
            - /swagger*
        # Don't log events requests.
        - level: None
          resources:
            - group: "" # core
              resources: ["events"]
        # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
        - level: Request
          users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
          verbs: ["update","patch"]
          resources:
            - group: "" # core
              resources: ["nodes/status", "pods/status"]
          omitStages:
            - "RequestReceived"
        - level: Request
          userGroups: ["system:nodes"]
          verbs: ["update","patch"]
          resources:
            - group: "" # core
              resources: ["nodes/status", "pods/status"]
          omitStages:
            - "RequestReceived"
        # deletecollection calls can be large, don't log responses for expected namespace deletions
        - level: Request
          users: ["system:serviceaccount:kube-system:namespace-controller"]
          verbs: ["deletecollection"]
          omitStages:
            - "RequestReceived"
        # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
        # so only log at the Metadata level.
        - level: Metadata
          resources:
            - group: "" # core
              resources: ["secrets", "configmaps"]
            - group: authentication.k8s.io
              resources: ["tokenreviews"]
          omitStages:
            - "RequestReceived"
        # Get repsonses can be large; skip them.
        - level: Request
          verbs: ["get", "list", "watch"]
          resources:
            - group: "" # core
            - group: "admissionregistration.k8s.io"
            - group: "apiextensions.k8s.io"
            - group: "apiregistration.k8s.io"
            - group: "apps"
            - group: "authentication.k8s.io"
            - group: "authorization.k8s.io"
            - group: "autoscaling"
            - group: "batch"
            - group: "certificates.k8s.io"
            - group: "extensions"
            - group: "metrics.k8s.io"
            - group: "networking.k8s.io"
            - group: "policy"
            - group: "rbac.authorization.k8s.io"
            - group: "scheduling.k8s.io"
            - group: "settings.k8s.io"
            - group: "storage.k8s.io"
          omitStages:
            - "RequestReceived"
        # Default level for known APIs
        - level: RequestResponse
          resources:
            - group: "" # core
            - group: "admissionregistration.k8s.io"
            - group: "apiextensions.k8s.io"
            - group: "apiregistration.k8s.io"
            - group: "apps"
            - group: "authentication.k8s.io"
            - group: "authorization.k8s.io"
            - group: "autoscaling"
            - group: "batch"
            - group: "certificates.k8s.io"
            - group: "extensions"
            - group: "metrics.k8s.io"
            - group: "networking.k8s.io"
            - group: "policy"
            - group: "rbac.authorization.k8s.io"
            - group: "scheduling.k8s.io"
            - group: "settings.k8s.io"
            - group: "storage.k8s.io"
          omitStages:
            - "RequestReceived"
        # Default level for all other requests.
        - level: Metadata
          omitStages:
            - "RequestReceived"
