Add graceful shutdown to worker nodes

cluster/config-defaults.yaml

@@ -60,6 +60,20 @@ karpenter_flex_types_enabled: "false"
 karpenter_flex_types_enabled: "true"
 {{end}}
 
+# Define Karpenter AMI family alias. This is used to determine the AMI used for
+# Karpenter nodes.
+#
+# The default is custom which is the Zalando Kubernetes on Ubuntu AMI.
+#
+# alternatives:
+#  - al2023@latest
+#  - bottlerocket@latest
+{{- if eq .Cluster.Provider "zalando-eks" }}
+karpenter_ami_family_alias: "bottlerocket@latest"
+{{- else }}
+karpenter_ami_family_alias: "custom"
+{{- end }}
+
 # configure whether spot instances should be enabled in Karpenter's capacity-types
 karpenter_enable_spot: "true"
 
@@ -585,7 +599,7 @@ static_egress_controller_enabled: "false"
 static_egress_controller_memory_min: "10Mi"
 
 # journald reader settings
-journald_reader_enabled: "true"
+journald_reader_enabled: "false"
 journald_reader_cpu: "1m"
 journald_reader_memory: "30Mi"
 
@@ -1078,7 +1092,7 @@ dns_coredns_mem: "195Mi"
 #  kubelet - kubelet configures static dns IP address - worker node rotation!
 #  only_interface - dns cache listens _only_ on the static interface
 #
-dns_static_interface: "listening"
+dns_static_interface: "only_interface"
 
 # special roles for test/pet clusters
 {{if eq .Cluster.Environment "e2e"}}

cluster/manifests/nvidia/nvidia-gpu-device-plugin.yaml

@@ -45,9 +45,17 @@ spec:
                 operator: In
                 values:
                 - nvidia
+              - key: ami-family
+                operator: NotIn
+                values:
+                - bottlerocket
             - matchExpressions:
               - key: zalando.org/nvidia-gpu
                 operator: Exists
+              - key: ami-family
+                operator: NotIn
+                values:
+                - bottlerocket
       priorityClassName: system-node-critical
       volumes:
       - name: device-plugin

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -4,20 +4,30 @@ kind: EC2NodeClass
 metadata:
   name: "{{ .NodePool.Name }}"
 spec:
+  # {{ if eq .NodePool.ConfigItems.karpenter_ami_family_alias "custom" }}
   amiFamily: Custom
   amiSelectorTerms:
     # Select on any AMI that has any of the following IDs
     - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_35_" .NodePool.ConfigItems.kuberuntu_ami_version "_amd64") }}
     - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_35_" .NodePool.ConfigItems.kuberuntu_ami_version "_arm64") }}
+  # {{ else }}
+  amiSelectorTerms:
+    - alias: "{{ .NodePool.ConfigItems.karpenter_ami_family_alias }}"
+  # {{ end }}
   metadataOptions:
     httpEndpoint: enabled
     # {{ if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
     httpProtocolIPv6: enabled
     # {{ else }}
     httpProtocolIPv6: disabled
     # {{ end }}
+    # {{ if or (hasPrefix .NodePool.ConfigItems.karpenter_ami_family_alias "bottlerocket") (hasPrefix .NodePool.ConfigItems.karpenter_ami_family_alias "al2023") }}
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+    # {{ else }}
     httpPutResponseHopLimit: 2
     httpTokens: optional
+    # {{ end }}
   subnetSelectorTerms:
     - tags:
         # {{ if eq .NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
@@ -96,8 +106,24 @@ spec:
 #    {{ end}}
 #  {{ end}}
 #{{ end}}
+{{- if eq .NodePool.ConfigItems.karpenter_ami_family_alias "custom" }}
   userData: |
 {{ .Values.UserData | indent 4 }}
+{{- else if hasPrefix .NodePool.ConfigItems.karpenter_ami_family_alias "al2023" }}
+  userData: |
+    ---
+    apiVersion: node.eks.aws/v1alpha1
+    kind: NodeConfig
+    spec:
+      kubelet:
+        flags:
+          # hack to pass admission-controller requirements
+          - "--node-labels=node.kubernetes.io/role=worker"
+{{- else if hasPrefix .NodePool.ConfigItems.karpenter_ami_family_alias "bottlerocket" }}
+  userData: |
+      [settings.kubernetes]
+      node-labels = {"node.kubernetes.io/role" = "worker"}
+{{- end }}
 ---
 apiVersion: karpenter.sh/v1
 kind: NodePool
@@ -112,6 +138,9 @@ spec:
     metadata:
       # Labels are arbitrary key-values that are applied to all nodes
       labels:
+# {{ if hasPrefix .NodePool.ConfigItems.karpenter_ami_family_alias "bottlerocket" }}
+        ami-family: "bottlerocket"
+# {{ end }}
         lifecycle-status: ready
         node.kubernetes.io/node-pool: "{{ .NodePool.Name }}"
         node.kubernetes.io/role: worker

cluster/node-pools/worker-splitaz/userdata.yaml

@@ -130,6 +130,8 @@ write_files:
           cacheAuthorizedTTL: "5m"
           cacheUnauthorizedTTL: "30s"
       protectKernelDefaults: true
+      shutdownGracePeriod: "120s"
+      shutdownGracePeriodCriticalPods: "80s"
       # variables are replaced on instance start-up.
       providerID: __PROVIDER_ID__
 {{- if or (eq .Cluster.ConfigItems.dns_static_interface "kubelet") (eq .Cluster.ConfigItems.dns_static_interface "only_interface") }}

test/e2e/Dockerfile

@@ -25,7 +25,7 @@ RUN curl -L -s --fail "https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux
 COPY --from=builder /go/bin/ginkgo /usr/local/bin/ginkgo
 
 # copy CLM
-COPY --from=container-registry.zalando.net/teapot/cluster-lifecycle-manager:latest /clm /usr/bin/clm
+COPY --from=container-registry-test.zalando.net/teapot/cluster-lifecycle-manager:pr-950-4 /clm /usr/bin/clm
 COPY --from=container-registry.zalando.net/teapot/aws-account-creator:latest /aws-account-creator /usr/bin/aws-account-creator
 
 ADD test/e2e/build/linux/${TARGETARCH}/e2e.test /workdir/test/e2e/e2e.test

test/e2e/run_e2e.sh

@@ -215,6 +215,8 @@ if [ "$e2e" = true ]; then
             "Mirror pods should be created for the main Kubernetes components \[Zalando\]"
             "Should audit API calls to create, update, patch, delete pods. \[Audit\] \[Zalando\]"
             "should validate permissions for \[Authorization\] \[RBAC\] \[Zalando\]" # TODO: Remains skipped until we remove the older RBAC setup
+            "Should react to spot termination notices \[Zalando\] \[Spot\]" # not compatible with bottlerocket AMI
+            "Should get AWS IAM credentials \(kube2iam\) \[AWS-IAM\] \[Zalando\]" # not compatible with bottlerocket AMI
         )
     fi