Monitor MachinePool Health

.github/workflows/publish-docker.yml

@@ -87,7 +87,7 @@ jobs:
         version: latest
         endpoint: builders # self-hosted
     - name: Login to GHCR
-      if: github.event_name != 'pull_request'
+      if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ok-to-image')
       uses: docker/login-action@v4
       with:
         registry: ghcr.io
@@ -99,7 +99,7 @@ jobs:
       with:
         context: .
         platforms: linux/amd64,linux/arm64
-        push: ${{ github.event_name != 'pull_request' }}
+        push: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ok-to-image') }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         target: ${{ matrix.image.target }}

api/compute/v1alpha1/common.go

@@ -9,6 +9,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
+const (
+	NamespaceMachinePoolLease = "ironcore-machinepool-lease"
+)
+
 const (
 	MachineMachinePoolRefNameField  = "spec.machinePoolRef.name"
 	MachineMachineClassRefNameField = "spec.machineClassRef.name"

api/compute/v1alpha1/conditions.go

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"slices"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// FindMachinePoolCondition returns a pointer to the condition of the given type,
+// or nil if no condition of that type is present.
+func FindMachinePoolCondition(conditions []MachinePoolCondition, typ MachinePoolConditionType) *MachinePoolCondition {
+	idx := slices.IndexFunc(conditions, func(cond MachinePoolCondition) bool {
+		return cond.Type == typ
+	})
+	if idx < 0 {
+		return nil
+	}
+	return &conditions[idx]
+}
+
+// SetMachinePoolCondition inserts or updates a condition of the given type in the
+// conditions slice. LastUpdateTime is always set to now. LastTransitionTime is set
+// to now only when the condition is newly inserted or its Status differs from the
+// previous value.
+func SetMachinePoolCondition(conditions []MachinePoolCondition, cond MachinePoolCondition) []MachinePoolCondition {
+	idx := slices.IndexFunc(conditions, func(c MachinePoolCondition) bool {
+		return c.Type == cond.Type
+	})
+
+	cond.LastUpdateTime = metav1.Now()
+
+	if idx < 0 || conditions[idx].Status != cond.Status {
+		cond.LastTransitionTime = metav1.Now()
+	} else {
+		cond.LastTransitionTime = conditions[idx].LastTransitionTime
+	}
+
+	if idx < 0 {
+		return append(conditions, cond)
+	}
+	conditions[idx] = cond
+	return conditions
+}

api/compute/v1alpha1/conditions_test.go

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1_test
+
+import (
+	"time"
+
+	computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
+	"github.com/onsi/gomega/types"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Conditions", func() {
+	DescribeTable("FindMachinePoolCondition",
+		func(conds []computev1alpha1.MachinePoolCondition, condType computev1alpha1.MachinePoolConditionType, match types.GomegaMatcher) {
+			Expect(computev1alpha1.FindMachinePoolCondition(conds, condType)).To(match)
+		},
+		Entry("returns the matching condition",
+			[]computev1alpha1.MachinePoolCondition{
+				{Type: "Other", Status: corev1.ConditionTrue},
+				{Type: computev1alpha1.MachinePoolReady, Status: corev1.ConditionFalse, Reason: "X"},
+			},
+			computev1alpha1.MachinePoolReady,
+			PointTo(MatchFields(IgnoreExtras, Fields{
+				"Type":   Equal(computev1alpha1.MachinePoolReady),
+				"Reason": Equal("X"),
+			})),
+		),
+		Entry("returns nil when no condition of the given type is present",
+			[]computev1alpha1.MachinePoolCondition{{Type: "Other"}},
+			computev1alpha1.MachinePoolReady,
+			BeNil(),
+		),
+		Entry("returns nil for an empty slice",
+			[]computev1alpha1.MachinePoolCondition{},
+			computev1alpha1.MachinePoolReady,
+			BeNil(),
+		),
+	)
+
+	Describe("SetMachinePoolCondition", func() {
+		It("should append the condition when it is absent", func() {
+			out := computev1alpha1.SetMachinePoolCondition(nil, computev1alpha1.MachinePoolCondition{
+				Type:   computev1alpha1.MachinePoolReady,
+				Status: corev1.ConditionTrue,
+			})
+
+			Expect(out).To(HaveLen(1))
+			Expect(out[0].Type).To(Equal(computev1alpha1.MachinePoolReady))
+
+			By("setting LastTransitionTime on first append")
+			Expect(out[0].LastTransitionTime.IsZero()).To(BeFalse())
+
+			By("setting LastUpdateTime on first append")
+			Expect(out[0].LastUpdateTime.IsZero()).To(BeFalse())
+		})
+
+		It("should update in place without advancing LastTransitionTime when status is unchanged", func() {
+			earlier := metav1.NewTime(time.Now().Add(-time.Hour))
+			in := []computev1alpha1.MachinePoolCondition{{
+				Type:               computev1alpha1.MachinePoolReady,
+				Status:             corev1.ConditionTrue,
+				LastTransitionTime: earlier,
+			}}
+
+			out := computev1alpha1.SetMachinePoolCondition(in, computev1alpha1.MachinePoolCondition{
+				Type:    computev1alpha1.MachinePoolReady,
+				Status:  corev1.ConditionTrue, // same status
+				Message: "still ready",
+			})
+
+			By("preserving LastTransitionTime when status is unchanged")
+			Expect(out[0].LastTransitionTime.Equal(&earlier)).To(BeTrue())
+
+			By("advancing LastUpdateTime")
+			Expect(out[0].LastUpdateTime.IsZero()).To(BeFalse())
+		})
+
+		It("should advance LastTransitionTime when the status changes", func() {
+			earlier := metav1.NewTime(time.Now().Add(-time.Hour))
+			in := []computev1alpha1.MachinePoolCondition{{
+				Type:               computev1alpha1.MachinePoolReady,
+				Status:             corev1.ConditionTrue,
+				LastTransitionTime: earlier,
+			}}
+
+			out := computev1alpha1.SetMachinePoolCondition(in, computev1alpha1.MachinePoolCondition{
+				Type:   computev1alpha1.MachinePoolReady,
+				Status: corev1.ConditionFalse,
+			})
+
+			Expect(out[0].LastTransitionTime.Equal(&earlier)).To(BeFalse())
+		})
+	})
+})

api/compute/v1alpha1/machinepool_types.go

@@ -85,6 +85,11 @@ type MachinePoolAddress struct {
 // MachinePoolConditionType is a type a MachinePoolCondition can have.
 type MachinePoolConditionType string
 
+const (
+	// MachinePoolReady means the machine pool is healthy and ready to accept machines.
+	MachinePoolReady MachinePoolConditionType = "Ready"
+)
+
 // MachinePoolCondition is one of the conditions of a MachinePool.
 type MachinePoolCondition struct {
 	// Type is the type of the condition.
@@ -97,6 +102,8 @@ type MachinePoolCondition struct {
 	Message string `json:"message"`
 	// ObservedGeneration represents the .metadata.generation that the condition was set based upon.
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+	// LastUpdateTime is the last time this condition was updated.
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
 	// LastTransitionTime is the last time the status of a condition has transitioned from one state to another.
 	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
 }

api/compute/v1alpha1/v1alpha1_suite_test.go

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestV1Alpha1(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Compute v1alpha1 Suite")
+}

cmd/ironcore-controller-manager/main.go

@@ -42,6 +42,8 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
@@ -50,6 +52,7 @@ import (
 	computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1"
 	ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1"
 	storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1"
+	coordinationv1 "k8s.io/api/coordination/v1"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -64,6 +67,7 @@ const (
 	machineEphemeralVolumeController           = "machineephemeralvolume"
 	machineSchedulerController                 = "machinescheduler"
 	machineClassController                     = "machineclass"
+	machinePoolLifecycleController             = "machinepoollifecycle"
 
 	// storage controllers
 	bucketScheduler           = "bucketscheduler"
@@ -114,6 +118,7 @@ func main() {
 	var volumeBindTimeout time.Duration
 	var virtualIPBindTimeout time.Duration
 	var networkInterfaceBindTimeout time.Duration
+	var machinePoolLifecycleGracePeriod time.Duration
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -134,13 +139,15 @@ func main() {
 	flag.DurationVar(&volumeBindTimeout, "volume-bind-timeout", 10*time.Second, "Time to wait until considering a volume bind to be failed.")
 	flag.DurationVar(&virtualIPBindTimeout, "virtual-ip-bind-timeout", 10*time.Second, "Time to wait until considering a virtual ip bind to be failed.")
 	flag.DurationVar(&networkInterfaceBindTimeout, "network-interface-bind-timeout", 10*time.Second, "Time to wait until considering a network interface bind to be failed.")
+	flag.DurationVar(&machinePoolLifecycleGracePeriod, "machine-pool-lifecycle-grace-period", 50*time.Second, "Grace period without a heartbeat before a machine pool's Ready condition is marked Unknown.")
 
 	controllers := switches.New(
 		// compute controllers
 		machineEphemeralNetworkInterfaceController,
 		machineEphemeralVolumeController,
 		machineSchedulerController,
 		machineClassController,
+		machinePoolLifecycleController,
 
 		// storage controllers
 		bucketScheduler,
@@ -257,6 +264,19 @@ func main() {
 		PprofBindAddress:       pprofAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "d0ae00be.ironcore.dev",
+		Cache: cache.Options{
+			ByObject: map[client.Object]cache.ByObject{
+				// The MachinePool lifecycle controller watches Leases only in
+				// NamespaceMachinePoolLease. Scoping the cache here keeps the
+				// underlying informer namespaced so the controller's RBAC can
+				// stay namespaced too.
+				&coordinationv1.Lease{}: {
+					Namespaces: map[string]cache.Config{
+						computev1alpha1.NamespaceMachinePoolLease: {},
+					},
+				},
+			},
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create manager")
@@ -321,6 +341,16 @@ func main() {
 		}
 	}
 
+	if controllers.Enabled(machinePoolLifecycleController) {
+		if err := (&computecontrollers.MachinePoolLifecycleReconciler{
+			Client:      mgr.GetClient(),
+			GracePeriod: machinePoolLifecycleGracePeriod,
+		}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "MachinePoolLifecycle")
+			os.Exit(1)
+		}
+	}
+
 	// storage controllers
 
 	if controllers.Enabled(bucketScheduler) {

config/apiserver/rbac/machinepool_role.yaml

@@ -140,3 +140,21 @@ rules:
   - patch
   - update
   - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: compute.ironcore.dev:system:machinepools
+  namespace: ironcore-machinepool-lease
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch

config/apiserver/standalone-etcdless/kustomization.yaml

@@ -7,3 +7,4 @@ kind: Kustomization
 resources:
   - ../etcdless
   - ../../namespaces/ironcore-system
+  - ../../namespaces/machinepool-lease

config/apiserver/standalone/kustomization.yaml

@@ -8,3 +8,4 @@ kind: Kustomization
 resources:
   - ../default
   - ../../namespaces/ironcore-system
+  - ../../namespaces/machinepool-lease

config/controller/rbac/role.yaml

@@ -74,6 +74,7 @@ rules:
   - compute.ironcore.dev
   resources:
   - machineclasses/status
+  - machinepools/status
   - machines/status
   verbs:
   - get

config/controller/standalone/kustomization.yaml

@@ -9,3 +9,4 @@ kind: Kustomization
 resources:
   - ../default
   - ../../namespaces/ironcore-system
+  - ../../namespaces/machinepool-lease

config/default/kustomization.yaml

@@ -8,3 +8,4 @@ resources:
   - ../apiserver/default
   - ../controller/default
   - ../namespaces/ironcore-system
+  - ../namespaces/machinepool-lease