add metric cleanup on group delete

Author: jessicaochen

pkg/accelerator-orchestrator/metrics/metrics.go

@@ -48,6 +48,14 @@ var (
 	)
 )
 
+// CleanupGroup removes gauge series labeled with the given group so stale
+// values don't persist in /metrics after the group is deleted. Cumulative
+// metrics (histograms, counters) are left intact so group ID reuse doesn't
+// appear as a counter reset.
+func CleanupGroup(groupID string) {
+	QueueDepth.DeletePartialMatch(prometheus.Labels{"group_id": groupID})
+}
+
 // Register registers all accelerator orchestrator Prometheus metrics with the default registry.
 func Register() {
 	prometheus.MustRegister(

pkg/accelerator-orchestrator/store/group_store.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"sync"
+
+	"github.com/llm-d-incubation/llm-d-rl-time-slicing/pkg/accelerator-orchestrator/metrics"
 )
 
 var (
@@ -106,6 +108,7 @@ func (s *GroupStore) Delete(ctx context.Context, id string) error {
 			return err
 		}
 		delete(s.groups, id)
+		metrics.CleanupGroup(id)
 	}
 	return nil
 }