Skip to content

Commit ca5409d

Browse files
waltformewaltforme
authored
Consistent ISC usage during the life of a vLLM instance (llm-d-incubation#492)
* Consistent ISC usage during the life of a vLLM instance Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Specify range for ModelServerConfig.Port Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Introduce an event recorder and begin to use it Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Simplify recoverInstanceStateFromLauncherPod Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Clean up ensureUnbound Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Protect the newly added annotations Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Change from int16 to int32 for port number assigned to an instance Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Drop dead block as well as the event recorder Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Sharpen snapshot comments and deduplicate ensureUnbound teardown Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Drop redundant ISC-key slices from snapshot state Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> * Report ISC-config failures as non-transient requester status Signed-off-by: Jun Duan <jun.duan.phd@outlook.com> --------- Signed-off-by: Jun Duan <jun.duan.phd@outlook.com>
1 parent abb27a3 commit ca5409d

6 files changed

Lines changed: 249 additions & 161 deletions

File tree

‎api/fma/v1alpha1/inferenceserverconfig_types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ type InferenceServerConfigSpec struct {
3535
type ModelServerConfig struct {
3636
// Port is the port on which the vLLM server will listen
3737
// Particularly, management of vLLM instances' sleep state is done through this port
38+
// +kubebuilder:validation:Minimum=1
39+
// +kubebuilder:validation:Maximum=65535
3840
Port int32 `json:"port"`
3941

4042
// Options are the vLLM startup options, excluding Port

‎config/crd/fma.llm-d.ai_inferenceserverconfigs.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ spec:
8282
Port is the port on which the vLLM server will listen
8383
Particularly, management of vLLM instances' sleep state is done through this port
8484
format: int32
85+
maximum: 65535
86+
minimum: 1
8587
type: integer
8688
required:
8789
- port

‎config/validating-admission-policies/fma-immutable-fields.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ spec:
2222
(
2323
oldObject.metadata.?annotations['dual-pods.llm-d.ai/requester'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/requester'].orValue('') &&
2424
oldObject.metadata.?annotations['dual-pods.llm-d.ai/status'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/status'].orValue('') &&
25+
oldObject.metadata.?annotations['dual-pods.llm-d.ai/instance-id'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/instance-id'].orValue('') &&
26+
oldObject.metadata.?annotations['dual-pods.llm-d.ai/server-port'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/server-port'].orValue('') &&
27+
oldObject.metadata.?annotations['dual-pods.llm-d.ai/vllm-config'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/vllm-config'].orValue('') &&
2528
oldObject.metadata.?annotations['dual-pods.llm-d.ai/isc-label-keys'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/isc-label-keys'].orValue('') &&
2629
oldObject.metadata.?annotations['dual-pods.llm-d.ai/isc-annotation-keys'].orValue('') == object.metadata.?annotations['dual-pods.llm-d.ai/isc-annotation-keys'].orValue('') &&
2730
oldObject.metadata.?labels['dual-pods.llm-d.ai/dual'].orValue('') == object.metadata.?labels['dual-pods.llm-d.ai/dual'].orValue('')

‎pkg/controller/dual-pods/controller.go‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ import (
9393

9494
const requesterAnnotationKey = "dual-pods.llm-d.ai/requester"
9595
const nominalHashAnnotationKey = "dual-pods.llm-d.ai/nominal"
96+
const launcherInstanceIDAnnotationKey = "dual-pods.llm-d.ai/instance-id"
97+
const launcherServerPortAnnotationKey = "dual-pods.llm-d.ai/server-port"
98+
const launcherVllmConfigAnnotationKey = "dual-pods.llm-d.ai/vllm-config"
9699
const iscLabelKeysAnnotationKey = "dual-pods.llm-d.ai/isc-label-keys"
97100
const iscAnnotationKeysAnnotationKey = "dual-pods.llm-d.ai/isc-annotation-keys"
98101

@@ -232,16 +235,16 @@ type controller struct {
232235
enqueueLogger klog.Logger
233236
coreclient coreclient.CoreV1Interface
234237
namespace string
235-
podInformer cache.SharedIndexInformer
236-
podLister corev1listers.PodLister
237-
cmInformer cache.SharedIndexInformer
238-
cmLister corev1listers.ConfigMapLister
239-
nodeInformer cache.SharedIndexInformer
240-
nodeLister corev1listers.NodeLister
241-
iscInformer cache.SharedIndexInformer
242-
iscLister fmalisters.InferenceServerConfigLister
243-
lcInformer cache.SharedIndexInformer
244-
lcLister fmalisters.LauncherConfigLister
238+
podInformer cache.SharedIndexInformer
239+
podLister corev1listers.PodLister
240+
cmInformer cache.SharedIndexInformer
241+
cmLister corev1listers.ConfigMapLister
242+
nodeInformer cache.SharedIndexInformer
243+
nodeLister corev1listers.NodeLister
244+
iscInformer cache.SharedIndexInformer
245+
iscLister fmalisters.InferenceServerConfigLister
246+
lcInformer cache.SharedIndexInformer
247+
lcLister fmalisters.LauncherConfigLister
245248
genctlr.KnowsProcessedSync[queueItem]
246249

247250
sleeperLimit int
@@ -292,8 +295,12 @@ type serverData struct {
292295
NominalProvidingPod *corev1.Pod
293296
NominalProvidingPodHash string
294297

295-
// ServerPort is meaningful if NominalProvidingPod is not nil
296-
ServerPort int16
298+
// ServerPort is where the inference server listens.
299+
// For direct (non-launcher-based) providers it is derived from
300+
// NominalProvidingPod. For launcher-based providers it is written by
301+
// commitInstanceState (on fresh bind) or recoverInstanceStateFromLauncherPod
302+
// (on controller-restart recovery).
303+
ServerPort int32
297304

298305
// UUIDs of the server's GPUs
299306
GPUIDs []string
@@ -304,12 +311,16 @@ type serverData struct {
304311
GPUIndices []string
305312
GPUIndicesStr *string
306313

307-
ProvidingPodName string
308-
InstanceID string // ISC hash; set when computed, independent of instance existence
309-
InstanceConfig *VllmConfig
310-
InstanceKnownToExist bool // meaningful only for launcher-based providers
311-
ISCLabelKeys []string // keys of ISC labels applied to providingPod
312-
ISCAnnotationKeys []string // keys of ISC annotations applied to providingPod
314+
ProvidingPodName string
315+
316+
// The next two fields form a snapshot of the bound launcher-based
317+
// vLLM instance's ISC-derived state. Each field may be reassigned
318+
// over time (e.g. on rebind), but whichever value (pointer) is
319+
// currently stored is deeply immutable for as long as it is stored.
320+
InstanceID string
321+
InstanceConfig *VllmConfig
322+
323+
InstanceKnownToExist bool // meaningful only for launcher-based providers
313324

314325
ReadinessRelayed *bool
315326

0 commit comments

Comments
 (0)