From ecd977367f2031900e30572a0c6d1a6159025f9c Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Fri, 30 Jan 2026 17:50:45 +0100 Subject: [PATCH] feat: add support for new field prefill.servicePort - to support prefill use a different port than decode's port - if prefill.servicePort is not set, fall back to use routing.servicePort - if both prefill.servicePort and prefill.container.ports are set, servicePort take precedence to be containerPorts Signed-off-by: Wen Zhou --- charts/llm-d-modelservice/Chart.yaml | 2 +- .../llm-d-modelservice/templates/_helpers.tpl | 19 +++++- charts/llm-d-modelservice/values.schema.json | 5 +- .../values.schema.tmpl.json | 5 +- charts/llm-d-modelservice/values.yaml | 6 ++ examples/output-cpu.yaml | 6 +- examples/output-dra.yaml | 6 +- examples/output-gaudi.yaml | 4 +- examples/output-heterogeneous-pd.yaml | 8 +-- examples/output-pd-mnnvl.yaml | 6 +- examples/output-pd.yaml | 6 +- examples/output-pvc-hf.yaml | 6 +- examples/output-pvc.yaml | 6 +- examples/output-requester.yaml | 4 +- examples/output-xpu-pd.yaml | 6 +- examples/output-xpu.yaml | 4 +- examples/values-simple-port-pd.yaml | 66 +++++++++++++++++++ 17 files changed, 124 insertions(+), 41 deletions(-) create mode 100644 examples/values-simple-port-pd.yaml diff --git a/charts/llm-d-modelservice/Chart.yaml b/charts/llm-d-modelservice/Chart.yaml index a31e0463..be1a0ee1 100644 --- a/charts/llm-d-modelservice/Chart.yaml +++ b/charts/llm-d-modelservice/Chart.yaml @@ -13,7 +13,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "v0.4.3" +version: "v0.4.4" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/llm-d-modelservice/templates/_helpers.tpl b/charts/llm-d-modelservice/templates/_helpers.tpl index 718a8d1e..427c91eb 100644 --- a/charts/llm-d-modelservice/templates/_helpers.tpl +++ b/charts/llm-d-modelservice/templates/_helpers.tpl @@ -221,9 +221,18 @@ Required number of GPU per worker -- dpl * tp {{/* Port on which vllm container should listen. Context is helm root context plus key "role" ("decode" or "prefill") +For prefill: uses prefill.servicePort if set, otherwise fall back to routing.servicePort +For decode with proxy: uses routing.proxy.targetPort +For decode without proxy: uses routing.servicePort */}} {{- define "llm-d-modelservice.vllmPort" -}} -{{- if or (eq .role "prefill") (eq .Values.routing.proxy.enabled false) }} +{{- if eq .role "prefill" }} +{{- if .Values.prefill.servicePort }} +{{- .Values.prefill.servicePort }} +{{- else }} +{{- .Values.routing.servicePort }} +{{- end }} +{{- else if eq .Values.routing.proxy.enabled false }} {{- .Values.routing.servicePort }} {{- else }} {{- .Values.routing.proxy.targetPort }} @@ -477,9 +486,13 @@ context is a dict with helm root context plus: {{- if $acceleratorEnv }}{{ $acceleratorEnv | nindent 2 }}{{- end }} {{- /* Add tracing environment variables from global config */}} {{- (include "llm-d-modelservice.tracingEnv" .) | nindent 2 }} - {{- with .container.ports }} + {{- /* Set containerPorts - prefill.servicePort takes precedence if it is set */}} + {{- if and (eq .role "prefill") .Values.prefill.servicePort }} ports: - {{- include "common.tplvalues.render" ( dict "value" . "context" $ ) | nindent 2 }} + - containerPort: {{ .Values.prefill.servicePort }} + {{- else if .container.ports }} + ports: + {{- include "common.tplvalues.render" ( dict "value" .container.ports "context" $ ) | nindent 2 }} {{- end }} {{- /* DEPRECATED; use extraConfig.livenessProbe instead */ -}} {{- with .container.livenessProbe }} diff --git a/charts/llm-d-modelservice/values.schema.json b/charts/llm-d-modelservice/values.schema.json index 3504bc56..5356e147 100644 --- a/charts/llm-d-modelservice/values.schema.json +++ b/charts/llm-d-modelservice/values.schema.json @@ -3557,7 +3557,7 @@ }, "parallelism": { "additionalProperties": false, - "description": "VLLM parallelism configuration", + "description": "Name of the scheduler to use for scheduling prefill pods (overrides global schedulerName) schedulerName: prefill-scheduler VLLM parallelism configuration", "properties": { "data": { "default": 1, @@ -3585,8 +3585,7 @@ } }, "required": [], - "title": "parallelism", - "type": "object" + "title": "parallelism" }, "replicas": { "default": 0, diff --git a/charts/llm-d-modelservice/values.schema.tmpl.json b/charts/llm-d-modelservice/values.schema.tmpl.json index 90138a04..44dc47c8 100644 --- a/charts/llm-d-modelservice/values.schema.tmpl.json +++ b/charts/llm-d-modelservice/values.schema.tmpl.json @@ -929,7 +929,7 @@ }, "parallelism": { "additionalProperties": false, - "description": "VLLM parallelism configuration", + "description": "Name of the scheduler to use for scheduling prefill pods (overrides global schedulerName) schedulerName: prefill-scheduler VLLM parallelism configuration", "properties": { "data": { "default": 1, @@ -957,8 +957,7 @@ } }, "required": [], - "title": "parallelism", - "type": "object" + "title": "parallelism" }, "replicas": { "default": 0, diff --git a/charts/llm-d-modelservice/values.yaml b/charts/llm-d-modelservice/values.yaml index b4073136..6a35af6d 100644 --- a/charts/llm-d-modelservice/values.yaml +++ b/charts/llm-d-modelservice/values.yaml @@ -459,10 +459,16 @@ prefill: enabled: false replicas: 0 + # @schema # additionalProperties: true # @schema nodeSelector: {} + + # Port on which the prefill vLLM container listens + # When not specified, falls back to routing.servicePort + # Uncomment to override: + # servicePort: 8000 # schedulerName -- Name of the scheduler to use for scheduling prefill pods (overrides global schedulerName) # schedulerName: prefill-scheduler diff --git a/examples/output-cpu.yaml b/examples/output-cpu.yaml index 7e75444e..6f20b079 100644 --- a/examples/output-cpu.yaml +++ b/examples/output-cpu.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: cpu-sim-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: cpu-sim-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -105,7 +105,7 @@ kind: Deployment metadata: name: cpu-sim-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-dra.yaml b/examples/output-dra.yaml index 4f929319..eaa7c932 100644 --- a/examples/output-dra.yaml +++ b/examples/output-dra.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: dra-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: dra-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -115,7 +115,7 @@ kind: ResourceClaimTemplate metadata: name: intel-gaudi-claim-template-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm llm-d.ai/role: decode diff --git a/examples/output-gaudi.yaml b/examples/output-gaudi.yaml index 928327e4..4581a8c0 100644 --- a/examples/output-gaudi.yaml +++ b/examples/output-gaudi.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: gaudi-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: gaudi-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-heterogeneous-pd.yaml b/examples/output-heterogeneous-pd.yaml index c1dbf8b8..40b7c479 100644 --- a/examples/output-heterogeneous-pd.yaml +++ b/examples/output-heterogeneous-pd.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: heterogeneous-pd-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: heterogeneous-pd-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -131,7 +131,7 @@ kind: Deployment metadata: name: heterogeneous-pd-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -224,7 +224,7 @@ kind: ResourceClaimTemplate metadata: name: nvidia-claim-template-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm llm-d.ai/role: decode diff --git a/examples/output-pd-mnnvl.yaml b/examples/output-pd-mnnvl.yaml index fdb3c0d4..0eaa5131 100644 --- a/examples/output-pd-mnnvl.yaml +++ b/examples/output-pd-mnnvl.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: pd-mnnvl-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: pd-mnnvl-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -131,7 +131,7 @@ kind: Deployment metadata: name: pd-mnnvl-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-pd.yaml b/examples/output-pd.yaml index d9127aa8..eb5a8e40 100644 --- a/examples/output-pd.yaml +++ b/examples/output-pd.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: pd-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: pd-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -127,7 +127,7 @@ kind: Deployment metadata: name: pd-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-pvc-hf.yaml b/examples/output-pvc-hf.yaml index f340b528..aff3f74e 100644 --- a/examples/output-pvc-hf.yaml +++ b/examples/output-pvc-hf.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: pvc-hf-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: pvc-hf-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -127,7 +127,7 @@ kind: Deployment metadata: name: pvc-hf-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-pvc.yaml b/examples/output-pvc.yaml index 49e65dd5..964d9c29 100644 --- a/examples/output-pvc.yaml +++ b/examples/output-pvc.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: pvc-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: pvc-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -126,7 +126,7 @@ kind: Deployment metadata: name: pvc-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-requester.yaml b/examples/output-requester.yaml index c7bf1387..f57dbd1a 100644 --- a/examples/output-requester.yaml +++ b/examples/output-requester.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: requester-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -142,7 +142,7 @@ kind: Deployment metadata: name: requester-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-xpu-pd.yaml b/examples/output-xpu-pd.yaml index b8e05f89..3957d8a3 100644 --- a/examples/output-xpu-pd.yaml +++ b/examples/output-xpu-pd.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: xpu-pd-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: xpu-pd-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: @@ -158,7 +158,7 @@ kind: Deployment metadata: name: xpu-pd-llm-d-modelservice-prefill labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/output-xpu.yaml b/examples/output-xpu.yaml index a1a9c899..5df3542b 100644 --- a/examples/output-xpu.yaml +++ b/examples/output-xpu.yaml @@ -6,7 +6,7 @@ kind: ServiceAccount metadata: name: xpu-llm-d-modelservice labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm --- @@ -16,7 +16,7 @@ kind: Deployment metadata: name: xpu-llm-d-modelservice-decode labels: - helm.sh/chart: llm-d-modelservice-v0.4.3 + helm.sh/chart: llm-d-modelservice-v0.4.4 app.kubernetes.io/version: "v0.3.0" app.kubernetes.io/managed-by: Helm spec: diff --git a/examples/values-simple-port-pd.yaml b/examples/values-simple-port-pd.yaml new file mode 100644 index 00000000..926d7930 --- /dev/null +++ b/examples/values-simple-port-pd.yaml @@ -0,0 +1,66 @@ +# Simple example to show port configuration in P/D (Prefill/Decode) setup +# when prefill and decode pods uses different service ports. + +modelArtifacts: + name: facebook/opt-125m + labels: + llm-d.ai/inferenceServing: "true" + llm-d.ai/model: facebook-opt-125m + uri: hf://facebook/opt-125m + size: 10Gi + +# Port configuration at routing level +routing: + servicePort: 8000 # External service port (decode sidecar receives requests here) + proxy: + targetPort: 8200 # Internal decode port (decode main llm-d container responses here) + +prefill: + create: true + replicas: 1 + servicePort: 8300 # External service port (prefill pod recieves requests here) + containers: + - name: "vllm" + image: "ghcr.io/llm-d/llm-d-cuda:latest" + modelCommand: vllmServe + args: + - "--kv-transfer-config" + - '{"kv_connector":"NixlConnector", "kv_role":"kv_producer"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_PORT + value: "5600" + + # PREFILL PORTS: Uses servicePort (8300) + ports: + - containerPort: 8300 # Main service port for incoming requests + protocol: TCP + - containerPort: 5600 # NIXL side channel for P/D communication + protocol: TCP + + mountModelVolume: true + +# Decode pod - handles token generation +decode: + create: true + replicas: 1 + containers: + - name: "vllm" + image: "ghcr.io/llm-d/llm-d-cuda:latest" + modelCommand: vllmServe + args: + - "--kv-transfer-config" + - '{"kv_connector":"NixlConnector", "kv_role":"kv_consumer"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_PORT + value: "5600" # Side channel for P/D communication + + # DECODE PORTS: Uses multiple ports (8000 and 8200) + ports: + - containerPort: 8000 + protocol: TCP + - containerPort: 8200 + protocol: TCP + - containerPort: 5600 # NIXL side channel for P/D communication + protocol: TCP + + mountModelVolume: true