diff --git a/config/scenarios/guides/optimized-baseline-xpu.yaml b/config/scenarios/guides/optimized-baseline-xpu.yaml new file mode 100644 index 000000000..dd7c28ba3 --- /dev/null +++ b/config/scenarios/guides/optimized-baseline-xpu.yaml @@ -0,0 +1,193 @@ +# OPTIMIZED BASELINE - Intel XPU (device-plugin) variant +# ============================================================================ +# Adapted from config/scenarios/guides/optimized-baseline.yaml for Intel GPU +# (xe driver, classic device-plugin `gpu.intel.com/xe`). References the +# upstream guide modelserver overlay at: +# llm-d/guides/optimized-baseline/modelserver/xpu/vllm +# +# Key differences vs the NVIDIA scenario: +# * accelerator.type=intel-xe, resource=gpu.intel.com/xe (not nvidia.com/gpu) +# and vLLM image = ghcr.io/llm-d/llm-d-xpu -- both supplied by the shared +# Intel XPU values overlay (config/templates/values/overlays/xpu.yaml), +# wired in via the spec's values_file.overlays. They are NOT duplicated +# here so every XPU guide can share one accelerator/image layer. +# * decode.vllm.customCommand has NO libcuda export and NO +# --gpu-memory-utilization; uses --dtype float16 --disable-sliding-window +# as in the upstream XPU overlay. +# * model = Qwen/Qwen3-0.6B (small, fits a single Arc/B-series GPU) +# * NO DRA (cluster uses the classic Intel GPU device plugin) +# ============================================================================ + +scenario: + - name: "optimized-baseline-xpu" + + # ========================================================================= + # MODEL + # ========================================================================= + model: + name: Qwen/Qwen3-0.6B + shortName: qwen-qwen3-06b + path: models/Qwen/Qwen3-0.6B + huggingfaceId: Qwen/Qwen3-0.6B + size: 50Gi + maxModelLen: 16000 + blockSize: 16 + gpuMemoryUtilization: 0.9 + + # ========================================================================= + # DEPLOYMENT METHOD - modelservice with standalone (epponly) router + # ========================================================================= + modelservice: + enabled: true + + standalone: + enabled: false + + kustomize: + enabled: false + + gateway: + className: epponly + + # ========================================================================= + # ROUTING / GAIE + # ========================================================================= + inferenceExtension: + pluginsConfigFile: "default-plugins.yaml" + + # ========================================================================= + # PREFILL - disabled (single-pool prefix/load-aware routing only) + # ========================================================================= + prefill: + enabled: false + replicas: 0 + + # ========================================================================= + # DECODE + # ========================================================================= + decode: + replicas: 2 + + # NOTE: node affinity for Intel GPUs (decode.acceleratorType -> + # intel.feature.node.kubernetes.io/gpu) is provided by the shared XPU + # overlay (config/templates/values/overlays/xpu.yaml), not here, so it + # can be reused by every XPU guide. + + # Pod-level securityContext. supplementalGroups is a Pod field (not a + # container field), so it must live here rather than in + # extraContainerConfig.securityContext. Group 107 = render, for /dev/dri. + # NOTE: do NOT set runAsUser/runAsGroup at the pod level. The modelservice + # chart injects a routing-proxy sidecar that declares runAsNonRoot: true; + # a pod-level runAsUser:0 would propagate to it and the kubelet rejects it + # ("runAsUser breaks non-root policy"). Root is needed only by the vllm + # container, which sets it via extraContainerConfig.securityContext below. + podSecurityContext: + supplementalGroups: + - 107 + + vllm: + # XPU vLLM serve command. Mirrors the upstream xpu/vllm overlay + # (no libcuda export, no --gpu-memory-utilization). Binds to the + # metrics port so the llm-d routing sidecar can bridge it. + # --enforce-eager: the XPU torch.compile/inductor path crashes during + # warmup ("level_zero backend failed with error: 20 + # UR_RESULT_ERROR_DEVICE_LOST"); eager mode is the stable path on XPU. + customCommand: | + vllm serve /model-cache/$MODEL_PATH \ + --host 0.0.0.0 \ + --served-model-name $MODEL_NAME \ + --port $VLLM_METRICS_PORT \ + --dtype float16 \ + --enforce-eager \ + --max-model-len $VLLM_MAX_MODEL_LEN \ + --disable-sliding-window \ + --disable-uvicorn-access-log \ + --disable-access-log-for-endpoints=/health,/metrics,/v1/models + + initContainers: + - name: preprocess + imageKey: benchmark + imagePullPolicy: Always + command: ["set_llmdbench_environment.py", "-e", "/shared-config/llmdbench_env.sh", "-i"] + volumeMounts: + - name: shared-config + mountPath: /shared-config + + parallelism: + tensor: 1 + data: 1 + dataLocal: 1 + workers: 1 + + resources: + limits: + memory: 24Gi + cpu: "8" + gpu.intel.com/xe: 1 + requests: + memory: 12Gi + cpu: "4" + gpu.intel.com/xe: 1 + + extraEnvVars: [] + + # Metrics port only (no NIXL side-channel - single-node, no KV + # disaggregation). Container-level securityContext runs as root; the + # render group (107) for /dev/dri access is set at the pod level via + # decode.podSecurityContext (supplementalGroups is a Pod field). + extraContainerConfig: + ports: + - containerPort: 8200 + name: metrics + protocol: TCP + securityContext: + runAsGroup: 0 + runAsUser: 0 + imagePullPolicy: IfNotPresent + + additionalVolumeMounts: [] + additionalVolumes: [] + + # ========================================================================= + # COMMON (vLLM volumes) + # ========================================================================= + vllmCommon: + volumes: + - name: shared-config + type: emptyDir + emptyDir: {} + - name: dshm + type: emptyDir + emptyDir: + medium: Memory + sizeLimit: 16Gi + + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: shared-config + mountPath: /shared-config + + # ========================================================================= + # STORAGE + # kind's rancher.io/local-path provisioner only supports ReadWriteOnce + # (NodePath rejects ReadWriteMany), so override the RWX defaults to RWO. + # Single-node kind => 2 decode replicas share the RWO volume on one node. + # ========================================================================= + storage: + modelPvc: + size: 50Gi + accessModes: + - ReadWriteOnce + workloadPvc: + accessModes: + - ReadWriteOnce + + # ========================================================================= + # WORKLOAD / HARNESS + # ========================================================================= + workDir: "~/data/optimized-baseline-xpu" + + harness: + name: inference-perf + experimentProfile: shared_prefix_synthetic.yaml diff --git a/config/specification/guides/optimized-baseline-xpu.yaml.j2 b/config/specification/guides/optimized-baseline-xpu.yaml.j2 new file mode 100644 index 000000000..2786c5d75 --- /dev/null +++ b/config/specification/guides/optimized-baseline-xpu.yaml.j2 @@ -0,0 +1,21 @@ +# Intel XPU variant of the optimized-baseline specification. +# [REQUIRED] +{% set base_dir = base_dir | default('../') -%} +base_dir: {{ base_dir }} + +# [REQUIRED] +values_file: + path: {{ base_dir }}/config/templates/values/defaults.yaml + # Shared Intel XPU machine layer (accelerator + vLLM XPU image), deep-merged + # on top of defaults before the scenario. Keeps those bits out of the guide + # scenario so every XPU guide reuses one overlay instead of duplicating them. + overlays: + - {{ base_dir }}/config/templates/values/overlays/xpu.yaml + +# [REQUIRED] +template_dir: + path: {{ base_dir }}/config/templates/jinja + +# [OPTIONAL] +scenario_file: + path: {{ base_dir }}/config/scenarios/guides/optimized-baseline-xpu.yaml diff --git a/config/templates/jinja/13_ms-values.yaml.j2 b/config/templates/jinja/13_ms-values.yaml.j2 index a6087aae4..0aa32d252 100644 --- a/config/templates/jinja/13_ms-values.yaml.j2 +++ b/config/templates/jinja/13_ms-values.yaml.j2 @@ -339,7 +339,7 @@ decode: {% if decode.podSecurityContext is defined and decode.podSecurityContext %} podSecurityContext: -{{ decode.podSecurityContext | toyaml | indent(4) }} +{{ decode.podSecurityContext | toyaml | indent(4, first=True) }} {% endif %} {# LWS / Multinode specific options #} @@ -759,7 +759,7 @@ prefill: {% if prefill.podSecurityContext is defined and prefill.podSecurityContext %} podSecurityContext: -{{ prefill.podSecurityContext | toyaml | indent(4) }} +{{ prefill.podSecurityContext | toyaml | indent(4, first=True) }} {% endif %} {# LWS / Multinode specific options #} diff --git a/config/templates/values/overlays/xpu.yaml b/config/templates/values/overlays/xpu.yaml new file mode 100644 index 000000000..57f18e9c8 --- /dev/null +++ b/config/templates/values/overlays/xpu.yaml @@ -0,0 +1,64 @@ +# ============================================================================ +# Intel XPU values overlay +# ============================================================================ +# Reusable, machine-related overrides for running any guide on Intel GPUs +# (xe driver, classic device-plugin `gpu.intel.com/xe`). +# +# This file is deep-merged ON TOP OF config/templates/values/defaults.yaml +# BEFORE the guide scenario is applied (defaults -> overlay -> scenario). +# It therefore only carries the accelerator/image bits that the guide +# scenarios do NOT hardcode, so a single overlay can be shared by every XPU +# guide instead of duplicating them in each scenario. Anything a guide +# scenario sets explicitly (customCommand, resources, model, ...) still wins +# over this overlay. +# +# Wire it into a spec via: +# values_file: +# path: {{ base_dir }}/config/templates/values/defaults.yaml +# overlays: +# - {{ base_dir }}/config/templates/values/overlays/xpu.yaml +# ============================================================================ + +# ---------------------------------------------------------------------------- +# ACCELERATOR - Intel GPU via classic device plugin (gpu.intel.com/xe) +# ---------------------------------------------------------------------------- +accelerator: + type: intel-xe + resource: "gpu.intel.com/xe" + count: 1 + # The llm-d-modelservice chart injects accelerator-specific env for + # intel-xe (VLLM_WORKER_MULTIPROC_METHOD=spawn). The benchmark macro already + # adds VLLM_WORKER_MULTIPROC_METHOD to the container env, so the chart default + # would produce a duplicate env key (rejected by server-side apply). Override + # the chart's intel-xe env list to empty. + env: + intel-xe: [] + +# ---------------------------------------------------------------------------- +# NODE AFFINITY - select Intel GPU nodes +# ---------------------------------------------------------------------------- +# defaults.yaml sets {decode,prefill}.acceleratorType to the NVIDIA GPU-product +# label (nvidia.com/gpu.product = NVIDIA-H100-80GB-HBM3). The modelservice +# template renders that into a REQUIRED nodeAffinity whenever `labelKey` is +# defined, so on an Intel node (which lacks that label) the decode/prefill pods +# stay Unschedulable. An empty dict in a scenario cannot clear it (deep-merge +# keeps the populated default), so we override it here with the generic Intel +# GPU node label emitted by NFD. +decode: + acceleratorType: + labelKey: intel.feature.node.kubernetes.io/gpu + labelValue: "true" +prefill: + acceleratorType: + labelKey: intel.feature.node.kubernetes.io/gpu + labelValue: "true" + +# ---------------------------------------------------------------------------- +# IMAGE - vLLM XPU build (overrides the CUDA default in defaults.yaml) +# ---------------------------------------------------------------------------- +images: + vllm: + repository: ghcr.io/llm-d/llm-d-xpu + tag: v0.7.0 + pullPolicy: IfNotPresent + sourceRepo: https://github.com/llm-d/llm-d diff --git a/llmdbenchmark/cli.py b/llmdbenchmark/cli.py index b979ba106..b07924143 100644 --- a/llmdbenchmark/cli.py +++ b/llmdbenchmark/cli.py @@ -112,6 +112,7 @@ def dispatch_cli(args: argparse.Namespace, logger: logging.Logger) -> None: defaults_file=specification_as_dict["values_file"]["path"], scenarios_file=specification_as_dict["scenario_file"]["path"], output_dir=config.plan_dir, + values_overlays=specification_as_dict["values_file"].get("overlays", []), version_resolver=version_resolver, cluster_resource_resolver=cluster_resource_resolver, cli_namespace=getattr(args, "namespace", None), @@ -1192,6 +1193,7 @@ def _render_plans_for_experiment(args, logger, setup_overrides=None): defaults_file=specification_as_dict["values_file"]["path"], scenarios_file=specification_as_dict["scenario_file"]["path"], output_dir=config.plan_dir, + values_overlays=specification_as_dict["values_file"].get("overlays", []), version_resolver=version_resolver, cluster_resource_resolver=cluster_resource_resolver, cli_namespace=getattr(args, "namespace", None), diff --git a/llmdbenchmark/parser/render_plans.py b/llmdbenchmark/parser/render_plans.py index 21f104dc3..74dd935d5 100644 --- a/llmdbenchmark/parser/render_plans.py +++ b/llmdbenchmark/parser/render_plans.py @@ -41,6 +41,7 @@ def __init__( defaults_file: Path, scenarios_file: Path, output_dir: Path, + values_overlays: list[Path] | None = None, logger=None, version_resolver=None, cluster_resource_resolver=None, @@ -57,6 +58,11 @@ def __init__( self.defaults_file = Path(defaults_file) self.scenarios_file = Path(scenarios_file) self.output_dir = Path(output_dir) + # Optional values overlays deep-merged on top of `defaults_file` + # before the scenario is applied. Used to layer hardware-specific + # base values (e.g. an Intel XPU overlay) without duplicating the + # entire defaults file. Each later overlay wins over earlier ones. + self.values_overlays: list[Path] = [Path(p) for p in (values_overlays or [])] self.version_resolver = version_resolver self.cluster_resource_resolver = cluster_resource_resolver self.cli_namespace = cli_namespace @@ -1314,6 +1320,8 @@ def eval(self) -> RenderResult: try: defaults = self._load_yaml(self.defaults_file) + for overlay in self.values_overlays: + defaults = self.deep_merge(defaults, self._load_yaml(overlay)) except Exception as e: msg = f"Failed to load defaults file: {e}" self.logger.log_error(msg)