llm-d-modelservice/examples/output-dra.yaml at eda8b883548d443b41613e3fdd6207192468c145 · zdtsw/llm-d-modelservice · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# generated by generate-example-output.sh
---
# Source: llm-d-modelservice/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: dra-llm-d-modelservice
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.3
    app.kubernetes.io/version: "v0.3.0"
    app.kubernetes.io/managed-by: Helm
---
# Source: llm-d-modelservice/templates/decode-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: dra-llm-d-modelservice-decode
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.3
    app.kubernetes.io/version: "v0.3.0"
    app.kubernetes.io/managed-by: Helm
spec:
  replicas: 1
  selector:
    matchLabels:
      llm-d.ai/inferenceServing: "true"
      llm-d.ai/model: random_model
      llm-d.ai/role: decode
  template:
    metadata:
      labels:
        llm-d.ai/inferenceServing: "true"
        llm-d.ai/model: random_model
        llm-d.ai/role: decode
    spec:

      serviceAccountName: dra-llm-d-modelservice

      volumes:
        - emptyDir: {}
          name: metrics-volume

        - name: model-storage
          persistentVolumeClaim:
            claimName: model-pvc
            readOnly: true
      resourceClaims:
        - name: intel-gaudi-decode-claim
          resourceClaimTemplateName: intel-gaudi-claim-template-decode
        - name: llm-d-existing-claim-sample
          resourceClaimTemplateName: llm-d-existing-claim-sample
      containers:
        - name: vllm
          image: intel/vllm:0.10.0-xpu

          command: ["vllm", "serve"]
          args:
            - meta-llama/Llama-3.1-8B-Instruct
            - --port
            - "8000"
            - --tensor-parallel-size
            - "2"
            - --served-model-name
            - "random/model"


            - --dtype=float16
            - --block-size=64
            - --max-num-seqs=64
            - --max-model-len=2048
            - --max-num-batched-token=4096
            - --disable-sliding-window
            - --gpu-memory-util=0.9
            - --quantization=fp8
          env:
          - name: OMPI_MCA_btl_vader_single_copy_mechanism
            value: none
          - name: PT_HPU_ENABLE_LAZY_COLLECTIVES
            value: "true"
          - name: VLLM_SKIP_WARMUP
            value: "true"
          - name: DP_SIZE
            value: "1"
          - name: TP_SIZE
            value: "2"
          - name: DP_SIZE_LOCAL
            value: "1"

          - name: HF_HUB_CACHE
            value: /model-cache/
          - name: HF_TOKEN
            valueFrom:
              secretKeyRef:
                name: llm-d-hf-token
                key: HF_TOKEN


          resources:
            limits:
              cpu: "4"
              memory: 16Gi
            requests:
              cpu: "2"
              memory: 8Gi
            claims:
              - name: intel-gaudi-decode-claim
              - name: llm-d-existing-claim-sample

          volumeMounts:
            - name: model-storage
              mountPath: /model-cache
---
# Source: llm-d-modelservice/templates/resource-claim-template.yaml
apiVersion: resource.k8s.io/v1
kind: ResourceClaimTemplate
metadata:
  name: intel-gaudi-claim-template-decode
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.3
    app.kubernetes.io/version: "v0.3.0"
    app.kubernetes.io/managed-by: Helm
    llm-d.ai/role: decode
spec:
  spec:
    devices:
      requests:
      - name: intel-gaudi
        exactly:
          deviceClassName: gaudi.intel.com
          count: 2