feat(epp): add latency-observation scorer and producer #1161
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - Coordinator Test | |
| # Runs coordinator unit tests and e2e tests. Path filter excludes paths not | |
| # exercised by the coordinator. The e2e-images and e2e-tests matrix jobs are | |
| # each followed by a *-status job that aggregates the matrix result, since | |
| # GitHub required status checks cannot reference individual matrix legs. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-*' | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| actions: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| src: ${{ steps.filter.outputs.src }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v7 | |
| - uses: dorny/paths-filter@v4 | |
| id: filter | |
| with: | |
| filters: | | |
| src: | |
| - '**/*.go' | |
| - Dockerfile.* | |
| - Makefile* | |
| - go.mod | |
| - go.sum | |
| - scripts/** | |
| - hack/** | |
| - test/** | |
| - config/coordinator/** | |
| - deploy/** | |
| - .github/actions/docker-build-and-push/action.yml | |
| - .github/actions/e2e-runner-setup/action.yml | |
| - .github/workflows/ci-coordinator.yaml | |
| - '!Dockerfile.sidecar' | |
| - '!cmd/pd-sidecar/**' | |
| - '!pkg/sidecar/**' | |
| - '!test/sidecar/**' | |
| - '!pkg/generator/**' | |
| - '!test/e2e/**' | |
| - '!test/integration/**' | |
| - '!test/profiling/**' | |
| - '!Makefile' | |
| unit-tests: | |
| needs: check-changes | |
| if: ${{ needs.check-changes.outputs.src == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v7 | |
| - name: Create Go cache dirs | |
| id: go-cache | |
| run: | | |
| mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache" | |
| echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT" | |
| echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go modules and build cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: | | |
| ${{ steps.go-cache.outputs.mod }} | |
| ${{ steps.go-cache.outputs.build }} | |
| key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }} | |
| restore-keys: | | |
| go-cache-${{ runner.os }}- | |
| - name: Run coordinator unit tests | |
| shell: bash | |
| env: | |
| GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }} | |
| GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }} | |
| run: make -f Makefile.coord.mk test-unit | |
| e2e-images: | |
| needs: check-changes | |
| if: ${{ needs.check-changes.outputs.src == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - name: builder | |
| source: build | |
| docker-file: Dockerfile.builder | |
| image-name: llm-d-builder | |
| archive: llm-d-builder | |
| - name: coordinator | |
| source: build | |
| docker-file: Dockerfile.coordinator | |
| image-name: llm-d-coordinator | |
| archive: llm-d-coordinator | |
| - name: epp | |
| source: build | |
| docker-file: Dockerfile.epp | |
| image-name: llm-d-router-endpoint-picker | |
| archive: llm-d-router-endpoint-picker | |
| - name: simulator | |
| source: pull | |
| image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 | |
| archive: llm-d-inference-sim | |
| name: e2e-images (${{ matrix.image.name }}) | |
| steps: | |
| - name: Checkout source | |
| if: ${{ matrix.image.source == 'build' }} | |
| uses: actions/checkout@v7 | |
| - name: Build ${{ matrix.image.name }} image | |
| if: ${{ matrix.image.source == 'build' }} | |
| uses: ./.github/actions/docker-build-and-push | |
| with: | |
| docker-file: ${{ matrix.image.docker-file }} | |
| image-name: ${{ matrix.image.image-name }} | |
| tag: dev | |
| registry: ghcr.io/llm-d | |
| push: 'false' | |
| buildx-outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.image.archive }}.tar | |
| commit-sha: ${{ github.sha }} | |
| build-ref: ${{ github.head_ref || github.ref_name }} | |
| - name: Pull ${{ matrix.image.name }} image | |
| if: ${{ matrix.image.source == 'pull' }} | |
| run: | | |
| docker pull --platform linux/amd64 "${{ matrix.image.image }}" | |
| docker save "${{ matrix.image.image }}" -o "${{ runner.temp }}/${{ matrix.image.archive }}.tar" | |
| - name: Upload ${{ matrix.image.name }} image | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: e2e-coord-image-${{ matrix.image.name }} | |
| path: ${{ runner.temp }}/${{ matrix.image.archive }}.tar | |
| retention-days: 1 | |
| compression-level: 0 | |
| e2e-images-status: | |
| needs: e2e-images | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check e2e-images result | |
| run: | | |
| result="${{ needs.e2e-images.result }}" | |
| if [[ "$result" != "success" && "$result" != "skipped" ]]; then | |
| echo "::error::e2e-images failed for at least one matrix image" | |
| exit 1 | |
| fi | |
| e2e-tests: | |
| needs: | |
| - check-changes | |
| - e2e-images | |
| if: ${{ needs.check-changes.outputs.src == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - topology: single-EPP | |
| target: test-e2e-coordinator-run | |
| - topology: 3-EPP | |
| target: test-e2e-coordinator-3epp-run | |
| name: e2e-tests (${{ matrix.topology }}) | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v7 | |
| - name: Set up e2e runner | |
| id: e2e-setup | |
| uses: ./.github/actions/e2e-runner-setup | |
| - name: Download builder image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: e2e-coord-image-builder | |
| path: ${{ runner.temp }}/e2e-images | |
| - name: Download coordinator image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: e2e-coord-image-coordinator | |
| path: ${{ runner.temp }}/e2e-images | |
| - name: Download EPP image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: e2e-coord-image-epp | |
| path: ${{ runner.temp }}/e2e-images | |
| - name: Download simulator image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: e2e-coord-image-simulator | |
| path: ${{ runner.temp }}/e2e-images | |
| - name: Load e2e images | |
| shell: bash | |
| run: | | |
| for image in "${RUNNER_TEMP}"/e2e-images/*.tar; do | |
| docker load --input "$image" | |
| done | |
| - name: Run coordinator e2e tests | |
| shell: bash | |
| env: | |
| GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }} | |
| GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }} | |
| run: make -f Makefile.coord.mk ${{ matrix.target }} | |
| e2e-tests-status: | |
| needs: e2e-tests | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check e2e-tests result | |
| run: | | |
| result="${{ needs.e2e-tests.result }}" | |
| if [[ "$result" != "success" && "$result" != "skipped" ]]; then | |
| echo "::error::e2e-tests failed for at least one topology" | |
| exit 1 | |
| fi |