Mobile Integration Tests (Vla) #48
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Mobile Integration Tests (Vla) | |
| # Migrated to .github/actions/run-mobile-integration-tests in QVAC-18168. | |
| # | |
| # VLA reuses the LLM (`llamacpp-embed`) Device Farm project + device pool | |
| # secrets on purpose — see the env-block comment below. Provision dedicated | |
| # `_VLA` secrets and rename here when VLA needs an isolated Device Farm | |
| # budget. | |
| # | |
| # See docs/integration-test-workflow-refactor.md. | |
| on: | |
| workflow_call: | |
| inputs: | |
| ref: | |
| description: "Git ref to checkout" | |
| type: string | |
| required: false | |
| repository: | |
| description: "Repository to checkout" | |
| type: string | |
| required: false | |
| package: | |
| description: "Full NPM package spec to test (optional for workflow_call)" | |
| type: string | |
| required: false | |
| # Manual on-demand path. A dispatched run targets ONE platform and the | |
| # device(s) chosen below — it never runs the full default pool. The | |
| # workflow_call path (benchmarks / weekend / on-merge) is unchanged. | |
| workflow_dispatch: | |
| inputs: | |
| platform: | |
| description: "Platform to run on (a manual run targets one platform)." | |
| type: choice | |
| required: true | |
| options: [Android, iOS] | |
| default: Android | |
| device: | |
| description: "Device from the common pool. Pick '(custom)' to type your own in 'devices_custom'. Valid names + how to find them: docs/ci/MOBILE-ON-DEMAND.md." | |
| type: choice | |
| required: false | |
| options: | |
| - "(custom)" | |
| - "Google Pixel 9" | |
| - "Google Pixel 8" | |
| - "Samsung Galaxy S25 Ultra" | |
| - "Apple iPhone 17" | |
| - "Apple iPhone 16 Pro" | |
| - "Apple iPhone 15" | |
| default: "(custom)" | |
| devices_custom: | |
| description: "Comma-separated device model(s), e.g. 'Pixel 9, Pixel 8'. Overrides the dropdown; use for new/uncommon devices or several at once. Must match a Device Farm model (list + how to check: docs/ci/MOBILE-ON-DEMAND.md)." | |
| type: string | |
| required: false | |
| default: "" | |
| device_model_operator: | |
| description: "Model match: EQUALS (exact fleet model only — default; dropdown values are exact fleet names) or CONTAINS (any model containing the value; use for shorthand like 'Pixel 9'). EQUALS avoids billing a different/slower variant." | |
| type: choice | |
| required: false | |
| options: [EQUALS, CONTAINS] | |
| default: EQUALS | |
| tests: | |
| description: "Optional test filter: a mocha --grep regex matched by test NAME, not file name (e.g. 'runAddonTest'; combine several with '|'). Empty = full mobile suite. Where to find valid names: docs/ci/MOBILE-ON-DEMAND.md." | |
| type: string | |
| required: false | |
| default: "" | |
| ref: | |
| description: "Git ref (branch/tag/SHA) to test. Leave blank to use the branch you dispatch from." | |
| type: string | |
| required: false | |
| default: "" | |
| package: | |
| description: "Full NPM package spec to test. Leave EMPTY (default) to test this branch's native prebuild artifact; set @qvac/vla-ggml@<ver> (published) or @tetherto/vla-ggml@<dev> (GPR) to force-install a specific build." | |
| type: string | |
| required: false | |
| default: "" | |
| env: | |
| NODE_VERSION: 'lts/*' | |
| ADDON_NAME: '@qvac/vla-ggml' | |
| PREBUILD_ARTIFACT_PREFIX: 'vla-' # Prefix for prebuild artifacts | |
| TEST_FRAMEWORK_REF: 'main' # Branch/tag of qvac-test-addon-mobile framework | |
| APP_BUNDLE_ID: 'io.tether.test.qvac' # Bundle ID for the test app (same for all addons) | |
| ADDON_DIR: 'packages/vla-ggml' | |
| # Note: VLA mobile tests intentionally share the Device Farm project + device | |
| # pools provisioned for `@qvac/llm-llamacpp-embed`, so the `_LLAMACPP_EMBED` | |
| # secret suffixes below are reused on purpose. Provision dedicated `_VLA` | |
| # secrets and rename here when VLA needs an isolated Device Farm budget. | |
| concurrency: | |
| # workflow_dispatch: at most one manual mobile run per (workflow, branch) at a | |
| # time — a fresh dispatch cancels the in-flight one so a re-run never stacks a | |
| # second Device Farm run. Different branches map to different groups, so the | |
| # same addon can still be tested on 2-3 branches in parallel. workflow_call | |
| # (benchmarks / weekend / on-merge) is keyed per run and never cancels — | |
| # inputs.platform is only ever set on a manual dispatch, and under workflow_call | |
| # github.* resolves to the CALLER, so run_id keeps each call self-contained. | |
| group: ${{ inputs.platform != '' && format('mobile-dispatch-{0}-{1}', github.workflow, github.ref) || format('mobile-call-{0}', github.run_id) }} | |
| cancel-in-progress: ${{ inputs.platform != '' }} | |
| jobs: | |
| # Defense-in-depth gate for workflow_dispatch. The repository's PR entry | |
| # point (on-pr-vla.yml) already runs authorize-pr before invoking this | |
| # workflow over workflow_call, so workflow_call runs are pre-authorized. | |
| # workflow_dispatch, however, lets anyone with the dispatch permission | |
| # supply an arbitrary `ref` — silence CodeQL's artifact-poisoning warning | |
| # by making the trust check explicit: refuse to run unless the dispatching | |
| # actor has write or higher on the repo. | |
| authorize: | |
| if: github.event_name != 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| permissions: {} | |
| steps: | |
| - run: echo "workflow_call run inherits authorize-pr from caller" | |
| authorize-dispatch: | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Verify dispatcher has write permission | |
| uses: scherermichael-oss/action-has-permission@136e061bfe093832d87f090dd768e14e27a740d3 # 1.0.6 | |
| with: | |
| required-permission: write | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Manual (workflow_dispatch) runs must name a device that exists on Device | |
| # Farm. This gate fails fast BEFORE any build or Device Farm run, so a typo | |
| # or an empty selection costs nothing. Skipped for workflow_call. | |
| validate-devices: | |
| name: Validate device selection | |
| if: inputs.platform != '' | |
| runs-on: ubuntu-latest | |
| environment: release | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout composite action source | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-mobile-integration-tests | |
| sparse-checkout-cone-mode: false | |
| # Validate the tests filter + shard fan-out against test-groups.json from | |
| # the SAME ref the build executes (inputs.ref || github.sha), not the | |
| # workflow ref, so a branch that renames/adds runners or shards can't pass | |
| # stale validation. Data only — nothing here is executed; the composite | |
| # action stays on the trusted workflow ref checked out above. | |
| - name: Checkout validation data (matches the built ref) | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || github.sha }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| sparse-checkout: | | |
| packages/vla-ggml/test/mobile/test-groups.json | |
| sparse-checkout-cone-mode: false | |
| path: validation-data | |
| persist-credentials: false | |
| - name: Validate requested devices | |
| uses: ./.github/actions/run-mobile-integration-tests/validate-devices | |
| with: | |
| platform: ${{ inputs.platform }} | |
| devices: ${{ inputs.devices_custom != '' && inputs.devices_custom || (inputs.device != '(custom)' && inputs.device || '') }} | |
| device-model-operator: ${{ inputs.device_model_operator }} | |
| tests: ${{ inputs.tests }} | |
| test-groups-path: validation-data/packages/vla-ggml/test/mobile/test-groups.json | |
| aws-role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} | |
| build-and-test: | |
| # workflow_call: validate-devices is skipped; its `skipped` result keeps | |
| # this job running exactly as before. | |
| # workflow_dispatch: a bad/empty device selection blocks this job. | |
| needs: [authorize, authorize-dispatch, validate-devices] | |
| if: | | |
| always() && | |
| (needs.authorize.result == 'success' || needs.authorize-dispatch.result == 'success') && | |
| (needs.validate-devices.result == 'success' || needs.validate-devices.result == 'skipped') | |
| name: Build ${{ matrix.platform }} and Run E2E Tests | |
| runs-on: ${{ matrix.runner }} | |
| environment: release | |
| timeout-minutes: 120 | |
| # No continue-on-error: a Device Farm test failure must fail this run on | |
| # every trigger (workflow_dispatch, workflow_call, schedule). Re-adding it | |
| # silently turns failed runs green. | |
| permissions: | |
| contents: read | |
| packages: read | |
| pull-requests: write | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| # On workflow_dispatch, run only the chosen platform (devices come from | |
| # the manual inputs). workflow_call runs both platforms as before. | |
| matrix: ${{ fromJSON(inputs.platform != '' && (inputs.platform == 'iOS' && '{"include":[{"platform":"iOS","os":"macos-14","runner":"macos-14"}]}' || '{"include":[{"platform":"Android","os":"ubuntu-22.04","runner":"qvac-ubuntu2204-x64"}]}') || '{"include":[{"platform":"Android","os":"ubuntu-22.04","runner":"qvac-ubuntu2204-x64"},{"platform":"iOS","os":"macos-14","runner":"macos-14"}]}') }} | |
| steps: | |
| - name: Validate Dispatch Inputs | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.package | |
| env: | |
| PKG: ${{ github.event.inputs.package }} | |
| run: | | |
| if [[ ! "$PKG" =~ ^@qvac/|^@tetherto/ ]]; then | |
| echo "::error::Invalid package scope. Only @qvac/* or @tetherto/* is allowed." | |
| exit 1 | |
| fi | |
| - name: Checkout composite action source | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-mobile-integration-tests | |
| sparse-checkout-cone-mode: false | |
| - name: Checkout addon repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || github.sha }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: addon | |
| fetch-depth: 0 | |
| - name: Validate mobile tests are up-to-date | |
| working-directory: addon/packages/vla-ggml | |
| run: npm run test:mobile:validate || true | |
| - name: Setup mobile test environment | |
| uses: ./.github/actions/run-mobile-integration-tests/setup | |
| with: | |
| platform: ${{ matrix.platform }} | |
| addon-workdir: 'packages/vla-ggml' | |
| addon-npm-name: '@qvac/vla-ggml' | |
| prebuild-artifact-prefix: 'vla-' | |
| pat-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Manual dispatch (inputs.platform set): default (empty `package`) tests | |
| # this branch's native prebuild; @qvac/vla-ggml@<ver> = published, @tetherto/vla-ggml@<dev> | |
| # = this branch's GPR build). workflow_call is untouched (artifact-first). | |
| package-version: ${{ inputs.platform != '' && inputs.package || '' }} | |
| force-npm-prebuild: ${{ (inputs.platform != '' && inputs.package != '') && 'true' || 'false' }} | |
| # OIDC session for the SmolVLA/GR00T S3 presign (the us-west-2 Device Farm | |
| # session is acquired separately in upload-to-devicefarm). Must run BEFORE | |
| # the build so the presigned URL / sha256 / size are baked into testAssets. | |
| # Presign against the US bucket; vla/* models are mirrored under | |
| # qvac_models_compiled/ by the EU->US sync job. | |
| - name: Configure AWS credentials for S3 presign | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: 7200 | |
| - name: Generate SmolVLA Model URL | |
| working-directory: addon/packages/vla-ggml | |
| env: | |
| AWS_REGION: us-west-2 | |
| MODEL_S3_BUCKET: tether-ai-dev-us | |
| run: | | |
| echo "🔑 Generating SmolVLA model URL for mobile tests..." | |
| chmod +x scripts/generate-smolvla-presigned-url.sh | |
| ./scripts/generate-smolvla-presigned-url.sh | |
| echo "✅ SmolVLA model URL generated and exported" | |
| - name: Generate GR00T Model URL | |
| working-directory: addon/packages/vla-ggml | |
| env: | |
| AWS_REGION: us-west-2 | |
| MODEL_S3_BUCKET: tether-ai-dev-us | |
| run: | | |
| echo "Generating GR00T model URL for mobile tests..." | |
| chmod +x scripts/generate-groot-presigned-url.sh | |
| ./scripts/generate-groot-presigned-url.sh | |
| echo "GR00T model URL generated and exported" | |
| - name: Bundle Model URLs in config files for APK to access | |
| working-directory: addon/packages/vla-ggml | |
| env: | |
| SMOLVLA_MODEL_URL: ${{ env.SMOLVLA_MODEL_URL }} | |
| SMOLVLA_MODEL_SIZE: ${{ env.SMOLVLA_MODEL_SIZE }} | |
| SMOLVLA_MODEL_SHA256: ${{ env.SMOLVLA_MODEL_SHA256 }} | |
| run: | | |
| echo "📝 Creating model URL config files for mobile app bundling..." | |
| # Presigned URL carries creds: mask it, then log only SET/NOTSET. | |
| # `${VAR:-NOTSET}` would expand the full URL into the log when set. | |
| if [ -n "${SMOLVLA_MODEL_URL:-}" ]; then | |
| echo "::add-mask::${SMOLVLA_MODEL_URL}" | |
| echo " SMOLVLA_MODEL_URL: SET" | |
| else | |
| echo " SMOLVLA_MODEL_URL: NOTSET" | |
| fi | |
| echo " SMOLVLA_MODEL_SIZE: ${SMOLVLA_MODEL_SIZE:-unknown}" | |
| echo " SMOLVLA_MODEL_SHA256: ${SMOLVLA_MODEL_SHA256:+SET}${SMOLVLA_MODEL_SHA256:-NOTSET}" | |
| mkdir -p test/mobile/testAssets | |
| if [ -n "${SMOLVLA_MODEL_URL:-}" ]; then | |
| jq -n \ | |
| --arg url "$SMOLVLA_MODEL_URL" \ | |
| --arg size "${SMOLVLA_MODEL_SIZE:-}" \ | |
| --arg sha256 "${SMOLVLA_MODEL_SHA256:-}" \ | |
| '{modelUrl: $url} | |
| + (if $size != "" then {sizeBytes: ($size | tonumber)} else {} end) | |
| + (if $sha256 != "" then {sha256: $sha256} else {} end)' \ | |
| > test/mobile/testAssets/smolvla-urls.json | |
| echo " ✅ SmolVLA URL bundled ($(wc -c < test/mobile/testAssets/smolvla-urls.json) bytes)" | |
| # Do not echo the URL itself (it's a presigned URL with creds). | |
| else | |
| echo " ⚠️ SmolVLA URL not available" | |
| exit 1 | |
| fi | |
| # pi05 mobile coverage is deferred pending a project-owned | |
| # CDN-fronted mirror; desktop pi05 e2e still runs against the | |
| # S3 oracle directly. The pi05 test is gated by `_skipMobilePi05` | |
| # in pi05.test.js so no pi05 URL/fixture bundling is needed here. | |
| # Copy the PyTorch reference fixtures into testAssets/ so the mobile | |
| # test app can load them via global.assetPaths and run the same | |
| # quality assertion the desktop integration test runs (max|delta| and | |
| # cosine similarity vs PyTorch). Kept here, gated on the always-present | |
| # SMOLVLA_MODEL_URL, so a GR00T S3 hiccup can't break fixture bundling. | |
| echo "" | |
| echo "Bundling integration test fixtures..." | |
| for f in pt_actions_libero_fixed.json \ | |
| pt_actions_libero_real.json \ | |
| libero_real_left.bin \ | |
| libero_real_right.bin; do | |
| src="test/integration/assets/$f" | |
| if [ -f "$src" ]; then | |
| cp "$src" "test/mobile/testAssets/$f" | |
| echo " $f ($(wc -c < test/mobile/testAssets/$f) bytes)" | |
| else | |
| echo " $f missing, mobile will skip the corresponding fixture" | |
| fi | |
| done | |
| - name: Bundle GR00T Model URL in config file for APK to access | |
| working-directory: addon/packages/vla-ggml | |
| env: | |
| GROOT_MODEL_URL: ${{ env.GROOT_MODEL_URL }} | |
| GROOT_MODEL_SIZE: ${{ env.GROOT_MODEL_SIZE }} | |
| GROOT_MODEL_SHA256: ${{ env.GROOT_MODEL_SHA256 }} | |
| run: | | |
| echo "Creating GR00T model URL config for mobile app bundling..." | |
| # Presigned URL carries creds: mask it, then log only SET/NOTSET. | |
| # `${VAR:-NOTSET}` would expand the full URL into the log when set. | |
| if [ -n "${GROOT_MODEL_URL:-}" ]; then | |
| echo "::add-mask::${GROOT_MODEL_URL}" | |
| echo " GROOT_MODEL_URL: SET" | |
| else | |
| echo " GROOT_MODEL_URL: NOTSET" | |
| fi | |
| echo " GROOT_MODEL_SIZE: ${GROOT_MODEL_SIZE:-unknown}" | |
| echo " GROOT_MODEL_SHA256: ${GROOT_MODEL_SHA256:+SET}${GROOT_MODEL_SHA256:-NOTSET}" | |
| mkdir -p test/mobile/testAssets | |
| if [ -n "${GROOT_MODEL_URL:-}" ]; then | |
| jq -n \ | |
| --arg url "$GROOT_MODEL_URL" \ | |
| --arg size "${GROOT_MODEL_SIZE:-}" \ | |
| --arg sha256 "${GROOT_MODEL_SHA256:-}" \ | |
| '{modelUrl: $url} | |
| + (if $size != "" then {sizeBytes: ($size | tonumber)} else {} end) | |
| + (if $sha256 != "" then {sha256: $sha256} else {} end)' \ | |
| > test/mobile/testAssets/groot-urls.json | |
| echo " GR00T URL bundled ($(wc -c < test/mobile/testAssets/groot-urls.json) bytes)" | |
| # Do not echo the URL itself (it's a presigned URL with creds). | |
| else | |
| echo " GR00T URL not available" | |
| exit 1 | |
| fi | |
| - name: Build mobile app | |
| id: build | |
| uses: ./.github/actions/run-mobile-integration-tests/build-mobile-app | |
| with: | |
| platform: ${{ matrix.platform }} | |
| addon-workdir: 'packages/vla-ggml' | |
| addon-npm-name: '@qvac/vla-ggml' | |
| ios-build-cert-base64: ${{ secrets.TEST_APP_APPLE_DISTRIBUTION_CERTIFICATE }} | |
| ios-build-cert-password: ${{ secrets.APPLE_P12_PASSWORD }} | |
| ios-keychain-password: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} | |
| ios-provisioning-profile-base64: ${{ secrets.TEST_APP_APPLE_PROVISIONING_PROFILE }} | |
| apple-team-id: ${{ secrets.APPLE_TEAM_ID }} | |
| # QVAC fix for the 20-minute default per-test ceiling that breaks | |
| # the 1.9 GB SmolVLA S3 download on Android. Paired with the | |
| # mocha-timeout-ms on upload-to-devicefarm below (70 min) so the | |
| # WDIO/Mocha session outlives any single test. | |
| android-per-test-timeout-minutes: '60' | |
| # Runs after the presigned URLs were bundled ({smolvla,groot}-urls.json), so | |
| # we bake them into a per-shard manifest; the host script reads each shard's | |
| # grep and pushes only that shard's multi-GB GGUF to a platform-local | |
| # staging dir. The phone copies it instead of pulling from the flaky | |
| # presigned-S3 link. | |
| # pi05 is deferred on mobile and stages nothing. | |
| - name: Generate model pre-stage commands (Android) | |
| id: prestage | |
| if: matrix.platform == 'Android' | |
| working-directory: addon/packages/vla-ggml | |
| run: | | |
| node scripts/generate-prestage-block.js android > "$RUNNER_TEMP/prestage-block.txt" | |
| { | |
| echo "block<<PRESTAGE_EOF" | |
| cat "$RUNNER_TEMP/prestage-block.txt" | |
| echo "PRESTAGE_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Generate model pre-stage commands (iOS) | |
| id: ios_prestage | |
| if: matrix.platform == 'iOS' | |
| working-directory: addon/packages/vla-ggml | |
| run: | | |
| node scripts/generate-prestage-block.js ios > "$RUNNER_TEMP/prestage-block.txt" | |
| { | |
| echo "block<<PRESTAGE_EOF" | |
| cat "$RUNNER_TEMP/prestage-block.txt" | |
| echo "PRESTAGE_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| # On a manual run with a test filter, override to a single grepped spec so | |
| # only those tests run (and the sharded fan-out collapses to one spec). | |
| # Empty filter / workflow_call keeps the auto-detected shard groups. | |
| - name: Resolve manual test filter | |
| id: manual_tests | |
| if: inputs.platform != '' && inputs.tests != '' | |
| shell: bash | |
| env: | |
| TESTS: ${{ inputs.tests }} | |
| run: | | |
| # Reject anything but mocha-safe grep characters so a manual `tests` | |
| # value cannot break out of the generated WDIO config on the Device | |
| # Farm host (defense-in-depth; the sink is also JSON-encoded). | |
| if ! printf '%s' "$TESTS" | grep -Eq '^[A-Za-z0-9_ |().*+-]+$'; then | |
| echo "::error::Invalid 'tests' filter '$TESTS'. Allowed: letters, digits, and _ space | ( ) . * + - (a mocha --grep on test NAMES, e.g. 'runAddonTest' or 'runFoo|runBar')." | |
| exit 1 | |
| fi | |
| MANUAL_GROUPS=$(jq -nc --arg g "$TESTS" '[{name:"manual", grep:$g}]') | |
| echo "groups=$MANUAL_GROUPS" >> "$GITHUB_OUTPUT" | |
| # VLA reuses the LLAMACPP_EMBED Device Farm project + pool secrets | |
| # (see env-block comment above). The composite auto-detects | |
| # test-groups.json under <addon-workdir>/test/mobile/; VLA ships one | |
| # (smolvla/groot), so the test specs come from it. | |
| - name: Upload to Device Farm | |
| id: upload | |
| uses: ./.github/actions/run-mobile-integration-tests/upload-to-devicefarm | |
| with: | |
| platform: ${{ matrix.platform }} | |
| addon-workdir: 'packages/vla-ggml' | |
| app-path: ${{ steps.build.outputs.app-path }} | |
| app-type: ${{ steps.build.outputs.app-type }} | |
| app-name: ${{ steps.build.outputs.app-name }} | |
| aws-role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} | |
| device-farm-project-arn: ${{ secrets.AWS_DEVICE_FARM_PROJECT_ARN_LLAMACPP_EMBED }} | |
| test-groups: ${{ steps.manual_tests.outputs.groups }} | |
| enables-perf: 'true' | |
| # 70-minute WDIO/Mocha session ceiling so the runner outlives | |
| # the per-test 60-minute ceiling enforced above. | |
| mocha-timeout-ms: '4200000' | |
| # Host pre-stages the running shard's GGUF before device tests. | |
| extra-pre-test-commands: ${{ matrix.platform == 'iOS' && steps.ios_prestage.outputs.block || steps.prestage.outputs.block }} | |
| - name: Schedule Device Farm test runs | |
| id: schedule | |
| uses: ./.github/actions/run-mobile-integration-tests/schedule-test-run | |
| with: | |
| platform: ${{ matrix.platform }} | |
| device-farm-project-arn: ${{ secrets.AWS_DEVICE_FARM_PROJECT_ARN_LLAMACPP_EMBED }} | |
| android-device-pool-arn: ${{ secrets.ANDROID_DEVICE_POOL_ARN_LLAMACPP_EMBED }} | |
| ios-device-pool-arn: ${{ secrets.IOS_DEVICE_POOL_ARN_LLAMACPP_EMBED }} | |
| app-upload-arn: ${{ steps.upload.outputs.app-upload-arn }} | |
| test-package-upload-arn: ${{ steps.upload.outputs.test-package-upload-arn }} | |
| test-specs: ${{ steps.upload.outputs.test-specs }} | |
| # workflow_dispatch: fan the chosen shard(s) across the chosen device(s). | |
| # workflow_call: default (dual-flagship input) keeps auto-sharded pool runs. | |
| scheduling-mode: ${{ inputs.platform != '' && 'manual-devices' || 'dual-flagship' }} | |
| device-models: ${{ inputs.platform != '' && (inputs.devices_custom != '' && inputs.devices_custom || (inputs.device != '(custom)' && inputs.device || '')) || '' }} | |
| device-model-operator: ${{ inputs.platform != '' && inputs.device_model_operator || '' }} | |
| - name: Monitor Device Farm test runs | |
| id: monitor | |
| uses: ./.github/actions/run-mobile-integration-tests/monitor-test-run | |
| with: | |
| run-arns: ${{ steps.schedule.outputs.run-arns }} | |
| run-count: ${{ steps.schedule.outputs.run-count }} | |
| - name: Cancel Device Farm runs on workflow cancellation | |
| if: cancelled() && steps.schedule.outputs.run-arns != '' && steps.schedule.outputs.run-arns != '[]' | |
| uses: ./.github/actions/run-mobile-integration-tests/cancel-device-farm-runs | |
| with: | |
| run-arns: ${{ steps.schedule.outputs.run-arns }} | |
| aws-role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} | |
| - name: Collect and upload Device Farm logs | |
| if: always() | |
| uses: ./.github/actions/run-mobile-integration-tests/collect-and-upload-logs | |
| with: | |
| platform: ${{ matrix.platform }} | |
| addon-npm-name: '@qvac/vla-ggml' | |
| enables-perf: 'true' | |
| aws-role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} | |
| run-arns: ${{ steps.schedule.outputs.run-arns }} | |
| # VLA's chunked perf-report markers land in bare_console.log inside | |
| # Customer_Artifacts.zip, so unzip-customer-artifacts=true is required. | |
| # merge=true unions per-device performance-report.json payloads | |
| # (Pixel 9 Pro + Galaxy S25 Ultra). The artifact name keeps the | |
| # `vla-perf-mobile-` prefix so any downstream combine-perf-reports | |
| # job still finds it. | |
| - name: Extract & upload addon perf report (VLA) | |
| if: always() && steps.schedule.outputs.run-count != '0' && steps.schedule.outputs.run-count != '' | |
| uses: ./.github/actions/run-mobile-integration-tests/extract-addon-perf | |
| with: | |
| platform: ${{ matrix.platform }} | |
| merge: 'true' | |
| unzip-customer-artifacts: 'true' | |
| step-summary-title: VLA Mobile Performance Report (${{ matrix.platform }}) | |
| artifact-name: vla-perf-mobile-${{ matrix.platform }}-${{ github.run_number }} | |
| - name: Comment results on PR | |
| if: always() | |
| uses: ./.github/actions/run-mobile-integration-tests/comment-on-pr | |
| with: | |
| addon-npm-name: '@qvac/vla-ggml' | |
| platform: ${{ matrix.platform }} | |
| test-result: ${{ steps.monitor.outputs.test-result }} | |
| test-total: ${{ steps.monitor.outputs.test-total }} | |
| test-passed: ${{ steps.monitor.outputs.test-passed }} | |
| test-failed: ${{ steps.monitor.outputs.test-failed }} | |
| test-skipped: ${{ steps.monitor.outputs.test-skipped }} | |
| user-test-passed: ${{ steps.monitor.outputs.user-test-passed }} | |
| user-test-failed: ${{ steps.monitor.outputs.user-test-failed }} |