test(e2e): fix the restore-flow races and retry-worker cascade #1794
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Run the E2E tests on a local stack that is using Backblaze for backend file storage | |
| name: e2e-bucket-tests | |
| concurrency: | |
| group: e2e-bucket-tests-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| - "!main" | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| permissions: | |
| packages: read | |
| contents: read | |
| jobs: | |
| detect-changes: | |
| runs-on: ubuntu-latest | |
| env: | |
| IS_CI_AUTOMATION: "yes" | |
| outputs: | |
| backend-changed: ${{ steps.check.outputs.backend-changed }} | |
| iac-changed: ${{ steps.check.outputs.iac-changed }} | |
| frontend-changed: ${{ steps.check.outputs.frontend-changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: check | |
| with: | |
| filters: | | |
| backend-changed: | |
| - 'packages/send/backend/**' | |
| iac-changed: | |
| - 'packages/send/pulumi/**' | |
| - '.github/workflows/validate.yml' | |
| frontend-changed: | |
| - 'packages/send/frontend/**' | |
| build-e2e-images: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.backend-changed == 'true' || needs.detect-changes.outputs.frontend-changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| contents: read | |
| env: | |
| # -b2 suffix distinguishes tags from the sibling e2e workflow, which uses -minio | |
| BACKEND_IMAGE: ghcr.io/${{ github.repository }}/e2e-backend:${{ github.sha }}-b2 | |
| FRONTEND_IMAGE: ghcr.io/${{ github.repository }}/e2e-frontend:${{ github.sha }}-b2 | |
| PROXY_IMAGE: ghcr.io/${{ github.repository }}/e2e-reverse-proxy:${{ github.sha }}-b2 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prepare backend build context | |
| run: | | |
| rsync -a \ | |
| --exclude=node_modules \ | |
| --exclude=.docker-build \ | |
| --exclude=.pnpm-store \ | |
| packages/send/backend/ \ | |
| packages/send/backend/.docker-build/ | |
| cp pnpm-lock.yaml packages/send/backend/.docker-build/ | |
| echo "node_modules/" > packages/send/backend/.docker-build/.dockerignore | |
| - name: Build backend image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/send/backend/.docker-build | |
| file: ./packages/send/backend/.docker-build/Dockerfile | |
| build-args: BUILD_ENV=production | |
| push: true | |
| tags: ${{ env.BACKEND_IMAGE }} | |
| cache-from: type=gha,scope=e2e-backend-b2 | |
| cache-to: type=gha,mode=max,scope=e2e-backend-b2 | |
| - name: Build reverse-proxy image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/send/backend/tls-dev-proxy | |
| push: true | |
| tags: ${{ env.PROXY_IMAGE }} | |
| cache-from: type=gha,scope=e2e-reverse-proxy-b2 | |
| cache-to: type=gha,mode=max,scope=e2e-reverse-proxy-b2 | |
| - name: Build frontend image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./ | |
| file: ./dockerfiles/Dockerfile-frontend | |
| build-args: BUILD_ENV=production | |
| push: true | |
| tags: ${{ env.FRONTEND_IMAGE }} | |
| cache-from: type=gha,scope=e2e-frontend-b2 | |
| cache-to: type=gha,mode=max,scope=e2e-frontend-b2 | |
| end-to-end-bucket-tests: | |
| needs: [detect-changes, build-e2e-images] | |
| if: needs.detect-changes.outputs.backend-changed == 'true' || needs.detect-changes.outputs.frontend-changed == 'true' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/${{ github.repository }}/cached-devcontainer:latest | |
| options: --shm-size=2gb --security-opt seccomp=unconfined | |
| env: | |
| IS_CI_AUTOMATION: "yes" | |
| BACKEND_IMAGE: ghcr.io/${{ github.repository }}/e2e-backend:${{ github.sha }}-b2 | |
| FRONTEND_IMAGE: ghcr.io/${{ github.repository }}/e2e-frontend:${{ github.sha }}-b2 | |
| PROXY_IMAGE: ghcr.io/${{ github.repository }}/e2e-reverse-proxy:${{ github.sha }}-b2 | |
| PLAYWRIGHT_BROWSERS_PATH: /github/home/.cache/ms-playwright | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up environments and install dependencies | |
| id: setup-and-install-dependencies | |
| shell: bash | |
| # Through `env:` rather than inline `${{ }}`: a secret containing a | |
| # quote would otherwise break -- or escape -- the shell command. | |
| env: | |
| B2_REGION: ${{ vars.TEST_B2_REGION }} | |
| B2_ENDPOINT: ${{ vars.TEST_B2_ENDPOINT }} | |
| B2_BUCKET_NAME: ${{ vars.TEST_B2_BUCKET_NAME }} | |
| B2_APPLICATION_KEY: ${{ secrets.TEST_B2_APPLICATION_KEY }} | |
| B2_APPLICATION_KEY_ID: ${{ secrets.TEST_B2_APPLICATION_KEY_ID }} | |
| run: | | |
| pnpm install --filter @thunderbird/tbpro-add-on && lerna run bootstrap | |
| pnpm i --filter "send-*" | |
| cd packages/send | |
| # Setup local environment (use public login) | |
| echo Y | pnpm setup:local | |
| cd backend | |
| # Add bucket vars. `|` as the delimiter, because an endpoint contains | |
| # "/", and `.*` on the left, because .env.sample ships a value for some | |
| # of these -- `^KEY=$` silently matched nothing and left them empty. | |
| sed -i "s|^STORAGE_BACKEND=.*$|STORAGE_BACKEND=b2|" .env | |
| for key in B2_REGION B2_ENDPOINT B2_BUCKET_NAME B2_APPLICATION_KEY B2_APPLICATION_KEY_ID; do | |
| sed -i "s|^$key=.*$|$key=${!key}|" .env | |
| done | |
| # This job exists to exercise Backblaze. There is no longer a | |
| # filesystem backend to fall back to -- a wrong value throws at boot -- | |
| # but a value of `s3` would still leave this a second copy of | |
| # end-to-end-tests, running against MinIO. | |
| grep -qx "STORAGE_BACKEND=b2" .env || { echo "::error::STORAGE_BACKEND is not b2"; exit 1; } | |
| for key in B2_ENDPOINT B2_APPLICATION_KEY B2_APPLICATION_KEY_ID B2_BUCKET_NAME; do | |
| grep -q "^$key=." .env || { echo "::error::$key is empty in .env"; exit 1; } | |
| done | |
| cd ../frontend | |
| # We set the split size to 1MB to make sure the uploads are split | |
| sed -i "s/^VITE_SPLIT_SIZE_IN_MB=100$/VITE_SPLIT_SIZE_IN_MB=1/" .env | |
| - name: Add env vars for backend | |
| shell: bash | |
| run: | | |
| cd packages/send/backend | |
| { | |
| echo "OIDC_CLIENT_ID=${{ secrets.OIDC_CLIENT_ID }}" | |
| echo "OIDC_CLIENT_SECRET=${{ secrets.OIDC_CLIENT_SECRET }}" | |
| echo "OIDC_TOKEN_INTROSPECTION_URL=${{ secrets.OIDC_TOKEN_INTROSPECTION_URL }}" | |
| } >> .env | |
| - name: Add env vars for frontend | |
| shell: bash | |
| run: | | |
| cd packages/send/frontend | |
| echo "VITE_OIDC_CLIENT_ID=${{ secrets.VITE_OIDC_CLIENT_ID }}" >> .env | |
| echo "VITE_OIDC_ROOT_URL=${{ secrets.VITE_OIDC_ROOT_URL }}" >> .env | |
| - name: Restore Playwright browsers cache | |
| uses: actions/cache/restore@v4 | |
| id: playwright-cache | |
| with: | |
| path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} | |
| key: playwright-${{ runner.os }}-${{ hashFiles('packages/send/e2e/package.json') }} | |
| - name: Restore apt packages cache | |
| uses: actions/cache/restore@v4 | |
| id: apt-cache | |
| with: | |
| path: /var/cache/apt/archives | |
| key: apt-playwright-${{ runner.os }}-${{ hashFiles('packages/send/e2e/package.json') }} | |
| restore-keys: | | |
| apt-playwright-${{ runner.os }}- | |
| - name: Install Playwright system dependencies | |
| shell: bash | |
| run: | | |
| rm -f /etc/apt/apt.conf.d/docker-clean | |
| pnpm --filter send-suite-e2e exec playwright install-deps | |
| # Pin the Node version used to install/run Playwright (see packages/send/e2e/.tool-versions). | |
| # The container's default Node (24.x) hits a stream regression (nodejs/node#63487) that | |
| # hangs `playwright install` during zip extraction; Node 22 LTS avoids it. We keep | |
| # @playwright/test at 1.59.1 for BrowserStack Automate compatibility. | |
| - name: Install pinned Node via mise | |
| working-directory: packages/send/e2e | |
| shell: bash | |
| run: | | |
| mise install | |
| mise exec -- node --version | |
| - name: Install Playwright browsers | |
| id: install-playwright-browsers | |
| if: steps.playwright-cache.outputs.cache-hit != 'true' | |
| timeout-minutes: 10 | |
| working-directory: packages/send/e2e | |
| shell: bash | |
| run: mise exec -- pnpm exec playwright install | |
| - name: Run stack and test | |
| id: run-stack-and-test | |
| timeout-minutes: 20 | |
| shell: bash | |
| run: | | |
| # Create .env file with secrets (for end to end run only) | |
| echo "TBPRO_USERNAME=${{ secrets.TBPRO_USERNAME }}" > .env | |
| echo "TBPRO_PASSWORD=${{ secrets.TBPRO_PASSWORD }}" >> .env | |
| cp .env packages/send/e2e | |
| mv .env packages/send/e2e/tests/desktop/dev/ | |
| cd packages/send/e2e | |
| mise exec -- pnpm run test:e2e:ci | |
| # Only save when the install step actually succeeded. A cancelled or failed | |
| # install (e.g. the Node 24 extract-zip hang) would otherwise save a partial | |
| # cache that poisons future runs with "Executable doesn't exist". | |
| - name: Save Playwright browsers cache | |
| if: always() && steps.playwright-cache.outputs.cache-hit != 'true' && steps.install-playwright-browsers.outcome == 'success' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} | |
| key: playwright-${{ runner.os }}-${{ hashFiles('packages/send/e2e/package.json') }} | |
| - name: Save apt packages cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: /var/cache/apt/archives | |
| key: ${{ steps.apt-cache.outputs.cache-primary-key }} | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: playwright-report | |
| path: packages/send/e2e/playwright-report/ | |
| retention-days: 30 |