fixup: add test coverage #492
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: deploy-stage | |
| concurrency: | |
| group: deploy-stage | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: write | |
| jobs: | |
| detect-changes: | |
| runs-on: ubuntu-latest | |
| env: | |
| IS_CI_AUTOMATION: "yes" | |
| outputs: | |
| src-changed: ${{ steps.check.outputs.src-changed }} | |
| keycloak-theme-changed: ${{ steps.check.outputs.keycloak-theme-changed }} | |
| iac-changed: ${{ steps.check.outputs.iac-changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: check | |
| with: | |
| filters: | | |
| src-changed: | |
| - 'assets/**' | |
| - 'static/**' | |
| - 'templates/**' | |
| - 'scripts/entry.sh' | |
| - 'Dockerfile' | |
| - 'manage.py' | |
| - 'MANIFEST.in' | |
| - 'package.json' | |
| - 'package-lock.json' | |
| - 'pyproject.toml' | |
| - 'README.md' | |
| - 'src/**' | |
| - 'uv.lock' | |
| - 'vite.config.mjs' | |
| keycloak-theme-changed: | |
| - 'keycloak/themes/**' | |
| - 'Dockerfile.keycloak' | |
| iac-changed: | |
| - 'pulumi/**' | |
| - '.github/**' | |
| # Build whichever image(s) changed and apply them to the stage stack in a | |
| # SINGLE `pulumi up`. Pulumi takes a stack-wide lock per update, so running | |
| # accounts and keycloak deploys as separate parallel jobs caused | |
| # "[409] Conflict: Another update is currently in progress." See issue #884. | |
| deploy: | |
| needs: detect-changes | |
| if: >- | |
| needs.detect-changes.outputs.src-changed == 'true' || | |
| needs.detect-changes.outputs.keycloak-theme-changed == 'true' || | |
| needs.detect-changes.outputs.iac-changed == 'true' | |
| environment: | |
| name: staging | |
| deployment: true | |
| runs-on: ubuntu-latest | |
| env: | |
| IS_CI_AUTOMATION: "yes" | |
| PULUMI_DIR: "pulumi" | |
| # Whether the accounts image was (re)built this run; gates the | |
| # accounts config update, its pulumi targets, and the release artifacts. | |
| BUILD_ACCOUNTS: ${{ needs.detect-changes.outputs.src-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} | |
| # Whether the keycloak image was (re)built this run. | |
| BUILD_KEYCLOAK: ${{ needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} | |
| outputs: | |
| # Non-empty only when the accounts image was built; used to gate create-release. | |
| accounts-image: ${{ steps.build-accounts.outputs.accounts-image }} | |
| steps: | |
| # Preparation for future steps | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: eu-central-1 | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| mask-password: "true" | |
| - name: Set up Python ${{ vars.PYTHON_VERSION}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ vars.PYTHON_VERSION}} | |
| - name: Set up virtual environment | |
| shell: bash | |
| run: | | |
| python -m pip install virtualenv | |
| cd $PULUMI_DIR | |
| virtualenv ./venv | |
| - name: Set up Pulumi environment | |
| id: pulumi-env | |
| shell: bash | |
| run: | | |
| cd $PULUMI_DIR | |
| source ./venv/bin/activate | |
| curl -fsSL https://get.pulumi.com | sh | |
| pip install -Ur requirements.txt | |
| # Both the accounts and keycloak images are built multi-arch via Buildx + | |
| # QEMU (linux/amd64 for ECS Fargate today, linux/arm64 for the Thunderbird | |
| # Pro EKS clusters on Graviton). Each <tag> is a manifest list, so every | |
| # runtime pulls its matching variant. Set these up whenever either image | |
| # is (re)built; they must run before the build steps below. | |
| - name: Set up QEMU | |
| if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true' | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| # Produce the accounts container image (only when accounts/IaC changed). | |
| - name: Build, tag, and push accounts image to Amazon ECR | |
| id: build-accounts | |
| if: env.BUILD_ACCOUNTS == 'true' | |
| env: | |
| ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:${{ github.sha }}" | |
| run: | | |
| # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the | |
| # manifest list to ECR. buildx --push publishes directly (no docker push). | |
| docker buildx build -t $ECR_TAG \ | |
| --platform linux/amd64,linux/arm64 --push . | |
| echo "accounts-image=$ECR_TAG" >> $GITHUB_OUTPUT | |
| # Produce the keycloak container image (only when keycloak/IaC changed). | |
| - name: Build, tag, and push keycloak image to Amazon ECR | |
| id: build-keycloak | |
| if: env.BUILD_KEYCLOAK == 'true' | |
| env: | |
| ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:keycloak-${{ github.sha }}" | |
| run: | | |
| # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the | |
| # manifest list to ECR. buildx --push publishes directly (no docker push). | |
| docker buildx build -f Dockerfile.keycloak -t $ECR_TAG \ | |
| --platform linux/amd64,linux/arm64 --push . | |
| echo "keycloak-image=$ECR_TAG" >> $GITHUB_OUTPUT | |
| - name: Get version from pyproject.toml | |
| id: read-version | |
| uses: SebRollen/toml-action@v1.2.0 | |
| with: | |
| file: 'pyproject.toml' | |
| field: 'project.version' | |
| # Publish the keycloak deployment artifact (when keycloak was rebuilt). | |
| # Production keycloak is deployed manually from this artifact; see | |
| # docs/keycloak/how-to-deploy-to-prod.rst. The inner file must be named | |
| # deployment.json, and this must run BEFORE the accounts step below | |
| # overwrites deployment.json with the accounts tag. | |
| - name: Produce the keycloak deployment artifact | |
| id: create-keycloak-artifact | |
| if: env.BUILD_KEYCLOAK == 'true' | |
| run: | | |
| echo '{"ecr_tag": "${{ steps.build-keycloak.outputs.keycloak-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json | |
| - name: Archive the keycloak deployment artifact | |
| if: env.BUILD_KEYCLOAK == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: deployment-keycloak | |
| path: deployment.json | |
| # The deployment artifact + release tooling track the accounts image only, | |
| # so these steps only run when the accounts image was (re)built. | |
| - name: Produce an artifact containing the tag and version | |
| id: create-artifact | |
| if: env.BUILD_ACCOUNTS == 'true' | |
| run: | | |
| echo '{"ecr_tag": "${{ steps.build-accounts.outputs.accounts-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json | |
| - name: Archive the deployment artifact | |
| id: tag-archive | |
| if: env.BUILD_ACCOUNTS == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: deployment | |
| path: deployment.json | |
| - name: Package Pulumi code | |
| if: env.BUILD_ACCOUNTS == 'true' | |
| shell: bash | |
| run: | | |
| # Exclude the venv: it was built on THIS runner with bin/python as an | |
| # absolute symlink into /opt/hostedtoolcache/Python/<patch>/..., so it | |
| # isn't portable to the prod runner (different cached patch version => | |
| # dangling symlink => pulumi fails on venv/bin/python). release.yml | |
| # rebuilds the venv from requirements.txt on the prod runner. | |
| tar --exclude='pulumi/venv' -cvjf pulumi.tbz pulumi/ | |
| - name: Archive the Pulumi artifact | |
| id: iac-archive | |
| if: env.BUILD_ACCOUNTS == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pulumi | |
| path: pulumi.tbz | |
| # Deploy to stage in a single Pulumi update so we only ever hold one | |
| # stack lock at a time. The image config keys and the --target list are | |
| # assembled dynamically from whichever image(s) were built above. | |
| - name: Deploy new image(s) to stage | |
| shell: bash | |
| env: | |
| PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} | |
| ACCOUNTS_IMAGE: ${{ steps.build-accounts.outputs.accounts-image }} | |
| KEYCLOAK_IMAGE: ${{ steps.build-keycloak.outputs.keycloak-image }} | |
| run: | | |
| # Update the PATH to include the right version of Pulumi; this is non-trivial or impossible | |
| # to do with the GHA workflow "env" settings above. | |
| export PATH="/home/runner/.pulumi/bin:$PATH" | |
| export PULUMI_CONFIG_PASSPHRASE='${{ secrets.PULUMI_PASSPHRASE }}' | |
| cd $PULUMI_DIR | |
| TARGETS=() | |
| # Merge each rebuilt image tag into the stage config and queue its targets. | |
| if [ -n "$ACCOUNTS_IMAGE" ]; then | |
| echo ".accounts_image: &ACCOUNTS_IMAGE $ACCOUNTS_IMAGE" > accounts-image.yaml | |
| yq -i '. *= load("accounts-image.yaml")' config.stage.yaml | |
| TARGETS+=(--target 'urn:pulumi:stage::accounts::tb:fargate:FargateClusterWithLogging$aws:ecs/taskDefinition:TaskDefinition::accounts-stage-fargate-accounts-taskdef') | |
| TARGETS+=(--target 'urn:pulumi:stage::accounts::aws:ecs/taskDefinition:TaskDefinition::accounts-stage-afc-accounts-taskdef-celery-stage') | |
| TARGETS+=(--target 'urn:pulumi:stage::accounts::aws:ecs/taskDefinition:TaskDefinition::accounts-stage-afc-accounts-taskdef-flower-stage') | |
| fi | |
| if [ -n "$KEYCLOAK_IMAGE" ]; then | |
| echo ".keycloak_image: &KEYCLOAK_IMAGE $KEYCLOAK_IMAGE" > keycloak-image.yaml | |
| yq -i '. *= load("keycloak-image.yaml")' config.stage.yaml | |
| TARGETS+=(--target 'urn:pulumi:stage::accounts::tb:fargate:FargateClusterWithLogging$aws:ecs/taskDefinition:TaskDefinition::accounts-stage-fargate-keycloak-taskdef') | |
| fi | |
| if [ ${#TARGETS[@]} -eq 0 ]; then | |
| echo "No images were built; nothing to deploy." | |
| exit 0 | |
| fi | |
| source ./venv/bin/activate | |
| pulumi login | |
| pulumi stack select thunderbird/stage | |
| pulumi up -y --diff "${TARGETS[@]}" --target-dependents | |
| create-release: | |
| needs: deploy | |
| # Releases track the accounts image, so only create one when accounts deployed. | |
| if: needs.deploy.result == 'success' && needs.deploy.outputs.accounts-image != '' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download deployment data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: | |
| deployment # Should pull "deployment.json" | |
| - name: Download Pulumi package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: | |
| pulumi # Should pull "pulumi.tbz" | |
| - name: Create release tag | |
| id: create-release-tag | |
| run: echo "tag_name=r-$(printf %04d $GITHUB_RUN_NUMBER)" >> $GITHUB_OUTPUT | |
| - name: Create draft release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| body: | | |
| ## Info | |
| Commit ${{ github.sha }} was deployed to `stage`. [See code diff](${{ github.event.compare }}). | |
| It was initialized by [${{ github.event.sender.login }}](${{ github.event.sender.html_url }}). | |
| ## How to Promote? | |
| In order to promote this to prod, edit the draft and press **"Publish release"**. | |
| draft: true | |
| fail_on_unmatched_files: true | |
| files: | | |
| deployment.json | |
| pulumi.tbz | |
| name: Release ${{ steps.create-release-tag.outputs.tag_name }} | |
| tag_name: ${{ steps.create-release-tag.outputs.tag_name }} | |
| e2e-tests-browserstack-stage: | |
| name: e2e-tests-browserstack-stage | |
| needs: | |
| - deploy | |
| if: always() && needs.deploy.result == 'success' | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: staging | |
| deployment: false | |
| env: | |
| ACCTS_OIDC_EMAIL: ${{ secrets.E2E_ACCTS_OIDC_EMAIL }} | |
| ACCTS_OIDC_PWORD: ${{ secrets.E2E_ACCTS_OIDC_PWORD }} | |
| ACCTS_OIDC_RECOVERY_EMAIL: ${{ secrets.E2E_ACCTS_OIDC_RECOVERY_EMAIL }} | |
| PRIMARY_THUNDERMAIL_EMAIL: ${{ secrets.E2E_PRIMARY_THUNDERMAIL_EMAIL }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: 'npm' | |
| cache-dependency-path: 'test/e2e/package-lock.json' | |
| - name: Install dependencies | |
| run: | | |
| cd ./test/e2e | |
| npm install | |
| - name: BrowserStack Env Setup | |
| uses: browserstack/github-actions/setup-env@master | |
| with: | |
| username: ${{ secrets.BROWSERSTACK_USERNAME }} | |
| access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} | |
| project-name: 'Thunderbird Accounts' | |
| build-name: 'TB Accounts E2E Tests: BUILD_INFO' | |
| - name: Run E2E Tests on stage via Browserstack | |
| run: | | |
| cd ./test/e2e | |
| cp .env.stage.example .env | |
| npm run e2e-test-browserstack-desktop-firefox |