From 0a25411c82b1b38e8b86cbec5a4650a43238943c Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 10:15:36 -0700 Subject: [PATCH 1/7] CI: split arm64/amd64 image builds onto native runners Replace the QEMU-emulated `docker buildx --platform amd64,arm64` build in the deploy-stage workflow with a per-arch native-runner strategy (adapted from sredevopsorg/multi-arch-docker-github-workflow): - New reusable workflow build-multiarch-image.yml builds one arch per native runner (ubuntu-latest for amd64, ubuntu-24.04-arm for arm64), pushes each to ECR by digest, then stitches them into a manifest list with `docker buildx imagetools create`. Emitted tag is unchanged. - merge.yml now calls it once for accounts and once for keycloak, gated on the same detect-changes outputs, and the deploy job consumes the resulting manifest tags instead of building inline (QEMU/buildx/ECR-login steps dropped from deploy). Native builds run in parallel and skip emulation overhead. The published tags stay multi-arch manifest lists, so release.yml's retag step and the ECS (amd64) / EKS Graviton (arm64) runtimes are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-multiarch-image.yml | 156 ++++++++++++++++++++ .github/workflows/merge.yml | 121 +++++++-------- 2 files changed, 219 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/build-multiarch-image.yml diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml new file mode 100644 index 00000000..1a503302 --- /dev/null +++ b/.github/workflows/build-multiarch-image.yml @@ -0,0 +1,156 @@ +--- +# Reusable workflow that builds one container image for linux/amd64 and +# linux/arm64 on NATIVE runners (no QEMU), pushes each arch to ECR by digest, +# then stitches the digests into a single multi-arch manifest list under the +# real tag. Callers get back the fully-qualified image reference via the +# `image` output. See merge.yml for how accounts + keycloak invoke this. +# +# Why native runners instead of `buildx --platform amd64,arm64` with QEMU: +# emulated arm64 builds are minutes-slower and occasionally miscompile. Each +# arch now builds on its own hardware and the two run in parallel. +name: build-multiarch-image + +on: + workflow_call: + inputs: + dockerfile: + description: Path to the Dockerfile to build. + required: false + type: string + default: Dockerfile + tag: + description: Tag to publish the manifest list under (appended to the ECR repo). + required: true + type: string + artifact-prefix: + description: >- + Unique prefix for this image's digest artifacts. Distinct images built + in the same run (accounts, keycloak) MUST use different prefixes so + their per-arch digest artifacts don't collide. + required: true + type: string + outputs: + image: + description: The fully-qualified multi-arch image reference that was pushed. + value: ${{ jobs.merge.outputs.image }} + +permissions: + id-token: write + contents: read + +jobs: + # One job per architecture, each on its matching native runner. Each pushes + # an anonymous (tag-less) image to ECR and records its digest as an artifact. + build: + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + env: + IS_CI_AUTOMATION: "yes" + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: "true" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push image by digest + id: build + env: + IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" + run: | + # Build this single platform and push it to ECR by digest only (no + # tag). push-by-digest keeps the pushed image anonymous so both arch + # runners can push concurrently without clobbering a shared tag; the + # merge job assigns the real tag afterwards. --metadata-file lets us + # read back the digest that ECR assigned. + docker buildx build \ + -f "${{ inputs.dockerfile }}" \ + --platform "${{ matrix.platform }}" \ + --output "type=image,name=${IMAGE},push-by-digest=true,name-canonical=true,push=true" \ + --metadata-file metadata.json \ + . + digest="$(jq -r '."containerimage.digest"' metadata.json)" + echo "digest=$digest" >> "$GITHUB_OUTPUT" + + - name: Export digest + run: | + # Record the digest as an empty file named after the sha256 hex, so + # the merge job can reconstruct `${IMAGE}@sha256:` references by + # listing the downloaded artifact directory. + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + # Combine the per-arch digests into one manifest list under the real tag. + merge: + needs: build + runs-on: ubuntu-latest + env: + IS_CI_AUTOMATION: "yes" + outputs: + image: ${{ steps.merge.outputs.image }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: "true" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-${{ inputs.artifact-prefix }}-* + merge-multiple: true + + - name: Create manifest list and push + id: merge + working-directory: ${{ runner.temp }}/digests + env: + IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" + run: | + # imagetools create references digests already in ECR, so nothing is + # rebuilt or re-pushed here -- it only writes the manifest list. The + # glob expands to one `${IMAGE}@sha256:` per downloaded digest. + TARGET="${IMAGE}:${{ inputs.tag }}" + docker buildx imagetools create -t "$TARGET" \ + $(printf "${IMAGE}@sha256:%s " *) + docker buildx imagetools inspect "$TARGET" + echo "image=$TARGET" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 3553322b..8e22d977 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -53,16 +53,57 @@ jobs: - 'pulumi/**' - '.github/**' - # Build whichever image(s) changed and apply them to the stage stack in a - # SINGLE `pulumi up`. Pulumi takes a stack-wide lock per update, so running - # accounts and keycloak deploys as separate parallel jobs caused - # "[409] Conflict: Another update is currently in progress." See issue #884. - deploy: + # Build the accounts image multi-arch (amd64 for ECS Fargate, arm64 for the + # Thunderbird Pro EKS Graviton clusters) on native runners, one arch each, + # then merge into a manifest list tagged with the git sha. See + # build-multiarch-image.yml. Only runs when accounts/IaC changed. + build-accounts-image: needs: detect-changes if: >- needs.detect-changes.outputs.src-changed == 'true' || + needs.detect-changes.outputs.iac-changed == 'true' + uses: ./.github/workflows/build-multiarch-image.yml + secrets: inherit + with: + dockerfile: Dockerfile + tag: ${{ github.sha }} + artifact-prefix: accounts + + # Same native-runner multi-arch build for the keycloak image. Only runs when + # keycloak/IaC changed. + build-keycloak-image: + needs: detect-changes + if: >- needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' + uses: ./.github/workflows/build-multiarch-image.yml + secrets: inherit + with: + dockerfile: Dockerfile.keycloak + tag: keycloak-${{ github.sha }} + artifact-prefix: keycloak + + # Apply whichever image(s) were built to the stage stack in a SINGLE + # `pulumi up`. Pulumi takes a stack-wide lock per update, so running accounts + # and keycloak deploys as separate parallel jobs caused "[409] Conflict: + # Another update is currently in progress." See issue #884. + deploy: + needs: + - detect-changes + - build-accounts-image + - build-keycloak-image + # Run whenever something changed AND neither build was cancelled/failed. + # always() is required because the build jobs are individually skipped when + # their image didn't change -- without it, a skipped need would skip deploy. + if: >- + always() && + (needs.detect-changes.outputs.src-changed == 'true' || + needs.detect-changes.outputs.keycloak-theme-changed == 'true' || + needs.detect-changes.outputs.iac-changed == 'true') && + needs.build-accounts-image.result != 'failure' && + needs.build-accounts-image.result != 'cancelled' && + needs.build-keycloak-image.result != 'failure' && + needs.build-keycloak-image.result != 'cancelled' environment: name: staging deployment: true @@ -70,14 +111,17 @@ jobs: env: IS_CI_AUTOMATION: "yes" PULUMI_DIR: "pulumi" - # Whether the accounts image was (re)built this run; gates the - # accounts config update, its pulumi targets, and the release artifacts. - BUILD_ACCOUNTS: ${{ needs.detect-changes.outputs.src-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} - # Whether the keycloak image was (re)built this run. - BUILD_KEYCLOAK: ${{ needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} + # Manifest-list tags produced by the build jobs above; empty when that + # image wasn't (re)built this run (its build job was skipped). + ACCOUNTS_IMAGE: ${{ needs.build-accounts-image.outputs.image }} + KEYCLOAK_IMAGE: ${{ needs.build-keycloak-image.outputs.image }} + # Whether each image was (re)built this run; gates the config update, + # pulumi targets, and release artifacts below. + BUILD_ACCOUNTS: ${{ needs.build-accounts-image.outputs.image != '' }} + BUILD_KEYCLOAK: ${{ needs.build-keycloak-image.outputs.image != '' }} outputs: # Non-empty only when the accounts image was built; used to gate create-release. - accounts-image: ${{ steps.build-accounts.outputs.accounts-image }} + accounts-image: ${{ needs.build-accounts-image.outputs.image }} steps: # Preparation for future steps - uses: actions/checkout@v4 @@ -91,12 +135,6 @@ jobs: aws-region: eu-central-1 role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - mask-password: "true" - - name: Set up Python ${{ vars.PYTHON_VERSION}} uses: actions/setup-python@v5 with: @@ -118,43 +156,10 @@ jobs: curl -fsSL https://get.pulumi.com | sh pip install -Ur requirements.txt - # Both the accounts and keycloak images are built multi-arch via Buildx + - # QEMU (linux/amd64 for ECS Fargate today, linux/arm64 for the Thunderbird - # Pro EKS clusters on Graviton). Each is a manifest list, so every - # runtime pulls its matching variant. Set these up whenever either image - # is (re)built; they must run before the build steps below. - - name: Set up QEMU - if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true' - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true' - uses: docker/setup-buildx-action@v3 - - # Produce the accounts container image (only when accounts/IaC changed). - - name: Build, tag, and push accounts image to Amazon ECR - id: build-accounts - if: env.BUILD_ACCOUNTS == 'true' - env: - ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:${{ github.sha }}" - run: | - # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the - # manifest list to ECR. buildx --push publishes directly (no docker push). - docker buildx build -t $ECR_TAG \ - --platform linux/amd64,linux/arm64 --push . - echo "accounts-image=$ECR_TAG" >> $GITHUB_OUTPUT - - # Produce the keycloak container image (only when keycloak/IaC changed). - - name: Build, tag, and push keycloak image to Amazon ECR - id: build-keycloak - if: env.BUILD_KEYCLOAK == 'true' - env: - ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:keycloak-${{ github.sha }}" - run: | - # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the - # manifest list to ECR. buildx --push publishes directly (no docker push). - docker buildx build -f Dockerfile.keycloak -t $ECR_TAG \ - --platform linux/amd64,linux/arm64 --push . - echo "keycloak-image=$ECR_TAG" >> $GITHUB_OUTPUT + # The accounts and keycloak images are built by the build-accounts-image / + # build-keycloak-image jobs above (native-runner multi-arch, one arch per + # runner, merged into a manifest list). This job consumes their manifest + # tags via ACCOUNTS_IMAGE / KEYCLOAK_IMAGE and only deploys. - name: Get version from pyproject.toml id: read-version @@ -172,7 +177,7 @@ jobs: id: create-keycloak-artifact if: env.BUILD_KEYCLOAK == 'true' run: | - echo '{"ecr_tag": "${{ steps.build-keycloak.outputs.keycloak-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json + echo '{"ecr_tag": "'"$KEYCLOAK_IMAGE"'", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json - name: Archive the keycloak deployment artifact if: env.BUILD_KEYCLOAK == 'true' @@ -187,7 +192,7 @@ jobs: id: create-artifact if: env.BUILD_ACCOUNTS == 'true' run: | - echo '{"ecr_tag": "${{ steps.build-accounts.outputs.accounts-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json + echo '{"ecr_tag": "'"$ACCOUNTS_IMAGE"'", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json - name: Archive the deployment artifact id: tag-archive @@ -223,8 +228,8 @@ jobs: shell: bash env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} - ACCOUNTS_IMAGE: ${{ steps.build-accounts.outputs.accounts-image }} - KEYCLOAK_IMAGE: ${{ steps.build-keycloak.outputs.keycloak-image }} + # ACCOUNTS_IMAGE / KEYCLOAK_IMAGE are inherited from the job-level env + # (the manifest tags output by the build-*-image jobs). run: | # Update the PATH to include the right version of Pulumi; this is non-trivial or impossible # to do with the GHA workflow "env" settings above. From 9c86064d0b072402afacbeb27f87c950520b753b Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 10:23:12 -0700 Subject: [PATCH 2/7] CI: TEMPORARY - trigger deploy-stage on ci/split-arch-image-build Lets this branch exercise the split-arch build (and stage deploy). Revert before merging. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/merge.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 8e22d977..eaf69c9b 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -9,6 +9,8 @@ on: push: branches: - main + # TEMPORARY: exercise the split-arch build on this branch. Remove before merge. + - ci/split-arch-image-build workflow_dispatch: permissions: From 7d14c7d98b0b3970c97722c5574ad81c6d24741b Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 10:28:35 -0700 Subject: [PATCH 3/7] CI: TEMPORARY dry-run mode - build only, no ECR push, no deploy For collecting evidence that the native-runner arm64/amd64 builds work, before wiring up ECR/deploy: - merge.yml: trigger on every push; deploy job disabled via an always-false condition (cascades to skip create-release + e2e); drop `secrets: inherit` from the build caller jobs. - build-multiarch-image.yml: build each arch natively but do NOT push; AWS/ECR steps, digest export/upload, and the imagetools merge are commented out. Merge job neutralized to a no-op. This also resolves the ci-cd-trust-boundary finding for the branch trigger: with no secrets:inherit and no role assumption on the build path, and deploy disabled, no secret-bearing job runs on branch pushes. All changes are marked TEMPORARY and reverted before merge. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-multiarch-image.yml | 180 ++++++++++++-------- .github/workflows/merge.yml | 34 ++-- 2 files changed, 124 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml index 1a503302..d25edc42 100644 --- a/.github/workflows/build-multiarch-image.yml +++ b/.github/workflows/build-multiarch-image.yml @@ -8,6 +8,15 @@ # Why native runners instead of `buildx --platform amd64,arm64` with QEMU: # emulated arm64 builds are minutes-slower and occasionally miscompile. Each # arch now builds on its own hardware and the two run in parallel. +# +# ============================ TEMPORARY: DRY-RUN ============================= +# This workflow is currently in a build-test mode: it builds each arch on its +# native runner but does NOT push to ECR and does NOT produce a manifest list. +# The AWS/ECR steps and the digest/merge machinery are commented out below so +# the build can be exercised on any branch with no cloud credentials and no +# side effects. Restore the commented blocks (and `secrets: inherit` in +# merge.yml) once the native builds are confirmed green. +# ============================================================================ name: build-multiarch-image on: @@ -39,8 +48,8 @@ permissions: contents: read jobs: - # One job per architecture, each on its matching native runner. Each pushes - # an anonymous (tag-less) image to ECR and records its digest as an artifact. + # One job per architecture, each on its matching native runner. During the + # dry-run this just proves the build compiles natively -- nothing is pushed. build: strategy: fail-fast: false @@ -58,58 +67,73 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-central-1 - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - mask-password: "true" + # TEMPORARY: AWS/ECR access disabled for the dry-run (no push == no creds). + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-region: eu-central-1 + # role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + # + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v2 + # with: + # mask-password: "true" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push image by digest + # TEMPORARY: build only, do NOT push. This still runs every Dockerfile + # stage on the native runner, so a compile/build failure fails the job -- + # which is the evidence we're collecting. Restore the push-by-digest block + # below (and the digest export/upload) once builds are green. + - name: Build image (dry-run, no push) id: build - env: - IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" run: | - # Build this single platform and push it to ECR by digest only (no - # tag). push-by-digest keeps the pushed image anonymous so both arch - # runners can push concurrently without clobbering a shared tag; the - # merge job assigns the real tag afterwards. --metadata-file lets us - # read back the digest that ECR assigned. docker buildx build \ -f "${{ inputs.dockerfile }}" \ --platform "${{ matrix.platform }}" \ - --output "type=image,name=${IMAGE},push-by-digest=true,name-canonical=true,push=true" \ - --metadata-file metadata.json \ + -t "${{ inputs.artifact-prefix }}:${{ matrix.arch }}-dryrun" \ . - digest="$(jq -r '."containerimage.digest"' metadata.json)" - echo "digest=$digest" >> "$GITHUB_OUTPUT" - - - name: Export digest - run: | - # Record the digest as an empty file named after the sha256 hex, so - # the merge job can reconstruct `${IMAGE}@sha256:` references by - # listing the downloaded artifact directory. - mkdir -p "${{ runner.temp }}/digests" - digest="${{ steps.build.outputs.digest }}" - touch "${{ runner.temp }}/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 + # - name: Build and push image by digest + # id: build + # env: + # IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" + # run: | + # # Build this single platform and push it to ECR by digest only (no + # # tag). push-by-digest keeps the pushed image anonymous so both arch + # # runners can push concurrently without clobbering a shared tag; the + # # merge job assigns the real tag afterwards. --metadata-file lets us + # # read back the digest that ECR assigned. + # docker buildx build \ + # -f "${{ inputs.dockerfile }}" \ + # --platform "${{ matrix.platform }}" \ + # --output "type=image,name=${IMAGE},push-by-digest=true,name-canonical=true,push=true" \ + # --metadata-file metadata.json \ + # . + # digest="$(jq -r '."containerimage.digest"' metadata.json)" + # echo "digest=$digest" >> "$GITHUB_OUTPUT" + # + # - name: Export digest + # run: | + # # Record the digest as an empty file named after the sha256 hex, so + # # the merge job can reconstruct `${IMAGE}@sha256:` references by + # # listing the downloaded artifact directory. + # mkdir -p "${{ runner.temp }}/digests" + # digest="${{ steps.build.outputs.digest }}" + # touch "${{ runner.temp }}/digests/${digest#sha256:}" + # + # - name: Upload digest + # uses: actions/upload-artifact@v4 + # with: + # name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} + # path: ${{ runner.temp }}/digests/* + # if-no-files-found: error + # retention-days: 1 # Combine the per-arch digests into one manifest list under the real tag. + # TEMPORARY: neutralized during the dry-run (nothing was pushed to merge). merge: needs: build runs-on: ubuntu-latest @@ -118,39 +142,45 @@ jobs: outputs: image: ${{ steps.merge.outputs.image }} steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-central-1 - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - mask-password: "true" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: ${{ runner.temp }}/digests - pattern: digests-${{ inputs.artifact-prefix }}-* - merge-multiple: true - - - name: Create manifest list and push + - name: Merge disabled (dry-run) id: merge - working-directory: ${{ runner.temp }}/digests - env: - IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" run: | - # imagetools create references digests already in ECR, so nothing is - # rebuilt or re-pushed here -- it only writes the manifest list. The - # glob expands to one `${IMAGE}@sha256:` per downloaded digest. - TARGET="${IMAGE}:${{ inputs.tag }}" - docker buildx imagetools create -t "$TARGET" \ - $(printf "${IMAGE}@sha256:%s " *) - docker buildx imagetools inspect "$TARGET" - echo "image=$TARGET" >> "$GITHUB_OUTPUT" + echo "Dry-run: skipping manifest merge; no images were pushed to ECR." + echo "image=" >> "$GITHUB_OUTPUT" + + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-region: eu-central-1 + # role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + # + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v2 + # with: + # mask-password: "true" + # + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # + # - name: Download digests + # uses: actions/download-artifact@v4 + # with: + # path: ${{ runner.temp }}/digests + # pattern: digests-${{ inputs.artifact-prefix }}-* + # merge-multiple: true + # + # - name: Create manifest list and push + # id: merge + # working-directory: ${{ runner.temp }}/digests + # env: + # IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" + # run: | + # # imagetools create references digests already in ECR, so nothing is + # # rebuilt or re-pushed here -- it only writes the manifest list. The + # # glob expands to one `${IMAGE}@sha256:` per downloaded digest. + # TARGET="${IMAGE}:${{ inputs.tag }}" + # docker buildx imagetools create -t "$TARGET" \ + # $(printf "${IMAGE}@sha256:%s " *) + # docker buildx imagetools inspect "$TARGET" + # echo "image=$TARGET" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index eaf69c9b..b2639f31 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -6,11 +6,9 @@ concurrency: cancel-in-progress: true on: + # TEMPORARY: run on every push so the split-arch build can be tested from any + # branch. Restore the `branches: [main]` filter before merge. push: - branches: - - main - # TEMPORARY: exercise the split-arch build on this branch. Remove before merge. - - ci/split-arch-image-build workflow_dispatch: permissions: @@ -65,7 +63,8 @@ jobs: needs.detect-changes.outputs.src-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' uses: ./.github/workflows/build-multiarch-image.yml - secrets: inherit + # TEMPORARY: no `secrets: inherit` during the dry-run -- the build no longer + # pushes to ECR, so it needs no AWS credentials. Restore before merge. with: dockerfile: Dockerfile tag: ${{ github.sha }} @@ -79,7 +78,7 @@ jobs: needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' uses: ./.github/workflows/build-multiarch-image.yml - secrets: inherit + # TEMPORARY: no `secrets: inherit` during the dry-run -- see accounts job above. with: dockerfile: Dockerfile.keycloak tag: keycloak-${{ github.sha }} @@ -94,18 +93,23 @@ jobs: - detect-changes - build-accounts-image - build-keycloak-image + # TEMPORARY: deploy disabled for the dry-run build test. This condition can + # never be true (no such branch), so the job is always skipped -- which + # cascades to skip create-release + e2e (they need it) and deploys nothing + # to stage. Remove this line to restore the real condition below before merge. + if: github.ref == 'refs/heads/__deploy-disabled-for-dryrun__' # Run whenever something changed AND neither build was cancelled/failed. # always() is required because the build jobs are individually skipped when # their image didn't change -- without it, a skipped need would skip deploy. - if: >- - always() && - (needs.detect-changes.outputs.src-changed == 'true' || - needs.detect-changes.outputs.keycloak-theme-changed == 'true' || - needs.detect-changes.outputs.iac-changed == 'true') && - needs.build-accounts-image.result != 'failure' && - needs.build-accounts-image.result != 'cancelled' && - needs.build-keycloak-image.result != 'failure' && - needs.build-keycloak-image.result != 'cancelled' + # if: >- + # always() && + # (needs.detect-changes.outputs.src-changed == 'true' || + # needs.detect-changes.outputs.keycloak-theme-changed == 'true' || + # needs.detect-changes.outputs.iac-changed == 'true') && + # needs.build-accounts-image.result != 'failure' && + # needs.build-accounts-image.result != 'cancelled' && + # needs.build-keycloak-image.result != 'failure' && + # needs.build-keycloak-image.result != 'cancelled' environment: name: staging deployment: true From d12a4098463678caee291ebec2257289105d2791 Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 11:03:31 -0700 Subject: [PATCH 4/7] CI: decouple multi-arch image publishing into a GHCR pipeline Reworks PR to separate image publishing from deployment: - merge.yml: revert the deploy-stage build to single-arch amd64 -> ECR (removes the QEMU/Buildx multi-arch build added in #1027/#974). Deploy behavior is otherwise unchanged; change detection stays its own job. - publish-images.yml (new): standalone pipeline that, on merge to main, detects which image changed (separate detect-changes job), reads the version, and builds accounts + keycloak as native multi-arch manifest lists pushed to GHCR. No cloud creds, no Pulumi, no deploy. - build-multiarch-image.yml: reusable native split-arch builder, now targeting GHCR via GITHUB_TOKEN. Builds amd64 (ubuntu-latest) and arm64 (ubuntu-24.04-arm) on native runners, pushes by digest, merges into one manifest list tagged :, :v, and :latest. Separate GHCR packages per image. - docs/ci/image-publish-pipeline.md: mermaid architecture diagram of both pipelines and why they're decoupled. Decoupling keeps cloud credentials out of the image-publish path (resolves the earlier ci-cd-trust-boundary concern) and moves arm64 off QEMU onto native Graviton runners. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-multiarch-image.yml | 200 +++++++++----------- .github/workflows/merge.yml | 119 +++++------- .github/workflows/publish-images.yml | 92 +++++++++ docs/ci/image-publish-pipeline.md | 73 +++++++ 4 files changed, 298 insertions(+), 186 deletions(-) create mode 100644 .github/workflows/publish-images.yml create mode 100644 docs/ci/image-publish-pipeline.md diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml index d25edc42..588458d2 100644 --- a/.github/workflows/build-multiarch-image.yml +++ b/.github/workflows/build-multiarch-image.yml @@ -1,34 +1,32 @@ --- -# Reusable workflow that builds one container image for linux/amd64 and -# linux/arm64 on NATIVE runners (no QEMU), pushes each arch to ECR by digest, -# then stitches the digests into a single multi-arch manifest list under the -# real tag. Callers get back the fully-qualified image reference via the -# `image` output. See merge.yml for how accounts + keycloak invoke this. +# Reusable workflow: build ONE container image for linux/amd64 and linux/arm64 +# on NATIVE runners (no QEMU), push each arch to GHCR by digest, then stitch the +# digests into a single multi-arch manifest list tagged :, :v and +# :latest. Callers get the fully-qualified : reference back via the `image` +# output. Invoked by publish-images.yml for the accounts and keycloak images. # # Why native runners instead of `buildx --platform amd64,arm64` with QEMU: # emulated arm64 builds are minutes-slower and occasionally miscompile. Each -# arch now builds on its own hardware and the two run in parallel. +# arch builds on its own hardware and the two run in parallel. # -# ============================ TEMPORARY: DRY-RUN ============================= -# This workflow is currently in a build-test mode: it builds each arch on its -# native runner but does NOT push to ECR and does NOT produce a manifest list. -# The AWS/ECR steps and the digest/merge machinery are commented out below so -# the build can be exercised on any branch with no cloud credentials and no -# side effects. Restore the commented blocks (and `secrets: inherit` in -# merge.yml) once the native builds are confirmed green. -# ============================================================================ +# Auth is GHCR + the automatic GITHUB_TOKEN (packages: write) -- no cloud +# credentials, which keeps this pipeline decoupled from the ECR/Pulumi deploy. name: build-multiarch-image on: workflow_call: inputs: + image: + description: Fully-qualified GHCR image name WITHOUT a tag (e.g. ghcr.io/owner/name). + required: true + type: string dockerfile: description: Path to the Dockerfile to build. required: false type: string default: Dockerfile - tag: - description: Tag to publish the manifest list under (appended to the ECR repo). + version: + description: Semantic version (no leading v) published as the :v tag. required: true type: string artifact-prefix: @@ -40,16 +38,16 @@ on: type: string outputs: image: - description: The fully-qualified multi-arch image reference that was pushed. + description: The :-tagged multi-arch image reference that was pushed. value: ${{ jobs.merge.outputs.image }} permissions: - id-token: write contents: read + packages: write jobs: - # One job per architecture, each on its matching native runner. During the - # dry-run this just proves the build compiles natively -- nothing is pushed. + # One job per architecture, each on its matching native runner. Each pushes an + # anonymous (tag-less) image to GHCR and records its digest as an artifact. build: strategy: fail-fast: false @@ -67,73 +65,51 @@ jobs: steps: - uses: actions/checkout@v4 - # TEMPORARY: AWS/ECR access disabled for the dry-run (no push == no creds). - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-region: eu-central-1 - # role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - # - # - name: Login to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 - # with: - # mask-password: "true" + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # TEMPORARY: build only, do NOT push. This still runs every Dockerfile - # stage on the native runner, so a compile/build failure fails the job -- - # which is the evidence we're collecting. Restore the push-by-digest block - # below (and the digest export/upload) once builds are green. - - name: Build image (dry-run, no push) + - name: Build and push image by digest id: build run: | + # Build this single platform and push it to GHCR by digest only (no + # tag). push-by-digest keeps the pushed image anonymous so both arch + # runners can push concurrently without clobbering a shared tag; the + # merge job assigns the real tags afterwards. --metadata-file lets us + # read back the digest that GHCR assigned. docker buildx build \ -f "${{ inputs.dockerfile }}" \ --platform "${{ matrix.platform }}" \ - -t "${{ inputs.artifact-prefix }}:${{ matrix.arch }}-dryrun" \ + --output "type=image,name=${{ inputs.image }},push-by-digest=true,name-canonical=true,push=true" \ + --metadata-file metadata.json \ . + digest="$(jq -r '."containerimage.digest"' metadata.json)" + echo "digest=$digest" >> "$GITHUB_OUTPUT" - # - name: Build and push image by digest - # id: build - # env: - # IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" - # run: | - # # Build this single platform and push it to ECR by digest only (no - # # tag). push-by-digest keeps the pushed image anonymous so both arch - # # runners can push concurrently without clobbering a shared tag; the - # # merge job assigns the real tag afterwards. --metadata-file lets us - # # read back the digest that ECR assigned. - # docker buildx build \ - # -f "${{ inputs.dockerfile }}" \ - # --platform "${{ matrix.platform }}" \ - # --output "type=image,name=${IMAGE},push-by-digest=true,name-canonical=true,push=true" \ - # --metadata-file metadata.json \ - # . - # digest="$(jq -r '."containerimage.digest"' metadata.json)" - # echo "digest=$digest" >> "$GITHUB_OUTPUT" - # - # - name: Export digest - # run: | - # # Record the digest as an empty file named after the sha256 hex, so - # # the merge job can reconstruct `${IMAGE}@sha256:` references by - # # listing the downloaded artifact directory. - # mkdir -p "${{ runner.temp }}/digests" - # digest="${{ steps.build.outputs.digest }}" - # touch "${{ runner.temp }}/digests/${digest#sha256:}" - # - # - name: Upload digest - # uses: actions/upload-artifact@v4 - # with: - # name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} - # path: ${{ runner.temp }}/digests/* - # if-no-files-found: error - # retention-days: 1 + - name: Export digest + run: | + # Record the digest as an empty file named after the sha256 hex, so + # the merge job can reconstruct `${IMAGE}@sha256:` references by + # listing the downloaded artifact directory. + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 - # Combine the per-arch digests into one manifest list under the real tag. - # TEMPORARY: neutralized during the dry-run (nothing was pushed to merge). + # Combine the per-arch digests into one manifest list under all three tags. merge: needs: build runs-on: ubuntu-latest @@ -142,45 +118,39 @@ jobs: outputs: image: ${{ steps.merge.outputs.image }} steps: - - name: Merge disabled (dry-run) + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-${{ inputs.artifact-prefix }}-* + merge-multiple: true + + - name: Create manifest list and push id: merge + working-directory: ${{ runner.temp }}/digests + env: + IMAGE: ${{ inputs.image }} + VERSION: ${{ inputs.version }} + SHA: ${{ github.sha }} run: | - echo "Dry-run: skipping manifest merge; no images were pushed to ECR." - echo "image=" >> "$GITHUB_OUTPUT" - - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-region: eu-central-1 - # role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - # - # - name: Login to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 - # with: - # mask-password: "true" - # - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - # - # - name: Download digests - # uses: actions/download-artifact@v4 - # with: - # path: ${{ runner.temp }}/digests - # pattern: digests-${{ inputs.artifact-prefix }}-* - # merge-multiple: true - # - # - name: Create manifest list and push - # id: merge - # working-directory: ${{ runner.temp }}/digests - # env: - # IMAGE: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}" - # run: | - # # imagetools create references digests already in ECR, so nothing is - # # rebuilt or re-pushed here -- it only writes the manifest list. The - # # glob expands to one `${IMAGE}@sha256:` per downloaded digest. - # TARGET="${IMAGE}:${{ inputs.tag }}" - # docker buildx imagetools create -t "$TARGET" \ - # $(printf "${IMAGE}@sha256:%s " *) - # docker buildx imagetools inspect "$TARGET" - # echo "image=$TARGET" >> "$GITHUB_OUTPUT" + # imagetools create references digests already in GHCR, so nothing is + # rebuilt or re-pushed here -- it only writes the manifest list, under + # all three tags at once. The glob expands to one + # `${IMAGE}@sha256:` per downloaded digest. + docker buildx imagetools create \ + -t "${IMAGE}:${SHA}" \ + -t "${IMAGE}:v${VERSION}" \ + -t "${IMAGE}:latest" \ + $(printf "${IMAGE}@sha256:%s " *) + docker buildx imagetools inspect "${IMAGE}:${SHA}" + echo "image=${IMAGE}:${SHA}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index b2639f31..c1c25ce0 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -6,9 +6,9 @@ concurrency: cancel-in-progress: true on: - # TEMPORARY: run on every push so the split-arch build can be tested from any - # branch. Restore the `branches: [main]` filter before merge. push: + branches: + - main workflow_dispatch: permissions: @@ -53,63 +53,16 @@ jobs: - 'pulumi/**' - '.github/**' - # Build the accounts image multi-arch (amd64 for ECS Fargate, arm64 for the - # Thunderbird Pro EKS Graviton clusters) on native runners, one arch each, - # then merge into a manifest list tagged with the git sha. See - # build-multiarch-image.yml. Only runs when accounts/IaC changed. - build-accounts-image: + # Build whichever image(s) changed and apply them to the stage stack in a + # SINGLE `pulumi up`. Pulumi takes a stack-wide lock per update, so running + # accounts and keycloak deploys as separate parallel jobs caused + # "[409] Conflict: Another update is currently in progress." See issue #884. + deploy: needs: detect-changes if: >- needs.detect-changes.outputs.src-changed == 'true' || - needs.detect-changes.outputs.iac-changed == 'true' - uses: ./.github/workflows/build-multiarch-image.yml - # TEMPORARY: no `secrets: inherit` during the dry-run -- the build no longer - # pushes to ECR, so it needs no AWS credentials. Restore before merge. - with: - dockerfile: Dockerfile - tag: ${{ github.sha }} - artifact-prefix: accounts - - # Same native-runner multi-arch build for the keycloak image. Only runs when - # keycloak/IaC changed. - build-keycloak-image: - needs: detect-changes - if: >- needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' - uses: ./.github/workflows/build-multiarch-image.yml - # TEMPORARY: no `secrets: inherit` during the dry-run -- see accounts job above. - with: - dockerfile: Dockerfile.keycloak - tag: keycloak-${{ github.sha }} - artifact-prefix: keycloak - - # Apply whichever image(s) were built to the stage stack in a SINGLE - # `pulumi up`. Pulumi takes a stack-wide lock per update, so running accounts - # and keycloak deploys as separate parallel jobs caused "[409] Conflict: - # Another update is currently in progress." See issue #884. - deploy: - needs: - - detect-changes - - build-accounts-image - - build-keycloak-image - # TEMPORARY: deploy disabled for the dry-run build test. This condition can - # never be true (no such branch), so the job is always skipped -- which - # cascades to skip create-release + e2e (they need it) and deploys nothing - # to stage. Remove this line to restore the real condition below before merge. - if: github.ref == 'refs/heads/__deploy-disabled-for-dryrun__' - # Run whenever something changed AND neither build was cancelled/failed. - # always() is required because the build jobs are individually skipped when - # their image didn't change -- without it, a skipped need would skip deploy. - # if: >- - # always() && - # (needs.detect-changes.outputs.src-changed == 'true' || - # needs.detect-changes.outputs.keycloak-theme-changed == 'true' || - # needs.detect-changes.outputs.iac-changed == 'true') && - # needs.build-accounts-image.result != 'failure' && - # needs.build-accounts-image.result != 'cancelled' && - # needs.build-keycloak-image.result != 'failure' && - # needs.build-keycloak-image.result != 'cancelled' environment: name: staging deployment: true @@ -117,17 +70,14 @@ jobs: env: IS_CI_AUTOMATION: "yes" PULUMI_DIR: "pulumi" - # Manifest-list tags produced by the build jobs above; empty when that - # image wasn't (re)built this run (its build job was skipped). - ACCOUNTS_IMAGE: ${{ needs.build-accounts-image.outputs.image }} - KEYCLOAK_IMAGE: ${{ needs.build-keycloak-image.outputs.image }} - # Whether each image was (re)built this run; gates the config update, - # pulumi targets, and release artifacts below. - BUILD_ACCOUNTS: ${{ needs.build-accounts-image.outputs.image != '' }} - BUILD_KEYCLOAK: ${{ needs.build-keycloak-image.outputs.image != '' }} + # Whether the accounts image was (re)built this run; gates the + # accounts config update, its pulumi targets, and the release artifacts. + BUILD_ACCOUNTS: ${{ needs.detect-changes.outputs.src-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} + # Whether the keycloak image was (re)built this run. + BUILD_KEYCLOAK: ${{ needs.detect-changes.outputs.keycloak-theme-changed == 'true' || needs.detect-changes.outputs.iac-changed == 'true' }} outputs: # Non-empty only when the accounts image was built; used to gate create-release. - accounts-image: ${{ needs.build-accounts-image.outputs.image }} + accounts-image: ${{ steps.build-accounts.outputs.accounts-image }} steps: # Preparation for future steps - uses: actions/checkout@v4 @@ -141,6 +91,12 @@ jobs: aws-region: eu-central-1 role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: "true" + - name: Set up Python ${{ vars.PYTHON_VERSION}} uses: actions/setup-python@v5 with: @@ -162,10 +118,31 @@ jobs: curl -fsSL https://get.pulumi.com | sh pip install -Ur requirements.txt - # The accounts and keycloak images are built by the build-accounts-image / - # build-keycloak-image jobs above (native-runner multi-arch, one arch per - # runner, merged into a manifest list). This job consumes their manifest - # tags via ACCOUNTS_IMAGE / KEYCLOAK_IMAGE and only deploys. + # Produce the accounts container image (only when accounts/IaC changed). + # This image is amd64-only (ECS Fargate). Multi-arch (amd64 + arm64) + # publishing lives in the decoupled publish-images.yml -> GHCR pipeline. + - name: Build, tag, and push accounts image to Amazon ECR + id: build-accounts + if: env.BUILD_ACCOUNTS == 'true' + env: + ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:${{ github.sha }}" + run: | + # Build a docker container and push it to ECR so that it can be deployed to ECS. + docker build -t $ECR_TAG --platform="linux/amd64" . + docker push $ECR_TAG + echo "accounts-image=$ECR_TAG" >> $GITHUB_OUTPUT + + # Produce the keycloak container image (only when keycloak/IaC changed). + - name: Build, tag, and push keycloak image to Amazon ECR + id: build-keycloak + if: env.BUILD_KEYCLOAK == 'true' + env: + ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:keycloak-${{ github.sha }}" + run: | + # Build a docker container and push it to ECR so that it can be deployed to ECS. + docker build -f Dockerfile.keycloak -t $ECR_TAG --platform="linux/amd64" . + docker push $ECR_TAG + echo "keycloak-image=$ECR_TAG" >> $GITHUB_OUTPUT - name: Get version from pyproject.toml id: read-version @@ -183,7 +160,7 @@ jobs: id: create-keycloak-artifact if: env.BUILD_KEYCLOAK == 'true' run: | - echo '{"ecr_tag": "'"$KEYCLOAK_IMAGE"'", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json + echo '{"ecr_tag": "${{ steps.build-keycloak.outputs.keycloak-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json - name: Archive the keycloak deployment artifact if: env.BUILD_KEYCLOAK == 'true' @@ -198,7 +175,7 @@ jobs: id: create-artifact if: env.BUILD_ACCOUNTS == 'true' run: | - echo '{"ecr_tag": "'"$ACCOUNTS_IMAGE"'", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json + echo '{"ecr_tag": "${{ steps.build-accounts.outputs.accounts-image }}", "version": "${{ steps.read-version.outputs.value }}"}' > deployment.json - name: Archive the deployment artifact id: tag-archive @@ -234,8 +211,8 @@ jobs: shell: bash env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} - # ACCOUNTS_IMAGE / KEYCLOAK_IMAGE are inherited from the job-level env - # (the manifest tags output by the build-*-image jobs). + ACCOUNTS_IMAGE: ${{ steps.build-accounts.outputs.accounts-image }} + KEYCLOAK_IMAGE: ${{ steps.build-keycloak.outputs.keycloak-image }} run: | # Update the PATH to include the right version of Pulumi; this is non-trivial or impossible # to do with the GHA workflow "env" settings above. diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 00000000..5ae7bc96 --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,92 @@ +--- +# Decoupled image-publish pipeline. On merge to main it builds the accounts and +# keycloak images as native multi-arch (amd64 + arm64) manifest lists and pushes +# them to GHCR. This is INDEPENDENT of the deploy-stage pipeline (merge.yml): +# it uses no cloud credentials, runs no Pulumi, and deploys nothing -- it only +# publishes images. See docs/ci/image-publish-pipeline.md for the architecture. +name: publish-images + +concurrency: + group: publish-images-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + # Separate job: decide which image(s) changed and read the version once, so + # the build jobs below only run for images that actually changed. + detect-changes: + runs-on: ubuntu-latest + outputs: + accounts-changed: ${{ steps.check.outputs.accounts-changed }} + keycloak-changed: ${{ steps.check.outputs.keycloak-changed }} + version: ${{ steps.read-version.outputs.value }} + steps: + - uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v3 + id: check + with: + filters: | + accounts-changed: + - 'assets/**' + - 'static/**' + - 'templates/**' + - 'scripts/entry.sh' + - 'Dockerfile' + - 'manage.py' + - 'MANIFEST.in' + - 'package.json' + - 'package-lock.json' + - 'pyproject.toml' + - 'README.md' + - 'src/**' + - 'uv.lock' + - 'vite.config.mjs' + keycloak-changed: + - 'keycloak/**' + - 'Dockerfile.keycloak' + - 'package.json' + - 'package-lock.json' + - 'scripts/entry-keycloak.sh' + - 'scripts/apply-mfa-config.sh' + + - name: Get version from pyproject.toml + id: read-version + uses: SebRollen/toml-action@v1.2.0 + with: + file: 'pyproject.toml' + field: 'project.version' + + accounts: + needs: detect-changes + # workflow_dispatch always rebuilds; pushes rebuild only on a relevant change. + if: >- + github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs.accounts-changed == 'true' + uses: ./.github/workflows/build-multiarch-image.yml + with: + image: ghcr.io/thunderbird/thunderbird-accounts + dockerfile: Dockerfile + version: ${{ needs.detect-changes.outputs.version }} + artifact-prefix: accounts + + keycloak: + needs: detect-changes + if: >- + github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs.keycloak-changed == 'true' + uses: ./.github/workflows/build-multiarch-image.yml + with: + image: ghcr.io/thunderbird/thunderbird-accounts-keycloak + dockerfile: Dockerfile.keycloak + version: ${{ needs.detect-changes.outputs.version }} + artifact-prefix: keycloak diff --git a/docs/ci/image-publish-pipeline.md b/docs/ci/image-publish-pipeline.md new file mode 100644 index 00000000..c7c7d528 --- /dev/null +++ b/docs/ci/image-publish-pipeline.md @@ -0,0 +1,73 @@ +# CI image pipelines + +Two **independent** pipelines produce container images. They share no jobs, no +credentials, and no triggers beyond both reacting to `main`. + +| Pipeline | Workflow | Registry | Arch | Purpose | +|---|---|---|---|---| +| **Deploy-stage** | `merge.yml` | ECR | amd64 only | Build + deploy to stage (ECS Fargate) via Pulumi | +| **Publish-images** | `publish-images.yml` + `build-multiarch-image.yml` | GHCR | amd64 + arm64 | Publish multi-arch manifest lists; no deploy | + +## Publish-images (new, decoupled) + +Native-runner split-arch build: each architecture builds on its own hardware +(no QEMU emulation) and pushes to GHCR by digest; a merge step stitches the +digests into one manifest list tagged `:`, `:v`, and `:latest`. +Auth is the automatic `GITHUB_TOKEN` (`packages: write`) — no cloud credentials. + +```mermaid +flowchart TD + push["push to main / workflow_dispatch"] --> detect + + subgraph publish["publish-images.yml (GHCR, no deploy)"] + detect["detect-changes (separate job)
paths-filter + read version"] + + detect -->|accounts changed| acc + detect -->|keycloak changed| kc + + subgraph acc["accounts → build-multiarch-image.yml"] + direction TB + a_amd["build amd64
(ubuntu-latest)
push by digest"] + a_arm["build arm64
(ubuntu-24.04-arm)
push by digest"] + a_merge["merge → manifest list
:sha :vX.Y.Z :latest"] + a_amd --> a_merge + a_arm --> a_merge + end + + subgraph kc["keycloak → build-multiarch-image.yml"] + direction TB + k_amd["build amd64
(ubuntu-latest)
push by digest"] + k_arm["build arm64
(ubuntu-24.04-arm)
push by digest"] + k_merge["merge → manifest list
:sha :vX.Y.Z :latest"] + k_amd --> k_merge + k_arm --> k_merge + end + end + + a_merge --> ghcr[("GHCR
ghcr.io/thunderbird/thunderbird-accounts{,-keycloak}")] + k_merge --> ghcr +``` + +## Deploy-stage (unchanged, for contrast) + +`merge.yml` still owns deployment: it builds the amd64 image, pushes to ECR, +and runs a single `pulumi up` to stage. It assumes the AWS deploy role via OIDC. + +```mermaid +flowchart TD + push2["push to main"] --> detect2["detect-changes"] + detect2 --> build2["build amd64 image
docker build --platform linux/amd64"] + build2 --> ecr[("ECR")] + build2 --> deploy["pulumi up → stage (ECS Fargate)"] + deploy --> release["draft GitHub release
(prod promotion)"] +``` + +## Why they're separate + +- **Blast radius** — the GHCR publish holds no cloud credentials, so a branch or + fork build can never assume the deploy role (the trust-boundary concern that + killed running the deploy workflow off feature branches). +- **Independent cadence** — image publishing and stage deployment can evolve, + fail, and be re-run without affecting each other. +- **Native multi-arch** — arm64 builds on real Graviton runners instead of QEMU + emulation, so they're faster and not subject to emulation miscompiles. From 699642517aaed4cf273d608903d06af2874e534a Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 11:23:56 -0700 Subject: [PATCH 5/7] CI: address 20-lens review of the GHCR publish pipeline publish-images.yml (change detection): - accounts-changed now includes keycloak/themes/** -- the accounts frontend imports the shared keycloak theme via the @kc vite alias, so `npm run build` bakes it into the accounts image; theme edits must rebuild accounts too. - fix vite.config.mjs -> vite.config.mts typo; add tsconfig.json, env.d.ts (all COPYd by the Dockerfile) so they trigger rebuilds. - broaden scripts/entry.sh -> scripts/** (Dockerfile COPYs the whole dir). - drop README.md as a trigger (docs-only, no runtime effect) to avoid a full multi-arch republish + :latest churn on doc changes. - concurrency cancel-in-progress: false so a merged commit's : image is never dropped and a run is never aborted mid-push. build-multiarch-image.yml (build): - --provenance=false so each per-arch push is a plain single manifest (clean 2-entry merged manifest list, no attestation index). - add GitHub Actions layer cache (type=gha) scoped per image+arch; ephemeral runners otherwise rebuild every layer on every merge. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-multiarch-image.yml | 9 +++++++++ .github/workflows/publish-images.yml | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml index 588458d2..0c8733a9 100644 --- a/.github/workflows/build-multiarch-image.yml +++ b/.github/workflows/build-multiarch-image.yml @@ -83,9 +83,18 @@ jobs: # runners can push concurrently without clobbering a shared tag; the # merge job assigns the real tags afterwards. --metadata-file lets us # read back the digest that GHCR assigned. + # --provenance=false keeps each per-arch push a single plain manifest + # (no attestation index), so the merged manifest list contains exactly + # the two platform entries. + # GitHub Actions layer cache, scoped per image+arch (these runners are + # ephemeral, so nothing persists between runs without it). The scope + # keeps accounts/keycloak and amd64/arm64 caches from colliding. docker buildx build \ -f "${{ inputs.dockerfile }}" \ --platform "${{ matrix.platform }}" \ + --provenance=false \ + --cache-from "type=gha,scope=${{ inputs.artifact-prefix }}-${{ matrix.arch }}" \ + --cache-to "type=gha,mode=max,scope=${{ inputs.artifact-prefix }}-${{ matrix.arch }}" \ --output "type=image,name=${{ inputs.image }},push-by-digest=true,name-canonical=true,push=true" \ --metadata-file metadata.json \ . diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 5ae7bc96..749e4856 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -8,7 +8,10 @@ name: publish-images concurrency: group: publish-images-${{ github.ref }} - cancel-in-progress: true + # Serialize (don't cancel) so every merged commit's : image gets + # published and a run is never aborted mid-push/merge. Queued runs still + # settle :latest to the newest finished build. + cancel-in-progress: false on: push: @@ -40,17 +43,26 @@ jobs: - 'assets/**' - 'static/**' - 'templates/**' - - 'scripts/entry.sh' + # The accounts frontend bundle imports the shared keycloak theme + # via the `@kc` vite alias (-> keycloak/themes/tbpro/assets), so + # `npm run build` compiles it into the accounts image. Theme edits + # must therefore rebuild accounts too. + - 'keycloak/themes/**' + # The accounts Dockerfile COPYs the whole scripts/ dir. + - 'scripts/**' - 'Dockerfile' - 'manage.py' - 'MANIFEST.in' - 'package.json' - 'package-lock.json' - 'pyproject.toml' - - 'README.md' - 'src/**' - 'uv.lock' - - 'vite.config.mjs' + # Frontend build inputs COPYd by the Dockerfile (note: .mts, plus + # tsconfig.json and env.d.ts). + - 'vite.config.mts' + - 'tsconfig.json' + - 'env.d.ts' keycloak-changed: - 'keycloak/**' - 'Dockerfile.keycloak' From fa36ae65f928221daa4094e6fb43a88e864b23c5 Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 11:26:45 -0700 Subject: [PATCH 6/7] CI: correct deploy-stage change detection for accounts image deps Same change-detection gaps the GHCR pipeline review surfaced, applied to the deploy pipeline's src-changed filter (which gates the accounts stage deploy): - fix vite.config.mjs -> vite.config.mts typo; add tsconfig.json + env.d.ts (all COPYd by the Dockerfile). - add scripts/** (Dockerfile COPYs the whole dir; was only scripts/entry.sh). - add keycloak/themes/** -- the accounts bundle imports the shared keycloak theme via the @kc vite alias, so a theme-only change previously rebuilt the keycloak image but NOT accounts, deploying a stale accounts frontend. Additive only (never under-builds); no change to deploy behavior otherwise. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/merge.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index c1c25ce0..a71caaed 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -35,7 +35,8 @@ jobs: - 'assets/**' - 'static/**' - 'templates/**' - - 'scripts/entry.sh' + # Dockerfile COPYs the whole scripts/ dir. + - 'scripts/**' - 'Dockerfile' - 'manage.py' - 'MANIFEST.in' @@ -45,7 +46,15 @@ jobs: - 'README.md' - 'src/**' - 'uv.lock' - - 'vite.config.mjs' + # Frontend build inputs COPYd by the Dockerfile (vite.config is + # .mts, not .mjs); tsconfig.json and env.d.ts were missing. + - 'vite.config.mts' + - 'tsconfig.json' + - 'env.d.ts' + # The accounts bundle imports the shared keycloak theme via the + # `@kc` vite alias, so `npm run build` compiles it into the accounts + # image -- theme edits must rebuild (and redeploy) accounts too. + - 'keycloak/themes/**' keycloak-theme-changed: - 'keycloak/themes/**' - 'Dockerfile.keycloak' From 1111e43f703c298423f65c9288c5712b69ca32b7 Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 8 Jul 2026 11:43:25 -0700 Subject: [PATCH 7/7] CI: pin actions in the new GHCR workflows to SHAs (latest versions) Update and SHA-pin every third-party action in the two new workflows (build-multiarch-image.yml, publish-images.yml). No existing workflow touched. actions/checkout v4 -> v7.0.0 docker/login-action v3 -> v4.4.0 docker/setup-buildx-action v3 -> v4.2.0 actions/upload-artifact v4 -> v7.0.1 actions/download-artifact v4 -> v8.0.1 (v8 pairs with upload v7) dorny/paths-filter v3 -> v4.0.2 SebRollen/toml-action v1.2.0 (pinned; already latest) Each is pinned to the commit SHA the release tag resolves to, with the version in a trailing comment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-multiarch-image.yml | 14 +++++++------- .github/workflows/publish-images.yml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml index 0c8733a9..d882087b 100644 --- a/.github/workflows/build-multiarch-image.yml +++ b/.github/workflows/build-multiarch-image.yml @@ -63,17 +63,17 @@ jobs: env: IS_CI_AUTOMATION: "yes" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build and push image by digest id: build @@ -111,7 +111,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }} path: ${{ runner.temp }}/digests/* @@ -128,17 +128,17 @@ jobs: image: ${{ steps.merge.outputs.image }} steps: - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Download digests - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/digests pattern: digests-${{ inputs.artifact-prefix }}-* diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 749e4856..4aee2a67 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -33,9 +33,9 @@ jobs: keycloak-changed: ${{ steps.check.outputs.keycloak-changed }} version: ${{ steps.read-version.outputs.value }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: check with: filters: | @@ -73,7 +73,7 @@ jobs: - name: Get version from pyproject.toml id: read-version - uses: SebRollen/toml-action@v1.2.0 + uses: SebRollen/toml-action@b1b3628f55fc3a28208d4203ada8b737e9687876 # v1.2.0 with: file: 'pyproject.toml' field: 'project.version'