CI: decouple multi-arch image publishing into a standalone GHCR pipeline

.github/workflows/build-multiarch-image.yml

@@ -0,0 +1,165 @@
+---
+# Reusable workflow: build ONE container image for linux/amd64 and linux/arm64
+# on NATIVE runners (no QEMU), push each arch to GHCR by digest, then stitch the
+# digests into a single multi-arch manifest list tagged :<sha>, :v<version> and
+# :latest. Callers get the fully-qualified :<sha> reference back via the `image`
+# output. Invoked by publish-images.yml for the accounts and keycloak images.
+#
+# Why native runners instead of `buildx --platform amd64,arm64` with QEMU:
+# emulated arm64 builds are minutes-slower and occasionally miscompile. Each
+# arch builds on its own hardware and the two run in parallel.
+#
+# Auth is GHCR + the automatic GITHUB_TOKEN (packages: write) -- no cloud
+# credentials, which keeps this pipeline decoupled from the ECR/Pulumi deploy.
+name: build-multiarch-image
+
+on:
+  workflow_call:
+    inputs:
+      image:
+        description: Fully-qualified GHCR image name WITHOUT a tag (e.g. ghcr.io/owner/name).
+        required: true
+        type: string
+      dockerfile:
+        description: Path to the Dockerfile to build.
+        required: false
+        type: string
+        default: Dockerfile
+      version:
+        description: Semantic version (no leading v) published as the :v<version> tag.
+        required: true
+        type: string
+      artifact-prefix:
+        description: >-
+          Unique prefix for this image's digest artifacts. Distinct images built
+          in the same run (accounts, keycloak) MUST use different prefixes so
+          their per-arch digest artifacts don't collide.
+        required: true
+        type: string
+    outputs:
+      image:
+        description: The :<sha>-tagged multi-arch image reference that was pushed.
+        value: ${{ jobs.merge.outputs.image }}
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  # One job per architecture, each on its matching native runner. Each pushes an
+  # anonymous (tag-less) image to GHCR and records its digest as an artifact.
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            arch: amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    env:
+      IS_CI_AUTOMATION: "yes"
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Log in to GHCR
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Build and push image by digest
+        id: build
+        run: |
+          # Build this single platform and push it to GHCR by digest only (no
+          # tag). push-by-digest keeps the pushed image anonymous so both arch
+          # runners can push concurrently without clobbering a shared tag; the
+          # merge job assigns the real tags afterwards. --metadata-file lets us
+          # read back the digest that GHCR assigned.
+          # --provenance=false keeps each per-arch push a single plain manifest
+          # (no attestation index), so the merged manifest list contains exactly
+          # the two platform entries.
+          # GitHub Actions layer cache, scoped per image+arch (these runners are
+          # ephemeral, so nothing persists between runs without it). The scope
+          # keeps accounts/keycloak and amd64/arm64 caches from colliding.
+          docker buildx build \
+            -f "${{ inputs.dockerfile }}" \
+            --platform "${{ matrix.platform }}" \
+            --provenance=false \
+            --cache-from "type=gha,scope=${{ inputs.artifact-prefix }}-${{ matrix.arch }}" \
+            --cache-to "type=gha,mode=max,scope=${{ inputs.artifact-prefix }}-${{ matrix.arch }}" \
+            --output "type=image,name=${{ inputs.image }},push-by-digest=true,name-canonical=true,push=true" \
+            --metadata-file metadata.json \
+            .
+          digest="$(jq -r '."containerimage.digest"' metadata.json)"
+          echo "digest=$digest" >> "$GITHUB_OUTPUT"
+
+      - name: Export digest
+        run: |
+          # Record the digest as an empty file named after the sha256 hex, so
+          # the merge job can reconstruct `${IMAGE}@sha256:<hex>` references by
+          # listing the downloaded artifact directory.
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: digests-${{ inputs.artifact-prefix }}-${{ matrix.arch }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # Combine the per-arch digests into one manifest list under all three tags.
+  merge:
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      IS_CI_AUTOMATION: "yes"
+    outputs:
+      image: ${{ steps.merge.outputs.image }}
+    steps:
+      - name: Log in to GHCR
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Download digests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-${{ inputs.artifact-prefix }}-*
+          merge-multiple: true
+
+      - name: Create manifest list and push
+        id: merge
+        working-directory: ${{ runner.temp }}/digests
+        env:
+          IMAGE: ${{ inputs.image }}
+          VERSION: ${{ inputs.version }}
+          SHA: ${{ github.sha }}
+        run: |
+          # imagetools create references digests already in GHCR, so nothing is
+          # rebuilt or re-pushed here -- it only writes the manifest list, under
+          # all three tags at once. The glob expands to one
+          # `${IMAGE}@sha256:<hex>` per downloaded digest.
+          docker buildx imagetools create \
+            -t "${IMAGE}:${SHA}" \
+            -t "${IMAGE}:v${VERSION}" \
+            -t "${IMAGE}:latest" \
+            $(printf "${IMAGE}@sha256:%s " *)
+          docker buildx imagetools inspect "${IMAGE}:${SHA}"
+          echo "image=${IMAGE}:${SHA}" >> "$GITHUB_OUTPUT"

.github/workflows/merge.yml

@@ -35,7 +35,8 @@ jobs:
               - 'assets/**'
               - 'static/**'
               - 'templates/**'
-              - 'scripts/entry.sh'
+              # Dockerfile COPYs the whole scripts/ dir.
+              - 'scripts/**'
               - 'Dockerfile'
               - 'manage.py'
               - 'MANIFEST.in'
@@ -45,7 +46,15 @@ jobs:
               - 'README.md'
               - 'src/**'
               - 'uv.lock'
-              - 'vite.config.mjs'
+              # Frontend build inputs COPYd by the Dockerfile (vite.config is
+              # .mts, not .mjs); tsconfig.json and env.d.ts were missing.
+              - 'vite.config.mts'
+              - 'tsconfig.json'
+              - 'env.d.ts'
+              # The accounts bundle imports the shared keycloak theme via the
+              # `@kc` vite alias, so `npm run build` compiles it into the accounts
+              # image -- theme edits must rebuild (and redeploy) accounts too.
+              - 'keycloak/themes/**'
             keycloak-theme-changed:
               - 'keycloak/themes/**'
               - 'Dockerfile.keycloak'
@@ -118,29 +127,18 @@ jobs:
           curl -fsSL https://get.pulumi.com | sh
           pip install -Ur requirements.txt
 
-      # Both the accounts and keycloak images are built multi-arch via Buildx +
-      # QEMU (linux/amd64 for ECS Fargate today, linux/arm64 for the Thunderbird
-      # Pro EKS clusters on Graviton). Each <tag> is a manifest list, so every
-      # runtime pulls its matching variant. Set these up whenever either image
-      # is (re)built; they must run before the build steps below.
-      - name: Set up QEMU
-        if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true'
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        if: env.BUILD_ACCOUNTS == 'true' || env.BUILD_KEYCLOAK == 'true'
-        uses: docker/setup-buildx-action@v3
-
       # Produce the accounts container image (only when accounts/IaC changed).
+      # This image is amd64-only (ECS Fargate). Multi-arch (amd64 + arm64)
+      # publishing lives in the decoupled publish-images.yml -> GHCR pipeline.
       - name: Build, tag, and push accounts image to Amazon ECR
         id: build-accounts
         if: env.BUILD_ACCOUNTS == 'true'
         env:
           ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:${{ github.sha }}"
         run: |
-          # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the
-          # manifest list to ECR. buildx --push publishes directly (no docker push).
-          docker buildx build -t $ECR_TAG \
-            --platform linux/amd64,linux/arm64 --push .
+          # Build a docker container and push it to ECR so that it can be deployed to ECS.
+          docker build -t $ECR_TAG --platform="linux/amd64" .
+          docker push $ECR_TAG
           echo "accounts-image=$ECR_TAG" >> $GITHUB_OUTPUT
 
       # Produce the keycloak container image (only when keycloak/IaC changed).
@@ -150,10 +148,9 @@ jobs:
         env:
           ECR_TAG: "${{ steps.login-ecr.outputs.registry }}/${{ vars.PROJECT }}:keycloak-${{ github.sha }}"
         run: |
-          # Build a multi-arch image (amd64 for ECS, arm64 for EKS) and push the
-          # manifest list to ECR. buildx --push publishes directly (no docker push).
-          docker buildx build -f Dockerfile.keycloak -t $ECR_TAG \
-            --platform linux/amd64,linux/arm64 --push .
+          # Build a docker container and push it to ECR so that it can be deployed to ECS.
+          docker build -f Dockerfile.keycloak -t $ECR_TAG --platform="linux/amd64" .
+          docker push $ECR_TAG
           echo "keycloak-image=$ECR_TAG" >> $GITHUB_OUTPUT
 
       - name: Get version from pyproject.toml

.github/workflows/publish-images.yml

@@ -0,0 +1,104 @@
+---
+# Decoupled image-publish pipeline. On merge to main it builds the accounts and
+# keycloak images as native multi-arch (amd64 + arm64) manifest lists and pushes
+# them to GHCR. This is INDEPENDENT of the deploy-stage pipeline (merge.yml):
+# it uses no cloud credentials, runs no Pulumi, and deploys nothing -- it only
+# publishes images. See docs/ci/image-publish-pipeline.md for the architecture.
+name: publish-images
+
+concurrency:
+  group: publish-images-${{ github.ref }}
+  # Serialize (don't cancel) so every merged commit's :<sha> image gets
+  # published and a run is never aborted mid-push/merge. Queued runs still
+  # settle :latest to the newest finished build.
+  cancel-in-progress: false
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  # Separate job: decide which image(s) changed and read the version once, so
+  # the build jobs below only run for images that actually changed.
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      accounts-changed: ${{ steps.check.outputs.accounts-changed }}
+      keycloak-changed: ${{ steps.check.outputs.keycloak-changed }}
+      version: ${{ steps.read-version.outputs.value }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        id: check
+        with:
+          filters: |
+            accounts-changed:
+              - 'assets/**'
+              - 'static/**'
+              - 'templates/**'
+              # The accounts frontend bundle imports the shared keycloak theme
+              # via the `@kc` vite alias (-> keycloak/themes/tbpro/assets), so
+              # `npm run build` compiles it into the accounts image. Theme edits
+              # must therefore rebuild accounts too.
+              - 'keycloak/themes/**'
+              # The accounts Dockerfile COPYs the whole scripts/ dir.
+              - 'scripts/**'
+              - 'Dockerfile'
+              - 'manage.py'
+              - 'MANIFEST.in'
+              - 'package.json'
+              - 'package-lock.json'
+              - 'pyproject.toml'
+              - 'src/**'
+              - 'uv.lock'
+              # Frontend build inputs COPYd by the Dockerfile (note: .mts, plus
+              # tsconfig.json and env.d.ts).
+              - 'vite.config.mts'
+              - 'tsconfig.json'
+              - 'env.d.ts'
+            keycloak-changed:
+              - 'keycloak/**'
+              - 'Dockerfile.keycloak'
+              - 'package.json'
+              - 'package-lock.json'
+              - 'scripts/entry-keycloak.sh'
+              - 'scripts/apply-mfa-config.sh'
+
+      - name: Get version from pyproject.toml
+        id: read-version
+        uses: SebRollen/toml-action@b1b3628f55fc3a28208d4203ada8b737e9687876 # v1.2.0
+        with:
+          file: 'pyproject.toml'
+          field: 'project.version'
+
+  accounts:
+    needs: detect-changes
+    # workflow_dispatch always rebuilds; pushes rebuild only on a relevant change.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.accounts-changed == 'true'
+    uses: ./.github/workflows/build-multiarch-image.yml
+    with:
+      image: ghcr.io/thunderbird/thunderbird-accounts
+      dockerfile: Dockerfile
+      version: ${{ needs.detect-changes.outputs.version }}
+      artifact-prefix: accounts
+
+  keycloak:
+    needs: detect-changes
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.keycloak-changed == 'true'
+    uses: ./.github/workflows/build-multiarch-image.yml
+    with:
+      image: ghcr.io/thunderbird/thunderbird-accounts-keycloak
+      dockerfile: Dockerfile.keycloak
+      version: ${{ needs.detect-changes.outputs.version }}
+      artifact-prefix: keycloak