Add rolling-safe HA for versiond routing

.github/workflows/devshard-testenv.yml

@@ -10,6 +10,13 @@ on:
       - 'common/runtimeconfig/**'
       - 'common/chain/**'
       - 'decentralized-api/nodemanager/**'
+      - 'versioned/**'
+      - 'versiond-router/**'
+      - 'proxy-router/**'
+      - 'proxy/**'
+      - 'deploy/join/docker-compose.yml'
+      - 'deploy/join/docker-compose.versiond.yml'
+      - 'deploy/join/docker-compose.edge-api-multi.yml'
       - '.github/workflows/devshard-testenv.yml'
   workflow_dispatch:
     inputs:
@@ -104,6 +111,9 @@ jobs:
           go-version: '1.25.9'
           cache-dependency-path: devshard/go.sum
 
+      - name: Generate test environment
+        run: make gen-compose
+
       # Each matrix job runs on its own runner with its own Docker daemon, so the
       # fixed citest subnet (172.31.0.0/24) never collides across suites. Suites
       # run in parallel across runners; tests within a suite stay sequential.

.github/workflows/integration.yml

@@ -19,6 +19,7 @@ on:
       - 'local-test-net/**'
       - 'testermint/**'
       - 'versioned/**'
+      - 'versiond-router/**'
       - 'devshard/**'
 
   issue_comment:
@@ -53,4 +54,4 @@ jobs:
       is_pr_comment: ${{ github.event_name == 'issue_comment' }}
       pr_number: ${{ github.event.issue.number || '' }}
       enable_upgrades: true
-      use_matrix: ${{ github.event_name == 'workflow_dispatch' && inputs.use_matrix || true }}
+      use_matrix: ${{ github.event_name == 'workflow_dispatch' && inputs.use_matrix || true }}

.github/workflows/join-postgres-upgrade.yml

@@ -0,0 +1,55 @@
+name: Join Postgres upgrade
+
+on:
+  pull_request:
+    paths:
+      - deploy/join/docker-compose.versiond.yml
+      - deploy/join/docker-compose.versiond-postgres-recovery.yml
+      - deploy/join/devshard-postgres-entrypoint.sh
+      - deploy/join/devshard-postgres-entrypoint_test.sh
+      - deploy/join/devshard-postgres-migration-preflight.sh
+      - deploy/join/devshard-postgres-migration-preflight_test.sh
+      - deploy/join/devshard-postgres-upgrade_test.sh
+      - deploy/join/testdata/postgres-v4.compose.yml
+      - deploy/join/testdata/postgres-v5.compose.yml
+      - .github/workflows/join-postgres-upgrade.yml
+  push:
+    branches: [main]
+    paths:
+      - deploy/join/docker-compose.versiond.yml
+      - deploy/join/docker-compose.versiond-postgres-recovery.yml
+      - deploy/join/devshard-postgres-entrypoint.sh
+      - deploy/join/devshard-postgres-entrypoint_test.sh
+      - deploy/join/devshard-postgres-migration-preflight.sh
+      - deploy/join/devshard-postgres-migration-preflight_test.sh
+      - deploy/join/devshard-postgres-upgrade_test.sh
+      - deploy/join/testdata/postgres-v4.compose.yml
+      - deploy/join/testdata/postgres-v5.compose.yml
+      - .github/workflows/join-postgres-upgrade.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test-upgrade:
+    name: Preserve v4 PostgreSQL data
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check deployment scripts
+        run: |
+          shellcheck \
+            deploy/join/devshard-postgres-entrypoint.sh \
+            deploy/join/devshard-postgres-entrypoint_test.sh \
+            deploy/join/devshard-postgres-migration-preflight.sh \
+            deploy/join/devshard-postgres-migration-preflight_test.sh \
+            deploy/join/devshard-postgres-upgrade_test.sh
+          deploy/join/devshard-postgres-entrypoint_test.sh
+          deploy/join/devshard-postgres-migration-preflight_test.sh
+
+      - name: Test in-place v4 migration and persistent restart
+        run: deploy/join/devshard-postgres-upgrade_test.sh

.github/workflows/release-image-smoke.yml

@@ -0,0 +1,140 @@
+name: Release image smoke
+
+on:
+  pull_request:
+    paths:
+      - deploy/join/docker-compose.yml
+      - deploy/join/docker-compose.versiond.yml
+      - deploy/join/docker-compose.versiond-external-postgres.yml
+      - deploy/join/docker-compose.edge-api-multi.yml
+      - deploy/join/docker-compose.observability.yml
+      - deploy/join/docker-compose.versiond-v5-compat.yml
+      - deploy/join/docker-compose.edge-api-v5-compat.yml
+      - deploy/join/docker-compose.proxy-v4-compat.yml
+      - deploy/join/versiond-router-slot/docker-compose.yml
+      - deploy/join/release-image-smoke_test.sh
+      - deploy/join/devshard-postgres-migration-preflight.sh
+      - deploy/join/devshard-postgres-migration-preflight_test.sh
+      - deploy/join/legacy-router-upgrade-barrier.sh
+      - deploy/join/legacy-router-upgrade-barrier_test.sh
+      - deploy/join/upgrade-devshard-v5.sh
+      - deploy/join/upgrade-devshard-v5_test.sh
+      - deploy/join/devshard-v5-release.env
+      - deploy/join/devshard-v5-release-contract.sh
+      - deploy/join/devshard-v5-release-contract_test.sh
+      - deploy/join/devshard-v5-release-gate.sh
+      - deploy/join/devshard-v5-release-gate_test.sh
+      - devshard/docs/network-update-0.2.15-v5.md
+      - devshard/docs/release-0.2.15-v5.md
+      - deploy/join/compose-topology.sh
+      - deploy/join/deployment-lock.sh
+      - deploy/join/enable-router-ha.sh
+      - deploy/join/enable-router-ha_test.sh
+      - deploy/join/versiond-compose-config_test.sh
+      - deploy/join/versiond-router-fleet.sh
+      - deploy/join/versiond-router-fleet_test.sh
+      - proxy/Dockerfile
+      - proxy/entrypoint.sh
+      - proxy/nginx.unified.conf.template
+      - proxy-router/**
+      - router-runtime/**
+      - versiond-router/Dockerfile
+      - versiond-router/entrypoint.sh
+      - versiond-router/haproxy.cfg.template
+      - .github/workflows/release-image-smoke.yml
+  workflow_dispatch:
+    inputs:
+      network_update_url:
+        description: 'Published gonka.ai Network Updates entry for this release'
+        required: true
+        type: string
+      upgrade_deadline_utc:
+        description: 'Published upgrade deadline (UTC RFC3339)'
+        required: true
+        type: string
+  schedule:
+    # Published immutable tags are checked continuously after release, not only
+    # when an operator remembers to dispatch the workflow.
+    - cron: '17 3 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  compose-contract:
+    name: Compose release image contract
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check base and HA image pins
+        run: deploy/join/release-image-smoke_test.sh --contract
+
+      - name: Check fail-safe upgrade workflow
+        run: |
+          shellcheck \
+            deploy/join/devshard-postgres-migration-preflight.sh \
+            deploy/join/legacy-router-upgrade-barrier.sh \
+            deploy/join/legacy-router-upgrade-barrier_test.sh \
+            deploy/join/compose-topology.sh \
+            deploy/join/deployment-lock.sh \
+            deploy/join/devshard-v5-release-contract.sh \
+            deploy/join/devshard-v5-release-contract_test.sh \
+            deploy/join/devshard-v5-release-gate.sh \
+            deploy/join/devshard-v5-release-gate_test.sh \
+            deploy/join/upgrade-devshard-v5.sh \
+            deploy/join/upgrade-devshard-v5_test.sh \
+            deploy/join/enable-router-ha.sh \
+            deploy/join/enable-router-ha_test.sh \
+            deploy/join/versiond-router-fleet.sh \
+            deploy/join/versiond-router-fleet_test.sh \
+            deploy/join/versiond-compose-config_test.sh \
+            router-runtime/catalog-cache \
+            router-runtime/catalog-cache_test.sh \
+            router-runtime/catalog-reconciler \
+            router-runtime/catalog-status
+          router-runtime/catalog-cache_test.sh
+          deploy/join/legacy-router-upgrade-barrier_test.sh
+          deploy/join/devshard-v5-release-contract_test.sh
+          deploy/join/devshard-v5-release-gate_test.sh
+          deploy/join/upgrade-devshard-v5_test.sh
+          deploy/join/enable-router-ha_test.sh
+          deploy/join/versiond-compose-config_test.sh
+
+  router-fleet-acceptance:
+    name: Router fleet Docker acceptance
+    needs: compose-contract
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Exercise rolling router fleet lifecycle
+        run: deploy/join/versiond-router-fleet_test.sh
+
+  registry-smoke:
+    name: Published image endpoints
+    needs: compose-contract
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Pull and smoke-test release images
+        run: deploy/join/release-image-smoke_test.sh
+
+      - name: Verify published host update contract
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          NETWORK_UPDATE_URL: ${{ inputs.network_update_url }}
+          UPGRADE_DEADLINE_UTC: ${{ inputs.upgrade_deadline_utc }}
+        run: |
+          deploy/join/devshard-v5-release-gate.sh \
+            --network-update-url "$NETWORK_UPDATE_URL" \
+            --deadline-utc "$UPGRADE_DEADLINE_UTC"

.github/workflows/verify.yml

@@ -224,17 +224,19 @@ jobs:
       - name: Build
         run: go build ./...
 
-      - name: Run Tests (fork PRs)
+      # Host shutdown races readiness promotion, admission leases, child reaping,
+      # and force signals. Keep the detector on the mandatory versioned suite.
+      - name: Run Tests with race detector (fork PRs)
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
-        run: go test ./... -v
+        run: go test -race ./... -v
 
       - name: Install go-junit-report
         if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
         run: go install github.com/jstemmer/go-junit-report/v2@v2.1.0
 
-      - name: Run Tests and Generate JUnit Report
+      - name: Run Tests with race detector and Generate JUnit Report
         if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
-        run: go test ./... -v 2>&1 | go-junit-report -set-exit-code > junit-report.xml
+        run: go test -race ./... -v 2>&1 | go-junit-report -set-exit-code > junit-report.xml
 
       - name: Publish Test Results
         uses: actions/upload-artifact@v4
@@ -251,6 +253,38 @@ jobs:
           path: versioned/junit-report.xml
           reporter: java-junit
 
+  # The routers are HAProxy config plus a render script. test-render renders every
+  # supported shape, asserts on the result, and validates each rendered file with
+  # the real HAProxy from the image the routers ship.
+  test-router-config:
+    runs-on: ubuntu-latest
+    name: Test router config rendering
+    strategy:
+      matrix:
+        router: [versiond-router, edge-api-router, proxy-router]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Render and validate ${{ matrix.router }} config
+        run: make -C ${{ matrix.router }} test-render
+
+      - name: Exercise proxy-router HA routing
+        if: matrix.router == 'proxy-router'
+        run: make -C proxy-router test-routing
+
+      - name: Exercise isolated versiond-router slots
+        if: matrix.router == 'versiond-router'
+        run: make -C versiond-router test-fleet
+
+  test-proxy-policy-config:
+    runs-on: ubuntu-latest
+    name: Test nginx policy rendering
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Render and validate private nginx policy config
+        run: make -C proxy test-render
+
   build-and-test-edge-api:
     runs-on: ubuntu-latest
     name: Build and Test edge-api (validation)
@@ -283,6 +317,13 @@ jobs:
         working-directory: edge-api
         run: go build ./...
 
+      # The module's own unit tests — readiness, the drain sequence, config
+      # validation. The JUnit step below covers common/queryapi only, and for a
+      # while nothing ran these at all.
+      - name: Run edge-api unit tests
+        working-directory: ./edge-api
+        run: go test -race ./...
+
       - name: Run edge-api tests (fork PRs)
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
         run: bash scripts/validate-edge-api.sh

.github/workflows/versiond-host-evacuation.yml

@@ -0,0 +1,59 @@
+# The one suite that exercises the whole Track B contract against a real stack:
+# a host leaves rotation before it stops accepting, its stream finishes, the
+# session recovers on the survivor, and the host rejoins only when ready. It was
+# briefly demoted to the manually-dispatched matrix; review put it back — the
+# properties it pins are exactly the ones a routine-looking change breaks.
+name: versiond host evacuation
+
+on:
+  pull_request:
+    paths:
+      - 'versioned/**'
+      - 'versiond-router/**'
+      - 'proxy-router/**'
+      - 'proxy/**'
+      - 'deploy/join/docker-compose.yml'
+      - 'deploy/join/docker-compose.versiond.yml'
+      - 'deploy/join/versiond-router-slot/**'
+      - 'deploy/join/versiond-router-fleet.sh'
+      - 'common/storage/**'
+      # devshardd and the devshardctl gateway span packages across the module;
+      # storage recovery, routing, state and transport changes all affect this
+      # acceptance contract. Documentation alone does not need the 40m stack.
+      - 'devshard/**'
+      - '!devshard/docs/**'
+      - '!devshard/**/*.md'
+      - '.github/workflows/versiond-host-evacuation.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: versiond-host-evacuation-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  acceptance:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    name: versiond host evacuation
+    defaults:
+      run:
+        working-directory: ./devshard/testenv
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.25.9'
+          cache-dependency-path: devshard/go.sum
+
+      - name: Generate test environment
+        run: make gen-compose
+
+      - name: Build linux devshardd
+        run: make build-devshardd
+
+      - name: Run host evacuation acceptance test
+        run: make citest-versiond-host-evacuation

.gitignore

@@ -41,6 +41,8 @@ prod-local/
 /logdy
 /inference-chain/junit-report.xml
 /deploy/**/config.env
+/deploy/**/.gonka-devshard-v5-upgrade-complete
+/deploy/join/devshards/postgres/
 /testermint/reboot.txt
 /local-test-net/node_payload_2.json
 /testermint/downloads/