Upgrade v0.2.14

.github/workflows/build-upgrades.yml

@@ -40,4 +40,4 @@ jobs:
           df -h
 
       - name: Run Makefile build
-        run: make build-for-upgrade
+        run: make build-for-upgrade

.github/workflows/publish_upgrade_binaries.yml

@@ -62,4 +62,4 @@ jobs:
         uses: actions/upload-artifact@v4 # Using v4, v3 also works
         with:
           name: release-tag-artifact # This name will be used to download it later
-          path: release_tag.txt
+          path: release_tag.txt

.github/workflows/test-workflow.yml

@@ -164,6 +164,7 @@ jobs:
         env:
           GENESIS_OVERRIDES_FILE: "inference-chain/test_genesis_overrides.json"
           USE_REGISTRY_CACHE: "1"
+          GHCR_CACHE_NAMESPACE: ${{ github.repository }}
         run: |
           make build-docker
           # devshardd self-checks build_version == the version versiond forces (dev via build/devshard-version).

.github/workflows/testermint-upgrade-rehearsal.yml

@@ -0,0 +1,279 @@
+name: Testermint Upgrade Rehearsal
+
+on:
+  workflow_dispatch:
+    inputs:
+      candidate_ref:
+        description: 'Candidate ref to test. Defaults to the dispatch ref.'
+        required: false
+        type: string
+        default: ''
+      target_upgrade:
+        description: 'Target upgrade name, e.g. v0.2.14. Defaults to newest semantic UpgradeName.'
+        required: false
+        type: string
+        default: ''
+      previous_release:
+        description: 'Previous canonical release, e.g. release/v0.2.13. Defaults to highest release below target.'
+        required: false
+        type: string
+        default: ''
+
+  push:
+    branches:
+      - upgrade-v*
+
+concurrency:
+  group: testermint-upgrade-rehearsal-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  packages: read
+  checks: write
+  actions: read
+
+jobs:
+  upgrade-rehearsal:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 240
+
+    env:
+      NEW_DIR: ${{ github.workspace }}/new
+      OLD_DIR: ${{ github.workspace }}/old
+      PREP_MANIFEST: ${{ github.workspace }}/upgrade-rehearsal-manifest.json
+      COMPLETE_MANIFEST: ${{ github.workspace }}/upgrade-rehearsal-completion-manifest.json
+
+    steps:
+      - name: Check initial disk space
+        run: df -h
+
+      - name: Free up runner disk space
+        timeout-minutes: 8
+        run: |
+          set -euo pipefail
+          for path in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do
+            if [[ -e "$path" ]]; then
+              echo "Removing $path"
+              timeout 150s sudo rm -rf "$path" || echo "Timed out removing $path; continuing"
+              df -h
+            fi
+          done
+          timeout 120s sudo docker system prune -af --volumes || echo "Docker prune timed out; continuing"
+          sudo apt-get clean
+          df -h
+
+      - name: Checkout candidate
+        uses: actions/checkout@v4
+        with:
+          path: new
+          fetch-depth: 0
+          ref: ${{ inputs.candidate_ref || github.ref }}
+
+      - name: Fetch candidate tags
+        working-directory: new
+        run: git fetch --tags --force
+
+      - name: Resolve rehearsal versions
+        id: versions
+        working-directory: new
+        run: |
+          python3 scripts/upgrade-rehearsal/resolve_versions.py \
+            --repo . \
+            --target-upgrade '${{ inputs.target_upgrade }}' \
+            --previous-release '${{ inputs.previous_release }}'
+
+      - name: Checkout previous release
+        uses: actions/checkout@v4
+        with:
+          path: old
+          fetch-depth: 0
+          ref: ${{ steps.versions.outputs.previous_release }}
+
+      - name: Set up Docker
+        uses: docker/setup-docker-action@v4
+        with:
+          version: '28.5.2'
+
+      - name: Verify Docker
+        run: |
+          docker --version
+          docker compose version
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install system packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y make zip unzip jq
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Set up Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-read-only: false
+
+      - name: Resolve previous production images
+        id: images
+        working-directory: old
+        run: python3 ../new/scripts/upgrade-rehearsal/resolve_previous_images.py --compose deploy/join/docker-compose.yml
+
+      - name: Pull and retag previous images for local Testermint
+        working-directory: ${{ github.workspace }}
+        env:
+          PREVIOUS_NODE_IMAGE: ${{ steps.images.outputs.node_image }}
+          PREVIOUS_API_IMAGE: ${{ steps.images.outputs.api_image }}
+          PREVIOUS_PROXY_IMAGE: ${{ steps.images.outputs.proxy_image }}
+          PREVIOUS_VERSIOND_IMAGE: ${{ steps.images.outputs.versiond_image }}
+          TARGET_NODE_IMAGE: ${{ steps.images.outputs.node_target_image }}
+          TARGET_API_IMAGE: ${{ steps.images.outputs.api_target_image }}
+          TARGET_PROXY_IMAGE: ${{ steps.images.outputs.proxy_target_image }}
+          TARGET_VERSIOND_IMAGE: ${{ steps.images.outputs.versiond_target_image }}
+        run: ./new/scripts/upgrade-rehearsal/prepare_previous_images.sh
+
+      - name: Build previous mock-server image
+        working-directory: old
+        run: make mock-server-build-docker
+
+      - name: Apply old-checkout prep test patch
+        working-directory: old
+        run: git apply ../new/testermint/upgrade-rehearsal/previous-release-prep.patch
+
+      - name: Run old-checkout prep test
+        timeout-minutes: 75
+        working-directory: old/testermint
+        env:
+          GONKA_REPO_ROOT: ${{ env.OLD_DIR }}
+          PREVIOUS_RELEASE: ${{ steps.versions.outputs.previous_release }}
+          UPGRADE_REHEARSAL_MANIFEST: ${{ env.PREP_MANIFEST }}
+        run: |
+          sudo -E ./gradlew test \
+            --tests "UpgradeRehearsalPrepTests.prepare upgrade rehearsal state" \
+            -x mock_server:test \
+            --stacktrace
+
+      - name: Build candidate upgrade archives
+        timeout-minutes: 90
+        working-directory: new
+        env:
+          # The default GitHub runner Docker driver cannot export registry
+          # caches. Keep this deterministic over fast; the rehearsal is already
+          # expensive and cache writes are not part of what it proves.
+          USE_REGISTRY_CACHE: '0'
+          VERSION: ${{ steps.versions.outputs.target_upgrade }}
+        run: |
+          make build-for-upgrade
+          sudo chown -R "$USER:$USER" public-html || true
+          find public-html -maxdepth 4 -type f -print
+
+      - name: Serve candidate upgrade archives on chain-public
+        working-directory: ${{ github.workspace }}
+        run: |
+          docker rm -f upgrade-binary-server || true
+          docker run -d \
+            --name upgrade-binary-server \
+            --network chain-public \
+            -v "${NEW_DIR}/public-html:/usr/local/apache2/htdocs:ro" \
+            httpd:2.4
+          docker run --rm --network chain-public curlimages/curl:8.11.1 \
+            -fsS http://upgrade-binary-server/v2/inferenced/inferenced-amd64.zip >/dev/null
+          docker run --rm --network chain-public curlimages/curl:8.11.1 \
+            -fsS http://upgrade-binary-server/v2/dapi/decentralized-api-amd64.zip >/dev/null
+
+      - name: Run current-checkout upgrade completion test
+        working-directory: new/testermint
+        env:
+          GONKA_REPO_ROOT: ${{ env.NEW_DIR }}
+          UPGRADE_REHEARSAL_TARGET: ${{ steps.versions.outputs.target_upgrade }}
+          UPGRADE_REHEARSAL_MANIFEST: ${{ env.PREP_MANIFEST }}
+          UPGRADE_REHEARSAL_COMPLETE_MANIFEST: ${{ env.COMPLETE_MANIFEST }}
+          UPGRADE_BINARY_BASE_URL: http://upgrade-binary-server
+        run: |
+          sudo -E ./gradlew test \
+            --tests "UpgradeRehearsalTests.complete upgrade rehearsal" \
+            -x mock_server:test \
+            --stacktrace
+
+      - name: Capture Docker state
+        if: always()
+        working-directory: ${{ github.workspace }}
+        run: |
+          mkdir -p upgrade-rehearsal-diagnostics
+          docker ps -a > upgrade-rehearsal-diagnostics/docker-ps.txt
+          docker images > upgrade-rehearsal-diagnostics/docker-images.txt
+          for container in genesis-node genesis-api join1-node join1-api join2-node join2-api upgrade-binary-server; do
+            docker logs "$container" --tail 300 > "upgrade-rehearsal-diagnostics/${container}.log" 2>&1 || true
+          done
+
+      - name: Upload rehearsal artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: testermint-upgrade-rehearsal-artifacts
+          if-no-files-found: ignore
+          retention-days: 14
+          path: |
+            upgrade-rehearsal-manifest.json
+            upgrade-rehearsal-completion-manifest.json
+            upgrade-rehearsal-diagnostics/
+            old/testermint/logs/
+            old/testermint/build/test-results/**/*.xml
+            new/testermint/logs/
+            new/testermint/build/test-results/**/*.xml
+
+      - name: Publish old prep test results
+        if: always()
+        uses: dorny/test-reporter@v1
+        continue-on-error: true
+        with:
+          name: Upgrade Rehearsal Prep
+          path: old/testermint/build/test-results/**/*.xml
+          reporter: java-junit
+          fail-on-error: false
+
+      - name: Publish completion test results
+        if: always()
+        uses: dorny/test-reporter@v1
+        continue-on-error: true
+        with:
+          name: Upgrade Rehearsal Completion
+          path: new/testermint/build/test-results/**/*.xml
+          reporter: java-junit
+          fail-on-error: false
+
+      - name: Write workflow summary
+        if: always()
+        run: |
+          {
+            echo "# Testermint Upgrade Rehearsal"
+            echo
+            echo "- Target upgrade: ${{ steps.versions.outputs.target_upgrade }}"
+            echo "- Previous release: ${{ steps.versions.outputs.previous_release }}"
+            echo "- Previous node image: ${{ steps.images.outputs.node_image }}"
+            echo "- Previous API image: ${{ steps.images.outputs.api_image }}"
+            echo
+            if [[ -f "${PREP_MANIFEST}" ]]; then
+              echo "## Prep Manifest"
+              echo '```json'
+              cat "${PREP_MANIFEST}"
+              echo
+              echo '```'
+            fi
+            if [[ -f "${COMPLETE_MANIFEST}" ]]; then
+              echo "## Completion Manifest"
+              echo '```json'
+              cat "${COMPLETE_MANIFEST}"
+              echo
+              echo '```'
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -55,4 +55,4 @@ actionlint
 inference-chain/proto/github.com/
 inference-chain/proto/api/
 inference-chain/proto/**/module/*.pb.go
-inference-chain/.docker-context
+/inference-chain/.docker-context/

Makefile

@@ -1,4 +1,9 @@
-.PHONY: release decentralized-api-release inference-chain-release tmkms-release proxy-release proxy-ssl-release bridge-release versiond-release check-docker build-testermint run-blockchain-tests test-blockchain local-build api-local-build node-local-build api-test node-test mock-server-build-docker proxy-build-docker proxy-ssl-build-docker bridge-build-docker run-bls-tests devshardctl-build devshardd-build print-devshard-version print-devshard-protocol-version versiond-build-docker testapp-server-build-docker
+.PHONY: release decentralized-api-release inference-chain-release tmkms-release proxy-release proxy-ssl-release bridge-release versiond-release check-docker build-testermint run-blockchain-tests test-blockchain local-build api-local-build node-local-build api-test node-test mock-server-build-docker proxy-build-docker proxy-ssl-build-docker bridge-build-docker run-bls-tests devshardctl-build devshardd-build devshardd-release print-devshard-version print-devshard-protocol-version versiond-build-docker testapp-server-build-docker
+
+# For binary release: default linux/amd64 before local Docker defaults.
+DEVSHARDD_RELEASE_DOCKER_PLATFORM := $(if $(DOCKER_PLATFORM),$(DOCKER_PLATFORM),linux/amd64)
+DEVSHARDD_RELEASE_DOCKER_GOOS := $(if $(DOCKER_GOOS),$(DOCKER_GOOS),linux)
+DEVSHARDD_RELEASE_DOCKER_GOARCH := $(if $(DOCKER_GOARCH),$(DOCKER_GOARCH),$(if $(filter linux/arm64,$(DEVSHARDD_RELEASE_DOCKER_PLATFORM)),arm64,amd64))
 
 include scripts/blst-portable.mk
 
@@ -15,10 +20,15 @@ print-devshard-protocol-version:
 	@echo $(DEVSHARD_PROTOCOL_VERSION)
 TAG_NAME := "release/v$(VERSION)"
 USE_REGISTRY_CACHE ?= 0
+GHCR_CACHE_NAMESPACE ?= gonka-ai
+PLATFORM ?= linux/amd64
+GOOS ?= linux
+GOARCH ?= amd64
+DEVSHARDD_RELEASE_DIR ?= build/devshardd-release
 ifeq ($(USE_REGISTRY_CACHE),1)
-_MOCK_CACHE_ARGS := --cache-from type=registry,ref=ghcr.io/gonka-ai/mock-server:buildcache --cache-to type=registry,ref=ghcr.io/gonka-ai/mock-server:buildcache,mode=min
+_MOCK_CACHE_ARGS := --cache-from type=registry,ref=ghcr.io/$(GHCR_CACHE_NAMESPACE)/mock-server:buildcache --cache-to type=registry,ref=ghcr.io/$(GHCR_CACHE_NAMESPACE)/mock-server:buildcache,mode=min
 _MOCK_BUILD_CMD := docker buildx build --load $(_MOCK_CACHE_ARGS)
-_DEVSHARDD_CACHE_ARGS := --cache-from type=registry,ref=ghcr.io/gonka-ai/devshardd:buildcache --cache-to type=registry,ref=ghcr.io/gonka-ai/devshardd:buildcache,mode=min
+_DEVSHARDD_CACHE_ARGS := --cache-from type=registry,ref=ghcr.io/$(GHCR_CACHE_NAMESPACE)/devshardd:buildcache --cache-to type=registry,ref=ghcr.io/$(GHCR_CACHE_NAMESPACE)/devshardd:buildcache,mode=min
 _DEVSHARDD_BUILD_CMD := docker buildx build --load $(_DEVSHARDD_CACHE_ARGS)
 else
 _MOCK_CACHE_ARGS :=
@@ -154,6 +164,29 @@ devshardd-build:
 	@echo "$(DEVSHARD_PROTOCOL_VERSION)" > build/devshard-protocol-version
 	@echo "Built build/devshardd ($$(file build/devshardd | grep -o 'statically linked\|dynamically linked'))"
 
+devshardd-release:
+	@$(MAKE) devshardd-build \
+		DEVSHARD_VERSION=$(DEVSHARD_VERSION) \
+		DEVSHARD_PROTOCOL_VERSION=$(DEVSHARD_PROTOCOL_VERSION) \
+		DOCKER_PLATFORM=$(DEVSHARDD_RELEASE_DOCKER_PLATFORM) \
+		DOCKER_GOOS=$(DEVSHARDD_RELEASE_DOCKER_GOOS) \
+		DOCKER_GOARCH=$(DEVSHARDD_RELEASE_DOCKER_GOARCH)
+	@set -e; \
+		release_dir="$(DEVSHARDD_RELEASE_DIR)"; \
+		if [ -z "$$release_dir" ] || [ "$$release_dir" = "/" ]; then \
+			echo "Refusing to clean unsafe DEVSHARDD_RELEASE_DIR='$$release_dir'"; \
+			exit 1; \
+		fi; \
+		rm -rf "$$release_dir"; \
+		mkdir -p "$$release_dir/stage"; \
+		cp build/devshardd "$$release_dir/stage/devshardd"; \
+		chmod 0755 "$$release_dir/stage/devshardd"; \
+		(cd "$$release_dir/stage" && zip -X -q ../devshardd.zip devshardd); \
+		rm -rf "$$release_dir/stage"; \
+		shasum -a 256 "$$release_dir/devshardd.zip" | awk '{print $$1}' > "$$release_dir/devshardd.zip.sha256"; \
+		echo "Built $$release_dir/devshardd.zip"; \
+		echo "SHA256: $$(cat "$$release_dir/devshardd.zip.sha256")"
+
 node-local-build:
 	@echo "Building inference-chain locally..."
 	@make -C inference-chain build
@@ -215,11 +248,15 @@ local-build: api-local-build node-local-build api-test node-test
 	@rm -f api-test-output.log node-test-output.log
 
 build-for-upgrade:
-	@rm public-html/v2/checksums.txt || true
-	@rm public-html/v2/urls.txt || true
-	@make -C inference-chain build-for-upgrade
-	@make -C decentralized-api build-for-upgrade
+	@rm -f public-html/v2/checksums.txt public-html/v2/urls.txt
+	@mkdir -p public-html/v2/inferenced public-html/v2/dapi
+	@rm -f public-html/v2/inferenced/inferenced-*.zip public-html/v2/dapi/decentralized-api-*.zip
+	@make -C inference-chain build-for-upgrade PLATFORM=linux/amd64 GOOS=linux GOARCH=amd64
+	@make -C decentralized-api build-for-upgrade PLATFORM=linux/amd64 GOOS=linux GOARCH=amd64
 
 build-for-upgrade-tests:
-	@make -C inference-chain build-for-upgrade TESTS=1
-	@make -C decentralized-api build-for-upgrade TESTS=1
+	@rm -f public-html/v2/checksums.txt public-html/v2/urls.txt
+	@mkdir -p public-html/v2/inferenced public-html/v2/dapi
+	@rm -f public-html/v2/inferenced/inferenced-*.zip public-html/v2/dapi/decentralized-api-*.zip
+	@make -C inference-chain build-for-upgrade TESTS=1 PLATFORM=linux/amd64 GOOS=linux GOARCH=amd64
+	@make -C decentralized-api build-for-upgrade TESTS=1 PLATFORM=linux/amd64 GOOS=linux GOARCH=amd64