main-process-update.yml

# Triger: only call from main workflow(re-usable workflows)
name: main-process-update

on:
  workflow_call:
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  prepare-execution-matrix:
    name: execution matrix

    runs-on: [ci]
    outputs:
      matrix: ${{ steps.create_matrix.outputs.matrix }}

    steps:
      - name: Clean Workspace
        uses: AutoModality/action-clean@v1.1.0

      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4.1.7
        with:
          ref: ${{ github.head_ref }} #Checking out head commit

      - name: Read .env file
        uses: xom9ikk/dotenv@v2.3.0

      - name: Create Execution matrix
        uses: CertainLach/create-matrix-action@v4
        id: create_matrix
        with:
          matrix: |
            network {opal}, symbol {OPL}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, fork_source {${{ env.OPAL_REPLICA_FROM }}}
            network {quartz}, symbol {QTZ}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, fork_source {${{ env.QUARTZ_REPLICA_FROM }}}
            network {unique}, symbol {UNQ}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, fork_source {${{ env.UNIQUE_REPLICA_FROM }}}

  main-process:
    needs: prepare-execution-matrix
    # The type of runner that the job will run on
    runs-on: [ci]

    timeout-minutes: 1380

    name: ${{ matrix.network }}-update

    continue-on-error: true #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.

    strategy:
      matrix:
        include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}}

    steps:
      - name: Clean Workspace
        uses: AutoModality/action-clean@v1.1.0

      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4.1.7
        with:
          ref: ${{ github.head_ref }} #Checking out head commit

      # Prepare SHA
      - name: Prepare SHA
        uses: ./.github/actions/prepare

      - name: Read .env file
        uses: xom9ikk/dotenv@v2.3.0

      - name: Log in to Docker Hub
        uses: docker/login-action@v3.2.0
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: Check and pull polkadot image
        id: polkadot
        uses: cloudposse/github-action-docker-image-exists@main
        with:
          registry: registry.hub.docker.com
          organization: parity
          repository: polkadot
          login: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}
          tag: ${{ matrix.relay_branch }}

      - name: Prepare mainnet
        uses: ./.github/actions/buildContainer
        id: mainnet
        with:
          container: uniquenetwork/ci-main-process
          tag: ${{ matrix.network }}-${{ env.MAINNET_BRANCH }}-${{ env.MAINNET_HASH}}
          context: .docker
          dockerfile: Dockerfile-unique-release
          args: |
            --build-arg FEATURES=${{ matrix.network }}-runtime
            --build-arg CODE_VERSION=${{ env.MAINNET_BRANCH}}
            --build-arg CODE_HASH=${{ env.MAINNET_HASH}}
          dockerhub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: Prepare target ${{ matrix.network }}
        uses: ./.github/actions/buildContainer
        id: target
        with:
          container: uniquenetwork/ci-main-process
          tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }}
          context: .
          dockerfile: .docker/Dockerfile-unique
          args: |
            --build-arg FEATURES=${{ matrix.network }}-runtime
          dockerhub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: Extract wasms
        uses: ./.github/actions/extractDocker
        id: wasms
        with:
          image: ${{ steps.target.outputs.name }}
          directory: /wasm

      # - name: Prepare target node
      #   uses: ./.github/actions/buildContainer
      #   id: node
      #   with:
      #     container: uniquenetwork/ci-main-process
      #     tag: ${{ matrix.network }}-${{ env.NODE_BRANCH }}-${{ env.NODE_HASH}}
      #     context: .docker
      #     dockerfile: Dockerfile-unique-release
      #     args: |
      #       --build-arg FEATURES=${{ matrix.network }}-runtime
      #       --build-arg CODE_VERSION=${{ env.NODE_BRANCH}}
      #       --build-arg CODE_HASH=${{ env.NODE_HASH}}
      #     dockerhub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
      #     dockerhub_token: ${{ secrets.DOCKER_HUB_TOKEN }}

      - uses: denoland/setup-deno@v2
        with:
          deno-version: ${{ env.DENO_VERSION }}
          cache: true

      - name: Install baedeker
        uses: UniqueNetwork/baedeker-action/setup@v1.2
        with:
          version: ${{ env.BAEDEKER_VERSION }}
          useCache: false

      - name: Setup library
        run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library

      - name: Get state from production
        run: |
          . "$HOME/.cargo/env"
          chainql -e 'cql.chain("${{ matrix.fork_source }}").latest._preloadKeys._raw' > .baedeker/vendor/dump.json
        # env:
        #   CHAINQL_WORKERS: 2
        #   CHAINQL_KEYS_CHUNK_SIZE: 20000

      - name: Start network
        uses: UniqueNetwork/baedeker-action@v1.2
        if: success()
        id: bdk
        with:
          jpath: |
            .baedeker/vendor
          input-modules: |
            lib:baedeker-library/ops/nginx-dev.libsonnet
            lib:baedeker-library/ops/devtools.libsonnet
          tla-str: |
            relay_spec=${{ env.RELAY_CHAIN_TYPE }}-local
            forked_spec=${{ matrix.network }}
            token_symbol=${{ matrix.symbol }}
          tla-code: |
            dump_spec=import 'dump.json'
          inputs: |
            .baedeker/main-process.jsonnet
            snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.image }}:${{ steps.polkadot.outputs.tag }}'}})
            # with data build uses old runtime, but new node, thus we use mainnet image for spec generation, and then target image for nodes.
            # snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.node.outputs.name }}'}}, extra_node_mixin={extraArgs: []}, for_chain = false)
            ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin={extraArgs: []})

      - name: Ensure network is alive
        working-directory: js-packages/tests
        id: alive1
        if: ${{ !cancelled() && steps.bdk.outcome == 'success' }}
        run: |
          deno install
          ../scripts/wait_for_first_block.sh
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: 'Reconcile: update all nodes except one'
        if: ${{ !cancelled() && steps.alive1.outcome == 'success' }}
        id: reconcile0
        uses: UniqueNetwork/baedeker-action/reconcile@v1.2
        with:
          baedeker: ${{ steps.bdk.outputs.baedeker }}
          # Chain should always be built with the mainnet spec, this we first set binary for all nodes, and then force chainspec to be still generated from mainnet
          inputs: |
            ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.target.outputs.name }}'}}, leave = 1, for_chain = false)
            ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin={extraArgs: []}, for_chain = false)
            snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}})

      - name: Ensure network is alive
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.reconcile0.outcome == 'success' }}
        id: alive1_1
        run: |
          deno install        
          ../scripts/wait_for_first_block.sh
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: 'Reconcile: update the remaining node'
        if: ${{ !cancelled() && steps.alive1_1.outcome == 'success' }}
        id: reconcile1
        uses: UniqueNetwork/baedeker-action/reconcile@v1.2
        with:
          baedeker: ${{ steps.bdk.outputs.baedeker }}
          # Chain should always be built with the mainnet spec, this we first set binary for all nodes, and then force chainspec to be still generated from mainnet
          inputs: |
            snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.target.outputs.name }}'}}, for_chain = false)
            snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}})

      - name: Ensure network is alive
        working-directory: js-packages/tests
        id: alive2
        if: ${{ !cancelled() && steps.reconcile1.outcome == 'success' }}
        run: |
          deno install        
          ../scripts/wait_for_first_block.sh
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: 'Reconcile: runtime updating'
        working-directory: js-packages/tests
        id: reconcile2
        if: ${{ !cancelled() && steps.alive2.outcome == 'success' }}
        run: |
          echo "Executing upgrade"
          deno -A ../scripts/authorizeEnactUpgrade.ts ${{ steps.wasms.outputs.dir }}/${{ matrix.network }}-runtime/${{ matrix.network }}_runtime.compact.compressed.wasm
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      ### temporary solution!!!!!!! ###
      ### TODO: delete this step after release v10140080 or put it into a nicer script
      # - name: wait 100 blocks for runtime and restart all parachain nodes
      #   if: ${{ !cancelled() && steps.reconcile2.outcome == 'success' }}
      #   run: |
      #     sleep 660
      #     echo "Restarting parachain nodes"
      #     for i in `docker ps -f name=relay-unique-node -q`; do docker restart $i; done

      - name: Ensure network is alive
        working-directory: js-packages/tests
        id: alive3
        if: ${{ !cancelled() && steps.reconcile2.outcome == 'success' }}
        run: |
          deno install
          ../scripts/wait_for_first_block.sh
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      # - name: "Debug: Tmate session"
      #   uses: mxschmitt/action-tmate@v3

      - name: 'Debug: Run several tests after upgrade'
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.alive3.outcome == 'success' }}
        run: |
          deno task testParallel --filter 'decimals' || true
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: 'Debug: wait 10 blocks for runtime'
        if: ${{ !cancelled() && steps.reconcile2.outcome == 'success' }}
        run: |
          sleep 60

      - name: Run Parallel Substrate tests after upgrade
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.alive3.outcome == 'success' }}
        run: |
          NOW=$(date +%s) && deno task testSubParallel
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: Run Sequential Substrate tests after upgrade
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.alive3.outcome == 'success' }}
        run: |
          NOW=$(date +%s) && deno task testSubSequential
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: Run Parallel Ethereum tests after upgrade
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.alive3.outcome == 'success' }}
        run: |
          NOW=$(date +%s) && deno task testEthParallel
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: Run Sequential Ethereum tests after upgrade
        working-directory: js-packages/tests
        if: ${{ !cancelled() && steps.alive3.outcome == 'success' }}
        run: |
          NOW=$(date +%s) && deno task testEthSequential
        env:
          RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}

      - name: Echo containers logs
        if: always() # run this step always
        run: |
          mkdir -p ${{ steps.bdk.outputs.composeProject }}/logs
          for i in `docker ps -a --format {{.Names}} | grep baedeker`; do docker logs $i >& ${{ steps.bdk.outputs.composeProject }}/logs/$i.log; done

      - name: Upload containers logs
        uses: actions/upload-artifact@v4
        if: always() # run this step always
        with:
          name: ${{ matrix.network }}-main-baedeker-logs
          path: ${{ steps.bdk.outputs.composeProject }}/logs
          retention-days: 7
          compression-level: 9

      - name: Remove all containers
        if: always() # run this step always
        run: |
          docker ps -aq | xargs docker rm -f

      - name: Remove builder cache
        if: always() # run this step always
        run: |
          docker system prune -f -a