#2541 - Refactor benchmark job to run on pushes and improve PR handling

Author: Jeckerson

.github/workflows/main.yml

@@ -239,23 +239,52 @@ jobs:
           retention-days: 7
 
   benchmark:
-    name: "Benchmark (PR vs ${{ github.base_ref }})"
-    if: github.event_name == 'pull_request'
+    name: "Benchmark"
+    # Triggered by every push EXCEPT pushes to the default branch (which is
+    # the comparison baseline) and tag pushes (releases). We deliberately
+    # don't add `pull_request` to `on:` above because that would double-run
+    # the other jobs (analyze, build-and-test) on each PR.
+    if: github.event_name == 'push' && github.ref != 'refs/heads/development' && !startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-24.04
     permissions:
       contents: read
       pull-requests: write
     env:
       PHP_VERSION: '8.5'
+      BASE_BRANCH: development
       BENCH_PHP_FLAGS: '-d extension=ext/modules/stub.so'
 
     steps:
-      - name: Checkout PR head
+      - name: Checkout head
         uses: actions/checkout@v6
         with:
           # Need full history so we can switch to the base branch in-place.
           fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Discover associated PR
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          branch="${GITHUB_REF#refs/heads/}"
+          echo "branch=$branch" >> "$GITHUB_OUTPUT"
+          # Look up the first open PR with this branch as head. Fork PRs are
+          # surfaced as `owner:branch`, so we narrow by repo as well.
+          pr_json=$(gh pr list \
+            --repo "$GITHUB_REPOSITORY" \
+            --head "$branch" \
+            --state open \
+            --json number,baseRefName,headRefName \
+            --jq '.[0] // empty' 2>/dev/null || true)
+          if [ -n "$pr_json" ]; then
+            echo "has_pr=true" >> "$GITHUB_OUTPUT"
+            echo "number=$(echo "$pr_json" | jq -r .number)" >> "$GITHUB_OUTPUT"
+            echo "base=$(echo "$pr_json" | jq -r .baseRefName)" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_pr=false" >> "$GITHUB_OUTPUT"
+            echo "base=${BASE_BRANCH}" >> "$GITHUB_OUTPUT"
+            echo "No open PR found for branch '$branch'; report will be uploaded as an artifact only."
+          fi
 
       - name: Setup PHP ${{ env.PHP_VERSION }}
         uses: shivammathur/setup-php@v2
@@ -275,14 +304,14 @@ jobs:
       - name: Resolve base ref SHA
         id: base
         run: |
-          base_sha=$(git rev-parse origin/${{ github.base_ref }})
+          base_sha=$(git rev-parse "origin/${{ steps.pr.outputs.base }}")
           echo "sha=$base_sha" >> "$GITHUB_OUTPUT"
-          echo "Base: ${{ github.base_ref }} ($base_sha)"
-          echo "Head: ${{ github.event.pull_request.head.sha }}"
+          echo "Base: ${{ steps.pr.outputs.base }} ($base_sha)"
+          echo "Head: $GITHUB_SHA"
 
-      - name: Build & bench base branch (${{ github.base_ref }})
+      - name: Build & bench base branch (${{ steps.pr.outputs.base }})
         run: |
-          git checkout ${{ steps.base.outputs.sha }}
+          git checkout "${{ steps.base.outputs.sha }}"
           if [ ! -f tests/Benchmark/bootstrap.php ] || [ ! -f phpbench.json ]; then
             echo "BASE_HAS_BENCH=false" >> "$GITHUB_ENV"
             echo "Base branch does not yet include the benchmark suite; skipping baseline run."
@@ -299,14 +328,14 @@ jobs:
             mv .phpbench "$RUNNER_TEMP/phpbench/storage"
           fi
 
-      - name: Restore PR head
+      - name: Restore head
         run: |
-          git checkout ${{ github.event.pull_request.head.sha }}
+          git checkout "$GITHUB_SHA"
           if [ -d "$RUNNER_TEMP/phpbench/storage" ]; then
             mv "$RUNNER_TEMP/phpbench/storage" .phpbench
           fi
 
-      - name: Build & bench PR head
+      - name: Build & bench head
         id: bench
         run: |
           composer install --prefer-dist --no-interaction --no-ansi --no-progress
@@ -330,17 +359,17 @@ jobs:
           fi
           # PHPBench may emit a "Module already loaded" warning when the .so is
           # auto-loaded from php.ini AND via -d extension=. Strip those lines so
-          # the PR comment stays readable.
+          # the comment stays readable.
           sed -i '/Warning: Module .* is already loaded/d' .phpbench/report.txt
 
           {
             echo "## Benchmark report"
             echo
             if [ "$BASE_HAS_BENCH" = "true" ]; then
-              echo "Comparison against \`${{ github.base_ref }}\` (\`${{ steps.base.outputs.sha }}\`)."
-              echo "Each row's \`mode\` column shows the PR-branch absolute throughput and the percent delta vs base. Positive deltas on \`Zephir*\` subjects mean the PR is faster; positive deltas on \`Php*\` baseline subjects are noise floor signal."
+              echo "Comparison against \`${{ steps.pr.outputs.base }}\` (\`${{ steps.base.outputs.sha }}\`) on \`${{ steps.pr.outputs.branch }}\` (\`$GITHUB_SHA\`)."
+              echo "Each row's \`mode\` column shows the head-branch absolute throughput and the percent delta vs base. Positive deltas on \`Zephir*\` subjects mean head is faster; deltas on \`Php*\` baseline subjects are noise-floor signal."
             else
-              echo "Base branch \`${{ github.base_ref }}\` does not yet contain the benchmark suite; absolute throughput only."
+              echo "Base branch \`${{ steps.pr.outputs.base }}\` does not yet contain the benchmark suite; absolute throughput only."
             fi
             echo
             echo '```'
@@ -354,21 +383,23 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: benchmark-report-${{ github.event.pull_request.number }}
+          name: benchmark-report-${{ steps.pr.outputs.branch }}
           path: |
             .phpbench/report.txt
             .phpbench/comment.md
           retention-days: 14
 
       - name: Post or update PR comment
-        if: always()
+        if: always() && steps.pr.outputs.has_pr == 'true'
         uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
         with:
           script: |
             const fs = require('fs');
             const body = fs.readFileSync('.phpbench/comment.md', 'utf8');
             const marker = '<!-- zephir-bench-report -->';
-            const issue_number = context.payload.pull_request.number;
+            const issue_number = parseInt(process.env.PR_NUMBER, 10);
             const { owner, repo } = context.repo;
 
             const { data: comments } = await github.rest.issues.listComments({

tests/Benchmark/README.md

@@ -24,14 +24,21 @@ If you build on the host and run inside the container without fullclean, the
 .libs object files from your host's PHP version will conflict. Always
 `fullclean` when crossing PHP versions.
 
-## Comparing PR vs base
+## Comparing two branches
 
-The same flow CI uses, in two steps:
+CI runs the bench job on every `push` to a non-default branch (see the
+`benchmark` job in `.github/workflows/main.yml`). If the pushed branch has an
+open PR against `development`, the report is posted as an auto-updating PR
+comment; otherwise it is uploaded as an artifact only. The default-branch
+push and tag pushes are skipped so the comparison baseline doesn't run
+against itself.
+
+Locally, the same flow in two steps:
 
 ```bash
 # 1. Capture a baseline tagged "base" on the branch you want to compare
 #    against. Build the extension first so the bench targets exist.
-git checkout develop
+git checkout development
 php zephir fullclean && php zephir build
 php -d extension=ext/modules/stub.so vendor/bin/phpbench run \
     --tag=base --progress=none