Merge pull request #2545 from zephir-lang/#2541-phpbench

#2541 - Add PHPBench benchmarks

Author: Jeckerson

.github/workflows/main.yml

@@ -237,3 +237,195 @@ jobs:
             !${{ github.workspace }}/ext/x64/Release/
             ${{ github.workspace }}/tests/output/
           retention-days: 7
+
+  benchmark:
+    name: "Benchmark"
+    # Triggered by every push EXCEPT pushes to the default branch (which is
+    # the comparison baseline) and tag pushes (releases). We deliberately
+    # don't add `pull_request` to `on:` above because that would double-run
+    # the other jobs (analyze, build-and-test) on each PR.
+    if: github.event_name == 'push' && github.ref != 'refs/heads/development' && !startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      pull-requests: write
+    env:
+      # Held at 8.3 because PHPBench 1.2.x and its Symfony Console deps
+      # haven't been updated for PHP 8.4's "Implicitly marking parameter as
+      # nullable" deprecation. Running on 8.4+ floods the PR comment with
+      # ~120 lines of upstream deprecation notices. Bumping when phpbench
+      # ships a release that ports those signatures to `?Type` syntax.
+      PHP_VERSION: '8.3'
+      BASE_BRANCH: development
+      BENCH_PHP_FLAGS: '-d extension=ext/modules/stub.so'
+
+    steps:
+      - name: Checkout head
+        uses: actions/checkout@v6
+        with:
+          # Need full history so we can switch to the base branch in-place.
+          fetch-depth: 0
+
+      - name: Discover associated PR
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          branch="${GITHUB_REF#refs/heads/}"
+          echo "branch=$branch" >> "$GITHUB_OUTPUT"
+          # Look up the first open PR with this branch as head. Fork PRs are
+          # surfaced as `owner:branch`, so we narrow by repo as well.
+          pr_json=$(gh pr list \
+            --repo "$GITHUB_REPOSITORY" \
+            --head "$branch" \
+            --state open \
+            --json number,baseRefName,headRefName \
+            --jq '.[0] // empty' 2>/dev/null || true)
+          if [ -n "$pr_json" ]; then
+            echo "has_pr=true" >> "$GITHUB_OUTPUT"
+            echo "number=$(echo "$pr_json" | jq -r .number)" >> "$GITHUB_OUTPUT"
+            echo "base=$(echo "$pr_json" | jq -r .baseRefName)" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_pr=false" >> "$GITHUB_OUTPUT"
+            echo "base=${BASE_BRANCH}" >> "$GITHUB_OUTPUT"
+            echo "No open PR found for branch '$branch'; report will be uploaded as an artifact only."
+          fi
+
+      - name: Setup PHP ${{ env.PHP_VERSION }}
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '${{ env.PHP_VERSION }}'
+          extensions: mbstring, fileinfo, gmp, sqlite, pdo_sqlite, psr-${{ env.PSR_VERSION }}, zip, mysqli, zephir_parser-${{ env.ZEPHIR_PARSER_VERSION }}
+          tools: pecl, phpize, php-config
+          coverage: none
+          ini-values: >-
+            variables_order=EGPCS,
+            enable_dl=On,
+            allow_url_fopen=On,
+            error_reporting=-1,
+            memory_limit=1G,
+            date.timezone=UTC
+
+      - name: Resolve base ref SHA
+        id: base
+        run: |
+          base_sha=$(git rev-parse "origin/${{ steps.pr.outputs.base }}")
+          echo "sha=$base_sha" >> "$GITHUB_OUTPUT"
+          echo "Base: ${{ steps.pr.outputs.base }} ($base_sha)"
+          echo "Head: $GITHUB_SHA"
+
+      - name: Build & bench base branch (${{ steps.pr.outputs.base }})
+        run: |
+          git checkout "${{ steps.base.outputs.sha }}"
+          if [ ! -f tests/Benchmark/bootstrap.php ] || [ ! -f phpbench.json ]; then
+            echo "BASE_HAS_BENCH=false" >> "$GITHUB_ENV"
+            echo "Base branch does not yet include the benchmark suite; skipping baseline run."
+            exit 0
+          fi
+          echo "BASE_HAS_BENCH=true" >> "$GITHUB_ENV"
+          composer install --prefer-dist --no-interaction --no-ansi --no-progress
+          php zephir fullclean >/dev/null 2>&1 || true
+          php zephir build 2>&1 | tail -3
+          php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run --tag=base --progress=none --no-interaction 2>&1 | tail -5
+          # Preserve the phpbench storage across the upcoming branch switch.
+          if [ -d .phpbench ]; then
+            mkdir -p "$RUNNER_TEMP/phpbench"
+            mv .phpbench "$RUNNER_TEMP/phpbench/storage"
+          fi
+
+      - name: Restore head
+        run: |
+          git checkout "$GITHUB_SHA"
+          if [ -d "$RUNNER_TEMP/phpbench/storage" ]; then
+            mv "$RUNNER_TEMP/phpbench/storage" .phpbench
+          fi
+
+      - name: Build & bench head
+        id: bench
+        run: |
+          composer install --prefer-dist --no-interaction --no-ansi --no-progress
+          php zephir fullclean >/dev/null 2>&1 || true
+          php zephir build 2>&1 | tail -3
+
+          mkdir -p .phpbench
+          if [ "$BASE_HAS_BENCH" = "true" ]; then
+            php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run \
+              --ref=base \
+              --report=aggregate \
+              --progress=none \
+              --no-interaction \
+              > .phpbench/report.txt 2>&1 || true
+          else
+            php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run \
+              --report=aggregate \
+              --progress=none \
+              --no-interaction \
+              > .phpbench/report.txt 2>&1 || true
+          fi
+          # Strip output that would clutter the PR comment:
+          #  - "Module already loaded" when the .so is in php.ini AND via -d
+          #    extension= on the command line;
+          #  - upstream PHP Deprecated/Notice/Warning lines (phpbench 1.2.x +
+          #    symfony/console haven't been ported to PHP 8.4's `?Type`
+          #    deprecation; out of scope to fix here).
+          sed -i \
+            -e '/Warning: Module .* is already loaded/d' \
+            -e '/^PHP Deprecated:/d' \
+            -e '/^PHP Notice:/d' \
+            -e '/^Deprecated:/d' \
+            .phpbench/report.txt
+
+          {
+            echo "## Benchmark report"
+            echo
+            if [ "$BASE_HAS_BENCH" = "true" ]; then
+              echo "Comparison against \`${{ steps.pr.outputs.base }}\` (\`${{ steps.base.outputs.sha }}\`) on \`${{ steps.pr.outputs.branch }}\` (\`$GITHUB_SHA\`)."
+              echo "Each row's \`mode\` column shows the head-branch absolute throughput and the percent delta vs base. Positive deltas on \`Zephir*\` subjects mean head is faster; deltas on \`Php*\` baseline subjects are noise-floor signal."
+            else
+              echo "Base branch \`${{ steps.pr.outputs.base }}\` does not yet contain the benchmark suite; absolute throughput only."
+            fi
+            echo
+            echo '```'
+            cat .phpbench/report.txt
+            echo '```'
+            echo
+            echo "_PHP ${{ env.PHP_VERSION }} on \`${{ runner.os }}\` (${{ runner.arch }}). Micro-benchmarks are noisy on shared runners (±5-20% per subject); treat any single-digit delta as inconclusive._"
+          } > .phpbench/comment.md
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: benchmark-report-${{ steps.pr.outputs.branch }}
+          path: |
+            .phpbench/report.txt
+            .phpbench/comment.md
+          retention-days: 14
+
+      - name: Post or update PR comment
+        if: always() && steps.pr.outputs.has_pr == 'true'
+        uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+        with:
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('.phpbench/comment.md', 'utf8');
+            const marker = '<!-- zephir-bench-report -->';
+            const issue_number = parseInt(process.env.PR_NUMBER, 10);
+            const { owner, repo } = context.repo;
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner, repo, issue_number, per_page: 100,
+            });
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            const finalBody = `${marker}\n${body}`;
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner, repo, comment_id: existing.id, body: finalBody,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner, repo, issue_number, body: finalBody,
+              });
+            }

.gitignore

@@ -34,6 +34,7 @@ phpmd.xml
 phpunit.xml
 .phpunit.result.cache
 .php-cs-fixer.cache
+.phpbench/
 
 # Build artefact
 zephir.phar

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Added
 - Compiler now recognizes the parser-emitted `yield` AST node (bare `yield;`, `yield expr;`, `yield key, value;`) [#1849](https://github.com/zephir-lang/zephir/issues/1849)
+- Added PHPBench-based runtime benchmarks suites under `tests/Benchmark/` [#2541](https://github.com/zephir-lang/zephir/issues/2541)
 
 ### Changed
 - `for k, v in expr` now skips the unreachable branch when the iterand's dynamic type is known [#1878](https://github.com/zephir-lang/zephir/issues/1878)

composer.json

@@ -37,6 +37,7 @@
         "ext-pdo": "*",
         "ext-pdo_sqlite": "*",
         "ext-zip": "*",
+        "phpbench/phpbench": "~1.2.0",
         "phpunit/phpunit": "^9.5",
         "psr/log": "1.1.*",
         "squizlabs/php_codesniffer": "^4.0"
@@ -55,7 +56,8 @@
     "autoload-dev": {
         "psr-4": {
             "Zephir\\Test\\": "tests/Zephir/",
-            "Extension\\": "tests/Extension/"
+            "Extension\\": "tests/Extension/",
+            "Benchmark\\": "tests/Benchmark/"
         },
         "classmap": [
             "tests/fixtures/mocks/"