#2541 - Refactor benchmark job to run on pushes and improve PR handling #2728
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Zephir CI | |
| on: | |
| schedule: | |
| - cron: '0 2 * * *' # Daily at 02:00 runs only on default branch | |
| push: | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| - '**/nightly.yml' | |
| - '**/release.yml' | |
| - '**/FUNDING.yml' | |
| env: | |
| RE2C_VERSION: 2.2 | |
| ZEPHIR_PARSER_VERSION: 2.0.1 | |
| PSR_VERSION: 1.2.0 | |
| CACHE_DIR: .cache | |
| jobs: | |
| analyze: | |
| name: Static Code Analysis | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup PHP | |
| uses: shivammathur/setup-php@v2 | |
| env: | |
| PHP_CS_FIXER_VERSION: 3.37.0 | |
| with: | |
| php-version: '8.0' | |
| coverage: none | |
| tools: php-cs-fixer:${{ env.PHP_CS_FIXER_VERSION }}, phpcs | |
| - name: Run PHP_CodeSniffer | |
| run: | | |
| phpcs --version | |
| phpcs --runtime-set ignore_warnings_on_exit true | |
| - name: Run Shell Check | |
| if: always() | |
| run: shellcheck .ci/*.sh | |
| build-and-test: | |
| name: "PHP-${{ matrix.php }}-${{ matrix.ts }}-${{ matrix.name }}-${{ matrix.arch }}" | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| php: [ '8.0', '8.1', '8.2', '8.3', '8.4', '8.5' ] | |
| ts: [ 'ts', 'nts' ] | |
| arch: [ 'x64' ] | |
| name: | |
| - ubuntu-gcc | |
| - macos-clang | |
| # matrix names should be in next format: | |
| # {php}-{ts}-{os.name}-{compiler}-{arch} | |
| include: | |
| # Linux | |
| - { name: ubuntu-gcc, os: ubuntu-24.04, compiler: gcc } | |
| # macOS | |
| - { name: macos-clang, os: macos-14, compiler: clang } | |
| # Windows | |
| - { php: '8.0', ts: 'ts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '14.29' } | |
| - { php: '8.0', ts: 'nts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '14.29' } | |
| - { php: '8.1', ts: 'ts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '14.29' } | |
| - { php: '8.1', ts: 'nts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '14.29' } | |
| # Disabled due PSR extension wasn't complied for >=8.2 | |
| #- { php: '8.2', ts: 'ts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '' } | |
| #- { php: '8.2', ts: 'nts', arch: 'x64', name: 'windows2022-vs16', os: 'windows-2022', compiler: 'vs16', toolset: '' } | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 5 | |
| - name: Install PHP ${{ matrix.php }} | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: '${{ matrix.php }}' | |
| extensions: mbstring, fileinfo, gmp, sqlite, pdo_sqlite, psr-${{ env.PSR_VERSION }}, zip, mysqli, zephir_parser-${{ env.ZEPHIR_PARSER_VERSION }} | |
| tools: pecl, phpize, php-config | |
| coverage: xdebug | |
| # variables_order: https://github.com/zephir-lang/zephir/pull/1537 | |
| # enable_dl: https://github.com/zephir-lang/zephir/pull/1654 | |
| # allow_url_fopen: https://github.com/zephir-lang/zephir/issues/1713 | |
| # error_reporting: https://github.com/zendframework/zend-code/issues/160 | |
| ini-values: >- | |
| variables_order=EGPCS, | |
| enable_dl=On, | |
| allow_url_fopen=On, | |
| error_reporting=-1, | |
| memory_limit=1G, | |
| date.timezone=UTC, | |
| xdebug.max_nesting_level=256 | |
| env: | |
| phpts: ${{ matrix.ts }} | |
| COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install Project Dependencies | |
| run: composer install --prefer-dist --no-interaction --no-ansi --no-progress | |
| - name: Fast Commands Test | |
| run: php zephir --help | |
| - name: Build Test Extension (Linux) | |
| if: runner.os == 'Linux' | |
| uses: ./.github/workflows/build-linux-ext | |
| with: | |
| compiler: ${{ matrix.compiler }} | |
| cflags: '-O2 -fvisibility=hidden -flto -DZEPHIR_RELEASE=1' | |
| ldflags: '--coverage' | |
| - name: Build Test Extension (macOS) | |
| if: runner.os == 'macOS' | |
| uses: ./.github/workflows/build-macos-ext | |
| with: | |
| compiler: ${{ matrix.compiler }} | |
| cflags: '-O2 -fvisibility=hidden -Wparentheses -flto -DZEPHIR_RELEASE=1' | |
| - name: Build Test Extension (Windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/workflows/build-win-ext | |
| with: | |
| php_version: ${{ matrix.php }} | |
| ts: ${{ matrix.ts }} | |
| msvc: ${{ matrix.compiler }} | |
| arch: ${{ matrix.arch }} | |
| toolset: ${{ matrix.toolset }} | |
| cflags: '/D ZEPHIR_RELEASE /Oi /Ot /Oy /Ob2 /Gs /GF /Gy /GL' | |
| ldflags: '/LTCG' | |
| env: | |
| CACHE_DIR: 'C:\Downloads' | |
| PHP_ROOT: 'C:\tools\php' | |
| - name: Stub Extension Info | |
| shell: pwsh | |
| run: | | |
| php --ini | |
| php --ri stub | |
| - name: Setup problem matchers for PHPUnit | |
| run: echo "::add-matcher::${{ runner.tool_cache }}/phpunit.json" | |
| - name: Unit Tests (Stub Extension) | |
| shell: pwsh | |
| run: | | |
| php vendor/bin/phpunit -c phpunit.ext.xml | |
| env: | |
| XDEBUG_MODE: coverage | |
| - name: Unit Tests (Zephir) | |
| if: always() | |
| run: php vendor/bin/phpunit --testsuite Zephir --coverage-php ./tests/output/clover.xml | |
| env: | |
| XDEBUG_MODE: coverage | |
| - name: "Upload coverage file artifact" | |
| uses: "actions/upload-artifact@v7" | |
| with: | |
| name: "unit-${{ matrix.php }}-${{ matrix.ts }}-${{ matrix.name }}.coverage" | |
| path: "tests/output/clover.xml" | |
| - name: Black-box Testing | |
| if: always() | |
| run: php vendor/bin/phpunit --testsuite BlackBox --no-coverage | |
| upload-coverage: | |
| permissions: | |
| contents: read | |
| name: "Upload coverage" | |
| runs-on: "ubuntu-22.04" | |
| needs: | |
| - "build-and-test" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| # - name: 'Qodana Scan' | |
| # uses: JetBrains/qodana-action@v2023.2 | |
| # env: | |
| # QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} | |
| # with: | |
| # args: --baseline,./qodana.sarif.json | |
| # - name: 'Qodana Scan' | |
| # run: | | |
| # docker run \ | |
| # -v $(pwd):/data/project/ \ | |
| # -v $(pwd):/data/base/ \ | |
| # -e QODANA_TOKEN="${{ secrets.CODECOV_TOKEN }}" \ | |
| # jetbrains/qodana-php \ | |
| # --baseline /data/base/qodana.sarif.json | |
| - name: "Create download folder" | |
| run: | | |
| mkdir -p reports | |
| - name: "Download coverage files" | |
| uses: "actions/download-artifact@v8" | |
| with: | |
| path: "reports" | |
| - name: "Display structure of downloaded files" | |
| run: ls -R | |
| working-directory: reports | |
| - name: "Upload to Codecov" | |
| uses: "codecov/codecov-action@v6" | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| directory: reports | |
| fail_ci_if_error: true | |
| verbose: true | |
| name: codecov-umbrella | |
| - name: Upload build artifacts after Failure | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: debug-PHP-${{ matrix.php }}-${{ matrix.ts }}-${{ matrix.name }}-${{ matrix.arch }} | |
| path: | | |
| ${{ github.workspace }}/*.log | |
| ${{ github.workspace }}/ext/ | |
| !${{ github.workspace }}/ext/kernel/ | |
| !${{ github.workspace }}/ext/stub/ | |
| !${{ github.workspace }}/ext/Release/ | |
| !${{ github.workspace }}/ext/x64/Release/ | |
| ${{ github.workspace }}/tests/output/ | |
| retention-days: 7 | |
| benchmark: | |
| name: "Benchmark" | |
| # Triggered by every push EXCEPT pushes to the default branch (which is | |
| # the comparison baseline) and tag pushes (releases). We deliberately | |
| # don't add `pull_request` to `on:` above because that would double-run | |
| # the other jobs (analyze, build-and-test) on each PR. | |
| if: github.event_name == 'push' && github.ref != 'refs/heads/development' && !startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| env: | |
| PHP_VERSION: '8.5' | |
| BASE_BRANCH: development | |
| BENCH_PHP_FLAGS: '-d extension=ext/modules/stub.so' | |
| steps: | |
| - name: Checkout head | |
| uses: actions/checkout@v6 | |
| with: | |
| # Need full history so we can switch to the base branch in-place. | |
| fetch-depth: 0 | |
| - name: Discover associated PR | |
| id: pr | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| branch="${GITHUB_REF#refs/heads/}" | |
| echo "branch=$branch" >> "$GITHUB_OUTPUT" | |
| # Look up the first open PR with this branch as head. Fork PRs are | |
| # surfaced as `owner:branch`, so we narrow by repo as well. | |
| pr_json=$(gh pr list \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --head "$branch" \ | |
| --state open \ | |
| --json number,baseRefName,headRefName \ | |
| --jq '.[0] // empty' 2>/dev/null || true) | |
| if [ -n "$pr_json" ]; then | |
| echo "has_pr=true" >> "$GITHUB_OUTPUT" | |
| echo "number=$(echo "$pr_json" | jq -r .number)" >> "$GITHUB_OUTPUT" | |
| echo "base=$(echo "$pr_json" | jq -r .baseRefName)" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_pr=false" >> "$GITHUB_OUTPUT" | |
| echo "base=${BASE_BRANCH}" >> "$GITHUB_OUTPUT" | |
| echo "No open PR found for branch '$branch'; report will be uploaded as an artifact only." | |
| fi | |
| - name: Setup PHP ${{ env.PHP_VERSION }} | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: '${{ env.PHP_VERSION }}' | |
| extensions: mbstring, fileinfo, gmp, sqlite, pdo_sqlite, psr-${{ env.PSR_VERSION }}, zip, mysqli, zephir_parser-${{ env.ZEPHIR_PARSER_VERSION }} | |
| tools: pecl, phpize, php-config | |
| coverage: none | |
| ini-values: >- | |
| variables_order=EGPCS, | |
| enable_dl=On, | |
| allow_url_fopen=On, | |
| error_reporting=-1, | |
| memory_limit=1G, | |
| date.timezone=UTC | |
| - name: Resolve base ref SHA | |
| id: base | |
| run: | | |
| base_sha=$(git rev-parse "origin/${{ steps.pr.outputs.base }}") | |
| echo "sha=$base_sha" >> "$GITHUB_OUTPUT" | |
| echo "Base: ${{ steps.pr.outputs.base }} ($base_sha)" | |
| echo "Head: $GITHUB_SHA" | |
| - name: Build & bench base branch (${{ steps.pr.outputs.base }}) | |
| run: | | |
| git checkout "${{ steps.base.outputs.sha }}" | |
| if [ ! -f tests/Benchmark/bootstrap.php ] || [ ! -f phpbench.json ]; then | |
| echo "BASE_HAS_BENCH=false" >> "$GITHUB_ENV" | |
| echo "Base branch does not yet include the benchmark suite; skipping baseline run." | |
| exit 0 | |
| fi | |
| echo "BASE_HAS_BENCH=true" >> "$GITHUB_ENV" | |
| composer install --prefer-dist --no-interaction --no-ansi --no-progress | |
| php zephir fullclean >/dev/null 2>&1 || true | |
| php zephir build 2>&1 | tail -3 | |
| php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run --tag=base --progress=none --no-interaction 2>&1 | tail -5 | |
| # Preserve the phpbench storage across the upcoming branch switch. | |
| if [ -d .phpbench ]; then | |
| mkdir -p "$RUNNER_TEMP/phpbench" | |
| mv .phpbench "$RUNNER_TEMP/phpbench/storage" | |
| fi | |
| - name: Restore head | |
| run: | | |
| git checkout "$GITHUB_SHA" | |
| if [ -d "$RUNNER_TEMP/phpbench/storage" ]; then | |
| mv "$RUNNER_TEMP/phpbench/storage" .phpbench | |
| fi | |
| - name: Build & bench head | |
| id: bench | |
| run: | | |
| composer install --prefer-dist --no-interaction --no-ansi --no-progress | |
| php zephir fullclean >/dev/null 2>&1 || true | |
| php zephir build 2>&1 | tail -3 | |
| mkdir -p .phpbench | |
| if [ "$BASE_HAS_BENCH" = "true" ]; then | |
| php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run \ | |
| --ref=base \ | |
| --report=aggregate \ | |
| --progress=none \ | |
| --no-interaction \ | |
| > .phpbench/report.txt 2>&1 || true | |
| else | |
| php ${{ env.BENCH_PHP_FLAGS }} vendor/bin/phpbench run \ | |
| --report=aggregate \ | |
| --progress=none \ | |
| --no-interaction \ | |
| > .phpbench/report.txt 2>&1 || true | |
| fi | |
| # PHPBench may emit a "Module already loaded" warning when the .so is | |
| # auto-loaded from php.ini AND via -d extension=. Strip those lines so | |
| # the comment stays readable. | |
| sed -i '/Warning: Module .* is already loaded/d' .phpbench/report.txt | |
| { | |
| echo "## Benchmark report" | |
| echo | |
| if [ "$BASE_HAS_BENCH" = "true" ]; then | |
| echo "Comparison against \`${{ steps.pr.outputs.base }}\` (\`${{ steps.base.outputs.sha }}\`) on \`${{ steps.pr.outputs.branch }}\` (\`$GITHUB_SHA\`)." | |
| echo "Each row's \`mode\` column shows the head-branch absolute throughput and the percent delta vs base. Positive deltas on \`Zephir*\` subjects mean head is faster; deltas on \`Php*\` baseline subjects are noise-floor signal." | |
| else | |
| echo "Base branch \`${{ steps.pr.outputs.base }}\` does not yet contain the benchmark suite; absolute throughput only." | |
| fi | |
| echo | |
| echo '```' | |
| cat .phpbench/report.txt | |
| echo '```' | |
| echo | |
| echo "_PHP ${{ env.PHP_VERSION }} on \`${{ runner.os }}\` (${{ runner.arch }}). Micro-benchmarks are noisy on shared runners (±5-20% per subject); treat any single-digit delta as inconclusive._" | |
| } > .phpbench/comment.md | |
| - name: Upload benchmark artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: benchmark-report-${{ steps.pr.outputs.branch }} | |
| path: | | |
| .phpbench/report.txt | |
| .phpbench/comment.md | |
| retention-days: 14 | |
| - name: Post or update PR comment | |
| if: always() && steps.pr.outputs.has_pr == 'true' | |
| uses: actions/github-script@v7 | |
| env: | |
| PR_NUMBER: ${{ steps.pr.outputs.number }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const body = fs.readFileSync('.phpbench/comment.md', 'utf8'); | |
| const marker = '<!-- zephir-bench-report -->'; | |
| const issue_number = parseInt(process.env.PR_NUMBER, 10); | |
| const { owner, repo } = context.repo; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner, repo, issue_number, per_page: 100, | |
| }); | |
| const existing = comments.find(c => c.body && c.body.includes(marker)); | |
| const finalBody = `${marker}\n${body}`; | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, repo, comment_id: existing.id, body: finalBody, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, repo, issue_number, body: finalBody, | |
| }); | |
| } |