action.yaml

name: 'Benchmarkoor'
description: 'Run Ethereum execution client benchmarks using benchmarkoor'
author: 'ethpandaops'
branding:
  icon: 'activity'
  color: 'gray-dark'

inputs:
  github-token:
    description: 'GitHub token for API access'
    required: true

  mode:
    description: 'Which phase(s) to run: "all" (build if a build-config is set, then run — the default), "build" (build only, no run), or "run" (run only, skip build even if a build-config is set).'
    required: false
    default: 'all'

  image:
    description: 'Docker image to extract the benchmarkoor binary from (e.g., ghcr.io/ethpandaops/benchmarkoor:master). If not provided, will build the image locally from git-ref/git-repo and extract from it.'
    required: false
    default: ''

  git-ref:
    description: 'Git branch or commit hash to build from. Only used when image is not provided. Defaults to master.'
    required: false
    default: ''

  git-repo:
    description: 'Git repository URL to clone for building the image. Only used when image is not provided. Defaults to ethpandaops/benchmarkoor.'
    required: false
    default: 'https://github.com/ethpandaops/benchmarkoor.git'

  upload-artifacts:
    description: 'Whether to upload run results as GitHub artifacts'
    required: false
    default: 'false'

  run-config-urls:
    description: 'Comma-separated URLs for config files to download and pass via --config (merged in order)'
    required: false
    default: ''

  run-config:
    description: 'Raw YAML config content to pass via --config (appended after URL config)'
    required: false
    default: ''

  run-args:
    description: 'Extra flags passed to benchmarkoor run command'
    required: false
    default: ''

  build-config-urls:
    description: 'Comma-separated URLs for config files to download and pass to `benchmarkoor build` via --config (merged in order). Setting this or build-config enables the build phase (state-actor snapshots + eest payloads), which runs before the benchmark run.'
    required: false
    default: ''

  build-config:
    description: 'Raw YAML config content to pass to `benchmarkoor build` via --config (appended after URL config). Setting this or build-config-urls enables the build phase.'
    required: false
    default: ''

  build-args:
    description: 'Extra flags passed to the benchmarkoor build command (e.g. --force, --limit-state-actor-target=)'
    required: false
    default: ''

  tmp-dir:
    description: 'Temporary directory path used for benchmarkoor data (overlayfs) and cache'
    required: false
    default: '/tmp'

runs:
  using: 'composite'
  steps:
    - name: Validate mode
      shell: bash
      run: |
        case "${{ inputs.mode }}" in
          all|build|run) ;;
          *) echo "Invalid mode '${{ inputs.mode }}' (expected: all, build, or run)" >&2; exit 1 ;;
        esac

    - name: Verify and install dependencies
      uses: ethpandaops/benchmarkoor/.github/actions/dependencies@0ff45ea539387c901ac4a7845e2c6e6aeb9d8f25

    - name: Build Docker image locally
      if: inputs.image == ''
      shell: bash
      run: |
        GIT_REF="${{ inputs.git-ref }}"
        if [ -z "$GIT_REF" ]; then
          GIT_REF="master"
        fi

        GIT_REPO="${{ inputs.git-repo }}"

        echo "Cloning ${GIT_REPO} at ref $GIT_REF"
        git clone "${GIT_REPO}" benchmarkoor-build
        cd benchmarkoor-build
        git checkout "$GIT_REF"

        echo "Building Docker image..."
        docker build -t benchmarkoor:local .
        cd ..

        rm -rf benchmarkoor-build

    - name: Extract benchmarkoor binary
      id: extract-binary
      shell: bash
      run: |
        if [ -n "${{ inputs.image }}" ]; then
          IMAGE="${{ inputs.image }}"
          echo "Pulling Docker image to extract binary: ${IMAGE}"
          docker pull "${IMAGE}" || {
            echo "Failed to pull Docker image: ${IMAGE}"
            exit 1
          }
        else
          IMAGE="benchmarkoor:local"
          echo "Using locally built image: ${IMAGE}"
        fi

        BIN="${{ runner.temp }}/benchmarkoor"

        # docker create + docker cp gives us the binary owned by the
        # runner user without needing a writable bind-mount or matching
        # uid inside the container.
        cid=$(docker create "${IMAGE}")
        trap "docker rm -f \"$cid\" >/dev/null 2>&1 || true" EXIT
        docker cp "$cid:/usr/local/bin/benchmarkoor" "$BIN"
        chmod +x "$BIN"

        echo "Extracted binary to $BIN"
        "$BIN" --version || true

        echo "binary=${BIN}" >> "$GITHUB_OUTPUT"

    - name: Build Benchmarkoor artifacts
      id: build-benchmarkoor
      if: (inputs.mode == 'all' || inputs.mode == 'build') && (inputs.build-config != '' || inputs.build-config-urls != '')
      shell: bash
      env:
        INPUT_BUILD_CONFIG: ${{ inputs.build-config }}
        INPUT_BUILD_CONFIG_URLS: ${{ inputs.build-config-urls }}
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"

        # Download URL configs if provided (comma-separated)
        if [ -n "$INPUT_BUILD_CONFIG_URLS" ]; then
          IFS=',' read -ra CONFIG_URLS <<< "$INPUT_BUILD_CONFIG_URLS"
          for i in "${!CONFIG_URLS[@]}"; do
            URL=$(echo "${CONFIG_URLS[$i]}" | xargs)  # trim whitespace
            echo "Downloading build config from ${URL}"
            curl -fsSL "${URL}" -o "${{ runner.temp }}/benchmarkoor-build-config-url-${i}.yaml"
          done
        fi

        # Write inline config if provided
        if [ -n "$INPUT_BUILD_CONFIG" ]; then
          echo "$INPUT_BUILD_CONFIG" > "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml"
        fi

        # Build argv as an array so quoted values survive intact. Config files
        # apply in order: URL configs, then inline config (last wins). This step
        # runs from the workspace, so any relative paths in the config (e.g. a
        # custom fill_dockerfile) resolve against the checkout.
        ARGS=("build")
        for cfg in "${{ runner.temp }}"/benchmarkoor-build-config-url-*.yaml; do
          [ -f "$cfg" ] && ARGS+=("--config=$cfg")
        done
        if [ -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml" ]; then
          ARGS+=("--config=${{ runner.temp }}/benchmarkoor-build-config-inline.yaml")
        fi

        # Append extra build args if provided (whitespace-split, same as run).
        if [ -n "${{ inputs.build-args }}" ]; then
          # shellcheck disable=SC2206 # intentional word-split for argv
          EXTRA_ARGS=(${{ inputs.build-args }})
          ARGS+=("${EXTRA_ARGS[@]}")
        fi

        # `sudo -E` for the same reasons as the run step: keeps the built
        # snapshots/fixtures root-owned (so the rootful run phase can read
        # them) and propagates caller env (e.g. STATE_DIR) into the build.
        echo "Running: sudo -E $BIN ${ARGS[*]}"
        sudo -E "$BIN" "${ARGS[@]}"

    - name: Get Job ID from GH API
      id: get-job-id
      if: inputs.mode == 'all' || inputs.mode == 'run'
      shell: bash
      env:
        GH_TOKEN: ${{ inputs.github-token }}
      run: |
        jobs=$(gh api -F per_page=100 -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs)
        job_id=$(echo "$jobs" | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}" and .status=="in_progress") | .id')
        echo "job_id=${job_id}" >> "$GITHUB_OUTPUT"

    - name: Run Benchmarkoor
      id: run-benchmarkoor
      if: inputs.mode == 'all' || inputs.mode == 'run'
      shell: bash
      env:
        INPUT_RUN_CONFIG: ${{ inputs.run-config }}
        INPUT_RUN_CONFIG_URLS: ${{ inputs.run-config-urls }}
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"
        mkdir -p "${{ runner.temp }}/results"

        # Download URL configs if provided (comma-separated)
        if [ -n "$INPUT_RUN_CONFIG_URLS" ]; then
          IFS=',' read -ra CONFIG_URLS <<< "$INPUT_RUN_CONFIG_URLS"
          for i in "${!CONFIG_URLS[@]}"; do
            URL=$(echo "${CONFIG_URLS[$i]}" | xargs)  # trim whitespace
            echo "Downloading config from ${URL}"
            curl -fsSL "${URL}" -o "${{ runner.temp }}/benchmarkoor-config-url-${i}.yaml"
          done
        fi

        # Write inline config if provided
        if [ -n "$INPUT_RUN_CONFIG" ]; then
          echo "$INPUT_RUN_CONFIG" > "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
        fi

        # Export benchmarkoor's own config env vars. Anything the caller
        # set via `env:` on the action step is already in our process
        # environment; `sudo -E` below propagates both sets to the
        # benchmarkoor process (which it needs to be: root, for
        # drop_memory_caches, schelk dm_era ops, rootful podman, etc.).
        export BENCHMARKOOR_RUNNER_GITHUB_TOKEN="${{ inputs.github-token }}"
        export BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_DIR="${{ runner.temp }}/results"
        export BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_OWNER="$(id -u):$(id -g)"
        export BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_DATADIR="${{ inputs.tmp-dir }}"
        export BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_CACHEDIR="${{ inputs.tmp-dir }}/benchmarkoor-cache"

        # Build argv as an array so quoted values survive intact.
        ARGS=("run")

        # Add config files in order: URL configs, then inline config (last wins).
        for cfg in "${{ runner.temp }}"/benchmarkoor-config-url-*.yaml; do
          [ -f "$cfg" ] && ARGS+=("--config=$cfg")
        done
        if [ -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" ]; then
          ARGS+=("--config=${{ runner.temp }}/benchmarkoor-config-inline.yaml")
        fi

        # GitHub Actions context as metadata labels.
        ARGS+=(
          "--metadata.label=github.run_id=${{ github.run_id }}"
          "--metadata.label=github.run_number=${{ github.run_number }}"
          "--metadata.label=github.job=${{ github.job }}"
          "--metadata.label=github.job_id=${{ steps.get-job-id.outputs.job_id }}"
          "--metadata.label=github.repository=${{ github.repository }}"
          "--metadata.label=github.workflow=${{ github.workflow }}"
          "--metadata.label=github.sha=${{ github.sha }}"
          "--metadata.label=github.actor=${{ github.actor }}"
          "--metadata.label=github.event_name=${{ github.event_name }}"
          "--metadata.label=github.ref=${{ github.ref }}"
        )

        # Append extra run args if provided (whitespace-split, same as before).
        if [ -n "${{ inputs.run-args }}" ]; then
          # shellcheck disable=SC2206 # intentional word-split for argv
          EXTRA_ARGS=(${{ inputs.run-args }})
          ARGS+=("${EXTRA_ARGS[@]}")
        fi

        echo "Running: sudo -E $BIN ${ARGS[*]}"

        set +e
        sudo -E "$BIN" "${ARGS[@]}"
        EXIT_CODE=$?
        set -e

        # Find run directories
        RUN_DIRS=""
        if [ -d "${{ runner.temp }}/results/runs" ]; then
          RUN_DIRS=$(find "${{ runner.temp }}/results/runs" -mindepth 1 -maxdepth 1 -type d | sort | tr '\n' ' ')
        fi

        echo "run-dirs=${RUN_DIRS}" >> "$GITHUB_OUTPUT"

        if [ $EXIT_CODE -ne 0 ]; then
          echo "Benchmarkoor failed with exit code $EXIT_CODE"
          exit $EXIT_CODE
        fi

        echo "Benchmarkoor completed successfully"

    - name: Generate markdown summaries
      if: always() && steps.run-benchmarkoor.outputs.run-dirs != ''
      shell: bash
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"

        for RUN_DIR_HOST in ${{ steps.run-benchmarkoor.outputs.run-dirs }}; do
          RUN_DIR_NAME=$(basename "$RUN_DIR_HOST")

          echo "Generating summary for ${RUN_DIR_NAME}"

          "$BIN" generate-markdown-summary \
            --run-dir="${RUN_DIR_HOST}" \
            --output="${RUN_DIR_HOST}/summary.md"

          if [ -f "${RUN_DIR_HOST}/summary.md" ]; then
            cat "${RUN_DIR_HOST}/summary.md" >> "$GITHUB_STEP_SUMMARY"
            echo "" >> "$GITHUB_STEP_SUMMARY"
          fi
        done

    - name: Create results archive
      if: always() && inputs.upload-artifacts == 'true' && inputs.mode != 'build'
      shell: bash
      run: |
        if [ -d "${{ runner.temp }}/results" ]; then
          tar -czf "${{ runner.temp }}/benchmarkoor-results.tar.gz" -C "${{ runner.temp }}" results
        fi

    - name: Upload artifacts
      uses: actions/upload-artifact@v4
      if: always() && inputs.upload-artifacts == 'true' && inputs.mode != 'build'
      with:
        name: benchmarkoor-${{ github.run_id }}
        path: ${{ runner.temp }}/benchmarkoor-results.tar.gz

    - name: Cleanup
      shell: bash
      if: always()
      run: |
        echo "Cleaning up"
        rm -rf "${{ runner.temp }}/results"
        rm -f "${{ runner.temp }}/benchmarkoor-config-url-"*.yaml
        rm -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
        rm -f "${{ runner.temp }}/benchmarkoor-build-config-url-"*.yaml
        rm -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml"
        rm -f "${{ runner.temp }}/benchmarkoor-results.tar.gz"
        rm -f "${{ runner.temp }}/benchmarkoor"