snarkVM/.github/workflows/benchmarks.yml at d7e7c570bb65d8b5855fc93e018d1e4e1b8a6fc2 · ProvableHQ/snarkVM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Run snarkVM Benchmarks

on:
  push:
    branches:
      - 'staging'
      - 'copilot/reduce-snarkvm-ci-benchmark-scope'
  workflow_dispatch:

jobs:
  # Run benchmarks and stores the output to a file
  benchmark:
    name: Benchmark
    # Pin to a specific image to reduce runner-image drift between comparisons.
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Install required Debian pacakges
        run: sudo apt install -y lld

      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}

      - name: Benchmark algorithms
        run: |
          cd algorithms
          cargo bench --locked --bench variable_base -- --output-format bencher | tee -a ../output.txt
          cargo bench --locked --bench poseidon_sponge -- --output-format bencher | tee -a ../output.txt
          cargo bench --bench varuna -- --output-format bencher | tee -a ../output.txt
          cd ..

      - name: Benchmark circuit/environment
        run: |
          cd circuit/environment
          cargo bench --locked --bench linear_combination -- --output-format bencher | tee -a ../../output.txt
          cd ../..

      - name: Benchmark console/account
        run: |
          cd console/account
          cargo bench --locked --bench account -- --output-format bencher | tee -a ../../output.txt
          cd ../..

      - name: Benchmark console/algorithms
        run: |
          cd console/algorithms
          cargo bench --locked --bench poseidon -- --output-format bencher | tee -a ../../output.txt
          cargo bench --locked --bench elligator2 -- --output-format bencher | tee -a ../../output.txt
          cd ../..

      - name: Benchmark console/collections
        run: |
          cd console/collections
          cargo bench --locked --bench merkle_tree -- --output-format bencher | tee -a ../../output.txt
          cd ../..

      - name: Benchmark console/types
        run: |
          cd console/types
          cargo bench --locked --bench group -- --output-format bencher | tee -a ../../output.txt
          cd ../..

      - name: Benchmark curves
        run: |
          cd curves
          cargo bench --locked --bench curves -- --output-format bencher | tee -a ../output.txt
          cd ..

      - name: Benchmark ledger (block)
        run: |
          cargo bench --package=snarkvm-ledger --bench block  --features=rocks,test-helpers,test -- --output-format bencher | tee -a output.txt

      - name: Benchmark ledger (dag)
        run: |
          cargo bench --package=snarkvm-ledger --bench dag -- --output-format bencher | tee -a output.txt

      - name: Benchmark ledger (transaction)
        run: |
          cargo bench --package=snarkvm-ledger --bench transaction -- --output-format bencher | tee -a output.txt

      - name: Benchmark ledger (store)
        run: |
          cargo bench --package=snarkvm-ledger --bench store -- --output-format bencher | tee -a output.txt

      - name: Benchmark ledger/puzzle
        run: |
          cargo bench --package=snarkvm-ledger-puzzle --bench puzzle --features=setup -- --output-format bencher | tee -a output.txt

      # Clean benchmark output to remove unnecessary lines
      - name: Clean benchmark output
        run: |
          sed -n -i '/bench:/p' output.txt

      # Download previous benchmark result from cache (if exists)
      - name: Download previous benchmark data
        uses: actions/cache@v4
        with:
          path: ./cache
          key: ${{ runner.os }}-benchmark

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: snarkVM Benchmarks
          tool: 'cargo'
          output-file-path: output.txt
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '150%'
          comment-on-alert: true
          fail-on-alert: true
          alert-comment-cc-users: '@vicsn'
          # Only push the results on staging (see below)
          auto-push: false
          # Enable Job Summary for PRs
          summary-always: true

      - name: Push benchmark result
        # Avoid pushing results on pull requests or experimental branches, to reduce noise in the data.
        # The results for all other runs are still accessible through the workflow summary.
        if: github.ref_name == 'staging'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: git push origin gh-pages