Make bench runtime more deterministic

Author: vicsn

.github/workflows/benchmarks.yml

@@ -11,7 +11,8 @@ jobs:
   # Run benchmarks and stores the output to a file
   benchmark:
     name: Benchmark
-    runs-on: ubuntu-latest
+    # Pin to a specific image to reduce runner-image drift between comparisons.
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
     steps:

Cargo.lock

@@ -19,6 +19,9 @@ use snarkvm_utilities::bytes::unchecked_deserialize;
 
 use criterion::{Criterion, criterion_group, criterion_main};
 
+/// Fixed RNG seed so benchmark inputs are reproducible across CI runs.
+const BENCH_RNG_SEED: u64 = 0x00BA_6DAB_5EED_00D1;
+
 /// Helper method to benchmark serialization.
 fn bench_serialization<T: Serialize + DeserializeOwned + ToBytes + FromBytes + Clone>(
     c: &mut Criterion,
@@ -76,7 +79,7 @@ fn bench_serialization<T: Serialize + DeserializeOwned + ToBytes + FromBytes + C
 }
 
 fn subdag_serialization(c: &mut Criterion) {
-    let rng = &mut TestRng::default();
+    let rng = &mut TestRng::fixed(BENCH_RNG_SEED);
     let subdag = sample_subdag(rng);
     let batch = subdag.iter().next().unwrap().1.iter().next().unwrap().clone();
     let batch_header = batch.batch_header().clone();
@@ -86,10 +89,6 @@ fn subdag_serialization(c: &mut Criterion) {
     bench_serialization(c, "Subdag", subdag.clone());
 }
 
-criterion_group! {
-    name = subdag;
-    config = Criterion::default().sample_size(10);
-    targets = subdag_serialization
-}
+criterion_group!(subdag, subdag_serialization);
 
 criterion_main!(subdag);

ledger/benches/dag.rs

@@ -36,6 +36,9 @@ type LedgerType = snarkvm_ledger_store::helpers::memory::ConsensusMemory<Mainnet
 #[cfg(feature = "rocks")]
 type LedgerType = snarkvm_ledger_store::helpers::rocksdb::ConsensusDB<MainnetV0>;
 
+/// Fixed RNG seed so benchmark inputs are reproducible across CI runs.
+const BENCH_RNG_SEED: u64 = 0xB34D_CAFE_CDEC_0123;
+
 fn initialize_vm<R: Rng + CryptoRng>(
     private_key: &PrivateKey<MainnetV0>,
     rng: &mut R,
@@ -60,7 +63,7 @@ fn initialize_vm<R: Rng + CryptoRng>(
 }
 
 fn deploy(c: &mut Criterion) {
-    let rng = &mut TestRng::default();
+    let rng = &mut TestRng::fixed(BENCH_RNG_SEED);
 
     // Sample a new private key and address.
     let private_key = PrivateKey::<MainnetV0>::new(rng).unwrap();
@@ -93,7 +96,7 @@ function hello:
 }
 
 fn execute(c: &mut Criterion) {
-    let rng = &mut TestRng::default();
+    let rng = &mut TestRng::fixed(BENCH_RNG_SEED ^ 1);
 
     // Sample a new private key and address.
     let private_key = PrivateKey::<MainnetV0>::new(rng).unwrap();
@@ -135,8 +138,12 @@ fn execute(c: &mut Criterion) {
 
         // Bench the Transaction.write_le method using the LimitedWriter.
         c.bench_function("LimitedWriter::new - transfer_public", |b| {
+            let max = MainnetV0::LATEST_MAX_TRANSACTION_SIZE();
             let mut buffer = Vec::with_capacity(3000);
-            b.iter(|| transaction.write_le(LimitedWriter::new(&mut buffer, MainnetV0::LATEST_MAX_TRANSACTION_SIZE())))
+            b.iter(|| {
+                buffer.clear();
+                transaction.write_le(LimitedWriter::new(&mut buffer, max))
+            })
         });
 
         // Bench the execution of transfer_public.
@@ -181,8 +188,12 @@ fn execute(c: &mut Criterion) {
 
         // Bench the Transaction.write_le method using the LimitedWriter.
         c.bench_function("LimitedWriter::new - transfer_private", |b| {
+            let max = MainnetV0::LATEST_MAX_TRANSACTION_SIZE();
             let mut buffer = Vec::with_capacity(3000);
-            b.iter(|| transaction.write_le(LimitedWriter::new(&mut buffer, MainnetV0::LATEST_MAX_TRANSACTION_SIZE())))
+            b.iter(|| {
+                buffer.clear();
+                transaction.write_le(LimitedWriter::new(&mut buffer, max))
+            })
         });
 
         // Bench the check_transaction method.
@@ -270,21 +281,24 @@ function main:
 
         // Bench the Transaction.write_le method using the LimitedWriter.
         c.bench_function("LimitedWriter::new - too_big.aleo", |b| {
-            let mut buffer = Vec::with_capacity(MainnetV0::LATEST_MAX_TRANSACTION_SIZE());
-            b.iter(|| transaction.write_le(LimitedWriter::new(&mut buffer, MainnetV0::LATEST_MAX_TRANSACTION_SIZE())))
+            let max = MainnetV0::LATEST_MAX_TRANSACTION_SIZE();
+            let mut buffer = Vec::with_capacity(max);
+            b.iter(|| {
+                buffer.clear();
+                transaction.write_le(LimitedWriter::new(&mut buffer, max))
+            })
         });
 
-        // Bench the check_transaction method.
-        c.bench_function("Transaction::Execute(too_big.aleo) - verify", |b| {
-            b.iter(|| vm.check_transaction(&transaction, None, rng))
+        // At genesis height the active cap is V1 (128 KiB); this transaction exceeds it and rejects before full verification.
+        c.bench_function("Transaction::Execute(too_big.aleo) - oversize_reject", |b| {
+            b.iter(|| {
+                vm.check_transaction(&transaction, None, rng)
+                    .expect_err("transaction must exceed V1 MAX_TRANSACTION_SIZE at genesis height");
+            })
         });
     }
 }
 
-criterion_group! {
-    name = transaction;
-    config = Criterion::default().sample_size(10);
-    targets = deploy, execute
-}
+criterion_group!(transaction, deploy, execute);
 
 criterion_main!(transaction);

ledger/benches/transaction.rs

@@ -27,7 +27,7 @@ edition = "2024"
 default = [ "rayon" ]
 serial = [ "snarkvm-console/serial" ]
 wasm = [ "snarkvm-console/wasm" ]
-test-helpers = [ "snarkvm-ledger-narwhal-transmission-id/test-helpers", "time" ]
+test-helpers = [ "snarkvm-ledger-narwhal-transmission-id/test-helpers" ]
 
 [dependencies.snarkvm-console]
 workspace = true
@@ -47,10 +47,6 @@ optional = true
 workspace = true
 features = [ "preserve_order" ]
 
-[dependencies.time]
-workspace = true
-optional = true
-
 [dev-dependencies.bincode]
 workspace = true
 

ledger/narwhal/batch-header/Cargo.toml

@@ -273,9 +273,11 @@ impl<N: Network> BatchHeader<N> {
 #[cfg(any(test, feature = "test-helpers"))]
 pub mod test_helpers {
     use super::*;
-    use console::{account::PrivateKey, network::MainnetV0, prelude::TestRng};
-
-    use time::OffsetDateTime;
+    use console::{
+        account::PrivateKey,
+        network::MainnetV0,
+        prelude::{TestRng, Uniform},
+    };
 
     type CurrentNetwork = MainnetV0;
 
@@ -322,8 +324,8 @@ pub mod test_helpers {
         let transmission_ids = snarkvm_ledger_narwhal_transmission_id::test_helpers::sample_transmission_ids(rng)
             .into_iter()
             .collect::<IndexSet<_>>();
-        // Checkpoint the timestamp for the batch.
-        let timestamp = OffsetDateTime::now_utc().unix_timestamp();
+        // Derive the timestamp from the RNG so sample data is reproducible for a fixed seed.
+        let timestamp = i64::rand(rng);
         // Return the batch header.
         BatchHeader::new(private_key, round, timestamp, committee_id, transmission_ids, previous_certificate_ids, rng)
             .unwrap()