1.5.x cloud benchmarks

.github/workflows/ci.yaml

@@ -836,10 +836,14 @@ jobs:
           - platform: linux/amd64
             name: linux/amd64
             target: x86_64-unknown-linux-gnu
+            target_cpu_env: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS
+            target_cpu: "-C target-cpu=x86-64-v3"
           - platform: linux/arm64
             name: linux/arm64
             target: aarch64-unknown-linux-gnu
             cross: true
+            target_cpu_env: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS
+            target_cpu: "-C target-cpu=neoverse-n1"
     name: docker-targets-build (${{ matrix.platform.platform }})
     steps:
       - uses: actions/checkout@v5
@@ -854,6 +858,12 @@ jobs:
         run: |
           platform=${{ matrix.platform.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+          # Target-scoped rustflags pin the instruction-set baseline for the
+          # published images: x86-64-v3 on amd64, Neoverse-N1 on arm64. The
+          # per-target CARGO_TARGET_*_RUSTFLAGS form is used because plain
+          # RUSTFLAGS is ignored when cross-compiling; CARGO_-prefixed vars are
+          # also passed through into the cross container automatically.
+          echo "${{ matrix.platform.target_cpu_env }}=${{ matrix.platform.target_cpu }}" >> $GITHUB_ENV
 
       - run: cargo install cross
         if: ${{ matrix.platform.cross }}

Cargo.toml

@@ -154,10 +154,13 @@ log = "0.4.26"
 mac_address = "1.1.8"
 mappable-rc = "0.1.1"
 md5 = "0.7.0"
+metrics = "0.24.2"
+metrics-exporter-prometheus = { version = "0.16.2", default-features = false }
 metrohash = "1.0.7"
 miette = { version = "7.6.0", features = ["fancy"] }
 mime = "0.3.17"
 mime_guess = "2.0.5"
+mimalloc = "0.1.52"
 minijinja = "2.7.0"
 
 nanoid = "0.4.0"
@@ -248,6 +251,7 @@ textwrap = "0.16.1"
 thiserror = "2.0.12"
 time = { version = "0.3.41", features = ["default", "macros"] }
 tokio = { version = "1.44", features = ["macros", "rt-multi-thread", "sync", "io-std", "net", "tracing", "process", "signal"] }
+tokio-metrics = { version = "0.5.0", features = ["metrics-rs-integration"] }
 tokio-postgres = "0.7.13"
 tokio-rustls = { version = "0.26.2" }
 tokio-stream = { version = "0.1", features = ["sync"] }
@@ -337,6 +341,7 @@ debug = "line-tables-only"
 
 [profile.release]
 panic = "abort"
+lto = "thin"
 
 [profile.benchmarks]
 inherits = "release"

golem-debugging-service/config/debug-worker-executor.sample.env

@@ -55,6 +55,8 @@ GOLEM__LIMITS__MAX_CONCURRENT_STREAMS=1024
 GOLEM__LIMITS__MAX_INVOCATION_CONTEXT_STACK_DEPTH=1024
 GOLEM__LIMITS__MAX_OPLOG_QUERY_PAGES_SIZE=100
 GOLEM__MEMORY__ACQUIRE_RETRY_DELAY="500ms"
+GOLEM__MEMORY__COMPONENT_SIZE_COEFFICIENT=2.0
+GOLEM__MEMORY__ENABLE_MEASURED_ADMISSION=true
 #GOLEM__MEMORY__SYSTEM_MEMORY_OVERRIDE=
 GOLEM__MEMORY__WORKER_ESTIMATE_COEFFICIENT=1.1
 GOLEM__MEMORY__WORKER_MEMORY_RATIO=0.8
@@ -228,6 +230,8 @@ GOLEM__LIMITS__MAX_CONCURRENT_STREAMS=1024
 GOLEM__LIMITS__MAX_INVOCATION_CONTEXT_STACK_DEPTH=1024
 GOLEM__LIMITS__MAX_OPLOG_QUERY_PAGES_SIZE=100
 GOLEM__MEMORY__ACQUIRE_RETRY_DELAY="500ms"
+GOLEM__MEMORY__COMPONENT_SIZE_COEFFICIENT=2.0
+GOLEM__MEMORY__ENABLE_MEASURED_ADMISSION=true
 #GOLEM__MEMORY__SYSTEM_MEMORY_OVERRIDE=
 GOLEM__MEMORY__WORKER_ESTIMATE_COEFFICIENT=1.1
 GOLEM__MEMORY__WORKER_MEMORY_RATIO=0.8

golem-debugging-service/config/debug-worker-executor.toml

@@ -96,6 +96,8 @@ max_oplog_query_pages_size = 100
 
 [memory]
 acquire_retry_delay = "500ms"
+component_size_coefficient = 2.0
+enable_measured_admission = true
 worker_estimate_coefficient = 1.1
 worker_memory_ratio = 0.8
 
@@ -364,6 +366,8 @@ without_time = false
 # 
 # [memory]
 # acquire_retry_delay = "500ms"
+# component_size_coefficient = 2.0
+# enable_measured_admission = true
 # worker_estimate_coefficient = 1.1
 # worker_memory_ratio = 0.8
 # 

golem-debugging-service/src/config.rs

@@ -98,6 +98,7 @@ impl DebugConfig {
             max_in_function_retry_delay: std::time::Duration::from_secs(20),
             max_websocket_connections: 100,
             quota_service: QuotaServiceConfig::default(),
+            runtime_metrics_sampling_interval: std::time::Duration::from_secs(5),
         }
     }
 }

golem-debugging-service/src/lib.rs

@@ -375,13 +375,19 @@ pub async fn run_debug_worker_executor<T: Bootstrap<DebugContext> + ?Sized + Sen
 ) -> anyhow::Result<RunDetails> {
     debug!("Initializing debug worker executor");
 
-    let total_system_memory = golem_config.memory.total_system_memory();
-    let system_memory = golem_config.memory.system_memory();
-    let worker_memory = golem_config.memory.worker_memory();
+    let memory_snapshot =
+        golem_worker_executor::services::active_workers::memory_probe::default_probe(
+            golem_config.memory.system_memory_override,
+        )
+        .snapshot();
+    let total_system_memory = memory_snapshot.limit_bytes;
+    let used_system_memory = memory_snapshot.current_bytes;
+    let worker_memory =
+        (total_system_memory as f64 * golem_config.memory.worker_memory_ratio) as u64;
     info!(
-        "Total system memory: {}, Available system memory: {}, Total memory available for workers: {}",
+        "Measured memory limit: {}, Currently used: {}, Usable for workers: {}",
         ISizeFormatter::new(total_system_memory, humansize::BINARY),
-        ISizeFormatter::new(system_memory, humansize::BINARY),
+        ISizeFormatter::new(used_system_memory, humansize::BINARY),
         ISizeFormatter::new(worker_memory, humansize::BINARY)
     );
 

golem-service-base/src/observability.rs

@@ -18,21 +18,42 @@ use axum::response::IntoResponse;
 use axum::routing::get;
 use http::Response;
 use prometheus::{Encoder, Registry, TextEncoder};
+use std::sync::Arc;
 use tokio::net::{TcpListener, ToSocketAddrs};
 use tokio::task::JoinSet;
 use tracing::{Instrument, info};
 
+/// A callback that renders additional metrics in Prometheus text exposition
+/// format, appended to the output of the `prometheus`-crate registry on the
+/// `/metrics` endpoint. Used to surface metrics from a second metrics façade
+/// (e.g. the `metrics`-crate recorder driving tokio-metrics) on the same
+/// scrape endpoint.
+pub type ExtraMetrics = Arc<dyn Fn() -> String + Send + Sync>;
+
 pub async fn start_health_and_metrics_server(
     addr: impl ToSocketAddrs,
     registry: Registry,
     body_message: &'static str,
     join_set: &mut JoinSet<Result<(), anyhow::Error>>,
+) -> Result<u16, anyhow::Error> {
+    start_health_and_metrics_server_with_extra(addr, registry, None, body_message, join_set).await
+}
+
+pub async fn start_health_and_metrics_server_with_extra(
+    addr: impl ToSocketAddrs,
+    registry: Registry,
+    extra: Option<ExtraMetrics>,
+    body_message: &'static str,
+    join_set: &mut JoinSet<Result<(), anyhow::Error>>,
 ) -> Result<u16, anyhow::Error> {
     let app = Router::new()
         .route("/healthcheck", get(move || async move { body_message }))
         .route(
             "/metrics",
-            get(|| async move { prometheus_metrics(registry.clone()) }),
+            get(move || {
+                let extra = extra.clone();
+                async move { prometheus_metrics(registry.clone(), extra) }
+            }),
         );
 
     let listener = TcpListener::bind(addr).await?;
@@ -51,13 +72,17 @@ pub async fn start_health_and_metrics_server(
     Ok(local_addr.port())
 }
 
-pub fn prometheus_metrics(registry: Registry) -> impl IntoResponse {
+pub fn prometheus_metrics(registry: Registry, extra: Option<ExtraMetrics>) -> impl IntoResponse {
     let encoder = TextEncoder::new();
     let mut buffer = Vec::new();
 
     let metric_families = registry.gather();
     encoder.encode(&metric_families, &mut buffer).unwrap();
 
+    if let Some(extra) = extra {
+        buffer.extend_from_slice(extra().as_bytes());
+    }
+
     Response::builder()
         .header("Content-Type", encoder.format_type())
         .body(Body::from(buffer))