[Tsavorite] Scalable origin-return buffer pool + direct-VM native allocator

This change introduces two independent performance features to Tsavorite/Garnet
for reducing GC pressure and improving buffer-pool scalability under many
concurrent IO-completion threads. Both are opt-in/rollback-guarded and default
to safe behavior.

1) Scalable origin-return SectorAlignedBufferPool (new default managed backend)
   - Replaces the single per-level ConcurrentQueue pool (which collapses under
     concurrent cross-thread Get/Return) with a per-thread magazine pool that
     routes each buffer back to its originating thread instead of the freeing
     (IO-completion) thread. This avoids the memory bloat / zero-reuse asymmetry
     where completion threads accumulate every buffer while issuing threads keep
     allocating fresh.
   - Ownership keyed by (pool, thread, size-class) via a [ThreadStatic]
     recyclable-slot scheme (no ThreadLocal<T>, no monotonic-index leak).
   - Seal-aware retirement/teardown protocol for dead threads and pool close,
     per-buffer exactly-once permit release with a standalone BudgetState +
     finalizer reclamation, a hard per-pool byte budget (anti-bloat), a
     constrained linear-then-geometric size-class ladder with O(1) lookup, and
     per-pool immutable mode + frozen UnpinOnReturn.
   - Kill-switch: --use-legacy-buffer-pool (UseLegacyBufferPool) restores the
     legacy ConcurrentQueue pool. Selected once per pool at construction via
     SectorAlignedBufferPool.UseOriginReturn.
   - Measured (Release, 160-core): origin-return scales 22 -> 1260 Mops/s
     (1 -> 64 threads) vs legacy 44 -> 1.0 (collapses); 8M ops -> 528
     allocations (8.2x working set); cross-thread p99 <= 2us.

2) Direct-VM native allocator (off | full)
   - Routes large memory regions (log pages / hash index / recovery frames) off
     the managed GC heap through direct OS virtual memory (mmap/VirtualAlloc).
     No native library is required.
   - New --native-allocator switch (NativeAllocator), NativeAllocatorSurfaces
     flags, NativeAllocatorInitializer/EnvironmentInitializer, DirectVirtualMemory,
     NativeMemoryTracker, and NativePageBlockRegistry.
   - New INFO field native_allocator_bytes surfaces off-heap committed bytes.
   - GC-sizing guidance documented in website/docs/getting-started/memory.md.

Tests: SectorAlignedBufferPoolTests (CI correctness), NativeAllocator*Tests
(hlog/recovery/server), config parsing tests for both switches, direct-VM
Docker validation, plus [Explicit] scaling/reuse/latency/soak stress harnesses.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: d0cba12e-76fa-44b7-8d5c-4c9d86c5c7b1

Author: badrishc

.github/actions/publish-native-prebuilts/action.yml

@@ -0,0 +1,111 @@
+# Shared publish step for the native-binary build workflow (Build Native Device). Given the
+# freshly built artifacts for the component, it copies them over the checked-in prebuilts and
+# commits the result back to the dispatched branch (updating an open PR in place), or opens a PR
+# when dispatched on a protected branch.
+#
+# The caller MUST run actions/checkout FIRST, on the branch being published and with the
+# push token (secrets.NATIVE_BINARIES_PAT || github.token). checkout persists those
+# credentials in git config, so the git push below uses them; this action does not check
+# out or take a push token itself.
+name: Publish native prebuilts
+description: Copy freshly built native binaries over the checked-in prebuilts and commit them back to the branch (or open a PR on main/dev).
+inputs:
+  branch:
+    description: Branch to publish onto (github.ref_name); must already be checked out.
+    required: true
+  gh-token:
+    description: Token for the gh CLI (opening a PR on protected branches).
+    required: true
+  artifact-pattern:
+    description: download-artifact pattern selecting this component's artifacts (e.g. native-*).
+    required: true
+  artifact-prefix:
+    description: Artifact-name prefix to strip to recover the RID (e.g. native-).
+    required: true
+  dest-base:
+    description: Checked-in runtimes base directory to overwrite (…/Device/runtimes).
+    required: true
+  label:
+    description: Component label for the commit / PR message (e.g. device).
+    required: true
+  run-id:
+    description: github.run_id, recorded in the commit / PR message for provenance.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Download ${{ inputs.label }} artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: staging-publish
+        pattern: ${{ inputs.artifact-pattern }}
+
+    - name: Copy artifacts over checked-in prebuilts
+      shell: bash
+      run: |
+        set -euo pipefail
+        base="${{ inputs.dest-base }}"
+        prefix="${{ inputs.artifact-prefix }}"
+        shopt -s nullglob
+        dirs=("staging-publish/${prefix}"*)
+        if [ ${#dirs[@]} -eq 0 ]; then
+          echo "No artifacts matched '${prefix}*' - nothing was built to publish." >&2
+          exit 1
+        fi
+        for d in "${dirs[@]}"; do
+          rid="$(basename "$d")"; rid="${rid#"$prefix"}"
+          echo "Updating $base/$rid/native/"
+          mkdir -p "$base/$rid/native"
+          cp -f "$d"/* "$base/$rid/native/"
+        done
+        # .so need the executable bit; Windows .dll/.pdb do not.
+        find "$base" -name '*.so' -exec chmod 755 {} +
+        git status --porcelain "$base"
+
+    - name: Commit and publish
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+      run: |
+        set -euo pipefail
+        base="${{ inputs.dest-base }}"
+        branch="${{ inputs.branch }}"
+        label="${{ inputs.label }}"
+        run_id="${{ inputs.run-id }}"
+
+        if [ -z "$(git status --porcelain "$base")" ]; then
+          echo "No changes to the checked-in $label prebuilts; nothing to do."
+          exit 0
+        fi
+
+        git config user.name  "github-actions[bot]"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        git add "$base"
+        git commit -m "[Tsavorite] Update prebuilt native $label binaries" \
+                   -m "Regenerated by the native build workflow (run $run_id) from the sources on '$branch'."
+
+        case "$branch" in
+          main|dev)
+            # Protected branches: never push directly; open a PR for review.
+            pr_branch="bot/update-native-${label}-${branch}"
+            git branch -M "$pr_branch"
+            git push --force-with-lease --set-upstream origin "$pr_branch"
+            if gh pr view "$pr_branch" --json number >/dev/null 2>&1; then
+              echo "PR already open for $pr_branch; branch updated."
+            else
+              gh pr create --base "$branch" --head "$pr_branch" \
+                --title "[Tsavorite] Update prebuilt native $label binaries" \
+                --body "Automated refresh of the checked-in $label prebuilts under \`$base\`, produced by the native build workflow (run $run_id)."
+            fi
+            ;;
+          *)
+            # Feature / PR branch: commit straight onto it so the open PR updates in place. Re-sync
+            # with the remote tip first in case the branch moved during the (multi-minute) build.
+            # Only $base files change here, which does not match either native workflow's source
+            # triggers - so this push cannot re-trigger a native build.
+            git pull --rebase origin "$branch"
+            git push origin "HEAD:${branch}"
+            echo "Pushed refreshed $label binaries onto '$branch'."
+            echo "If ci.yml did not start automatically (GITHUB_TOKEN push), push another commit or re-run ci.yml to validate against the new binaries."
+            ;;
+        esac

.github/workflows/native-build.yml

@@ -190,10 +190,10 @@ jobs:
           $cfg = & $dumpbin /loadconfig $dll
           $cfg | Write-Host
           if (-not ($cfg | Select-String -Quiet -Pattern 'Guard')) {
-            throw "Control Flow Guard (/guard:cf) not present in $dll — security flags did not take effect"
+            throw "Control Flow Guard (/guard:cf) not present in $dll - security flags did not take effect"
           }
           if (-not ($cfg | Select-String -Quiet -Pattern 'Security Cookie')) {
-            throw "Stack Security Cookie (/GS,/sdl) not present in $dll — security flags did not take effect"
+            throw "Stack Security Cookie (/GS,/sdl) not present in $dll - security flags did not take effect"
           }
           Write-Host "Security mitigations verified: Control Flow Guard + Security Cookie present."
 
@@ -206,6 +206,7 @@ jobs:
   # Publishes the freshly built binaries. When dispatched on a feature branch, it commits
   # them directly onto that branch so an open PR updates in place; on a protected branch
   # (main/dev) it opens a PR instead. Runs only for a manual dispatch with update_repo = true.
+  # The download/copy/commit-back logic lives in the publish-native-prebuilts composite action.
   publish:
     name: Publish binaries to ${{ github.ref_name }}
     needs: [build-linux, build-windows]
@@ -225,71 +226,15 @@ jobs:
           fetch-depth: 0
           # If a maintainer configured a PAT / App token, use it so the resulting push
           # re-triggers ci.yml; otherwise fall back to GITHUB_TOKEN (push lands but does not
-          # start new runs — ci.yml then runs on the developer's next push or a manual re-run).
+          # start new runs - ci.yml then runs on the developer's next push or a manual re-run).
           token: ${{ secrets.NATIVE_BINARIES_PAT || github.token }}
 
-      - name: Download all native artifacts
-        uses: actions/download-artifact@v4
+      - uses: ./.github/actions/publish-native-prebuilts
         with:
-          path: staging
-          pattern: native-*
-
-      - name: Copy artifacts over checked-in prebuilts
-        run: |
-          set -euo pipefail
-          base="libs/storage/Tsavorite/cs/src/core/Device/runtimes"
-          for d in staging/native-*; do
-            rid="$(basename "$d" | sed 's/^native-//')"
-            echo "Updating runtimes/$rid/native/"
-            mkdir -p "$base/$rid/native"
-            cp -f "$d"/* "$base/$rid/native/"
-          done
-          # .so need the executable bit; Windows .dll/.pdb do not.
-          find "$base" -name '*.so' -exec chmod 755 {} +
-          git status --porcelain "$base"
-
-      - name: Commit and publish
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -euo pipefail
-          base="libs/storage/Tsavorite/cs/src/core/Device/runtimes"
-          branch="${{ github.ref_name }}"
-
-          if [ -z "$(git status --porcelain "$base")" ]; then
-            echo "No changes to the checked-in prebuilts; nothing to do."
-            exit 0
-          fi
-
-          git config user.name  "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add "$base"
-          git commit -m "[Tsavorite] Update prebuilt native device binaries
-
-          Regenerated by the Build Native Device workflow (run ${{ github.run_id }}) from the native sources on '$branch'."
-
-          case "$branch" in
-            main|dev)
-              # Protected branches: never push directly; open a PR for review.
-              pr_branch="bot/update-native-binaries-${branch}"
-              git branch -M "$pr_branch"
-              git push --force-with-lease --set-upstream origin "$pr_branch"
-              if gh pr view "$pr_branch" --json number >/dev/null 2>&1; then
-                echo "PR already open for $pr_branch; branch updated."
-              else
-                gh pr create --base "$branch" --head "$pr_branch" \
-                  --title "[Tsavorite] Update prebuilt native device binaries" \
-                  --body "Automated refresh of the checked-in native prebuilts under \`$base\`, produced by the **Build Native Device** workflow (run ${{ github.run_id }})."
-              fi
-              ;;
-            *)
-              # Feature / PR branch: commit the refreshed binaries straight onto it so the open
-              # PR updates in place. Re-sync with the remote tip first in case the branch moved
-              # during the (multi-minute) build, then push. Only the runtimes/ files change here,
-              # and this workflow triggers on cc/** — so this push cannot re-trigger native-build.
-              git pull --rebase origin "$branch"
-              git push origin "HEAD:${branch}"
-              echo "Pushed refreshed native binaries onto '$branch'."
-              echo "If ci.yml did not start automatically (GITHUB_TOKEN push), push another commit or re-run ci.yml to validate against the new binaries."
-              ;;
-          esac
+          branch: ${{ github.ref_name }}
+          gh-token: ${{ github.token }}
+          artifact-pattern: native-*
+          artifact-prefix: native-
+          dest-base: libs/storage/Tsavorite/cs/src/core/Device/runtimes
+          label: device
+          run-id: ${{ github.run_id }}

libs/host/Configuration/Options.cs

@@ -87,6 +87,14 @@ internal sealed class Options : ICloneable
         [Option("index-max-size", Required = false, HelpText = "Max size of hash index in bytes (rounds down to power of 2)")]
         public string IndexMaxMemorySize { get; set; }
 
+        [Option("native-allocator", Required = false, HelpText = "Route large memory regions through a native (off-managed-heap) direct-VM allocator (mmap/VirtualAlloc). Values: off (default, fully managed), full (routes log pages / hash index / recovery frames off the GC heap). No native library is required. When enabled, this memory is outside the managed GC heap: size GCHeapHardLimit to leave headroom and monitor 'native_allocator_bytes' in INFO memory.")]
+        [NativeAllocatorModeValidation(false)]
+        public string NativeAllocator { get; set; }
+
+        [OptionValidation]
+        [Option("use-legacy-buffer-pool", Required = false, HelpText = "Select the per-level ConcurrentQueue SectorAlignedBufferPool instead of the default origin-return (per-thread magazine) pool. The origin-return pool returns each buffer to the thread that allocated it, so it scales with concurrent IO-completion threads under a per-pool byte budget.")]
+        public bool? UseLegacyBufferPool { get; set; }
+
         [PercentageValidation(false)]
         [Option("mutable-percent", Required = false, HelpText = "Percentage of log memory that is kept mutable")]
         public int MutablePercent { get; set; }
@@ -737,6 +745,41 @@ public bool IsValid(out List<string> invalidOptions, ILogger logger = null)
             return isValid;
         }
 
+        /// <summary>
+        /// Parse the <c>--native-allocator</c> mode string into its surface set. Returns <c>false</c> for an
+        /// unrecognized value (empty/unset maps to <see cref="NativeAllocatorSurfaces.None"/>).
+        /// </summary>
+        internal static bool TryParseNativeAllocatorMode(string mode, out NativeAllocatorSurfaces surfaces)
+        {
+            if (string.IsNullOrWhiteSpace(mode))
+            {
+                surfaces = NativeAllocatorSurfaces.None;
+                return true;
+            }
+            switch (mode.Trim().ToLowerInvariant())
+            {
+                case "off":
+                case "none":
+                case "managed":
+                    surfaces = NativeAllocatorSurfaces.None;
+                    return true;
+                case "full":
+                case "all":
+                    surfaces = NativeAllocatorSurfaces.Full;
+                    return true;
+                default:
+                    surfaces = NativeAllocatorSurfaces.None;
+                    return false;
+            }
+        }
+
+        static NativeAllocatorSurfaces ParseNativeAllocatorMode(string mode)
+        {
+            if (!TryParseNativeAllocatorMode(mode, out var surfaces))
+                throw new GarnetException($"Invalid --native-allocator value '{mode}'. Expected one of: off, full.");
+            return surfaces;
+        }
+
         public GarnetServerOptions GetServerOptions(ILogger logger = null)
         {
             var enableStorageTier = EnableStorageTier.GetValueOrDefault();
@@ -865,6 +908,8 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno
                 ObjectLogSegmentSize = ObjectLogSegmentSize,
                 IndexMemorySize = IndexMemorySize,
                 IndexMaxMemorySize = IndexMaxMemorySize,
+                NativeAllocatorSurfaces = ParseNativeAllocatorMode(NativeAllocator),
+                UseLegacyBufferPool = UseLegacyBufferPool.GetValueOrDefault(),
                 MutablePercent = MutablePercent,
                 EnableReadCache = EnableReadCache.GetValueOrDefault(),
                 ReadCacheMemorySize = ReadCacheMemorySize,

libs/host/Configuration/OptionsValidators.cs

@@ -412,6 +412,32 @@ protected override ValidationResult IsValid(object value, ValidationContext vali
         }
     }
 
+    /// <summary>
+    /// Validation logic for the <c>--native-allocator</c> mode string (off | full).
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Property)]
+    internal sealed class NativeAllocatorModeValidationAttribute : OptionValidationAttribute
+    {
+        internal NativeAllocatorModeValidationAttribute(bool isRequired = true) : base(isRequired)
+        {
+        }
+
+        protected override ValidationResult IsValid(object value, ValidationContext validationContext)
+        {
+            if (TryInitialValidation<string>(value, validationContext, out var initValidationResult, out var mode))
+                return initValidationResult;
+
+            if (!Options.TryParseNativeAllocatorMode(mode, out _))
+            {
+                var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty;
+                var errorMessage = $"{baseError} Expected one of: off, full. Actual value: {mode}";
+                return new ValidationResult(errorMessage, [validationContext.MemberName]);
+            }
+
+            return ValidationResult.Success;
+        }
+    }
+
     /// <summary>
     /// Validation logic for an integer representing a percentage (range between 0 and 100)
     /// </summary>

libs/host/GarnetServer.cs

@@ -199,6 +199,16 @@ private void InitializeServer()
                 string.Join(',', opts.EndPoints.Select(endpoint => endpoint.ToString())));
             logger?.LogInformation("Environment .NET {netVersion}; {osPlatform}; {processArch}", Environment.Version, Environment.OSVersion.Platform, RuntimeInformation.ProcessArchitecture);
 
+            // Resolve and install native (off-managed-heap) allocators before any store or buffer pool is created.
+            // Always call Initialize (even for None) so the resolved CLI/config scope is authoritative and any
+            // process-wide leftover (e.g. from the GARNET_NATIVE_ALLOCATOR env var) is reset to the managed path.
+            NativeAllocatorInitializer.Initialize(opts.NativeAllocatorSurfaces, loggerFactory?.CreateLogger("NativeAllocator"));
+
+            // Select the managed SectorAlignedBufferPool backend before any pool is constructed (each pool captures
+            // this static once at construction). Default is the origin-return per-thread magazine pool; the flag
+            // selects the per-level ConcurrentQueue pool instead.
+            SectorAlignedBufferPool.UseOriginReturn = !opts.UseLegacyBufferPool;
+
             // Flush initialization logs from memory logger
             FlushMemoryLogger(this.initLogger, "ArgParser", this.loggerFactory);
 

libs/host/defaults.conf

@@ -42,6 +42,16 @@
 	/* Max size of hash index in bytes (rounds down to power of 2) */
 	"IndexMaxMemorySize": "",
 
+	/* Route large memory regions through a native (off-managed-heap) direct-VM allocator (mmap/VirtualAlloc).
+	   Values: "off" (default, fully managed), "full" (routes log pages / hash index / recovery frames off the
+	   GC heap). No native library is required. */
+	"NativeAllocator": "off",
+
+	/* Use the legacy per-level ConcurrentQueue SectorAlignedBufferPool instead of the default origin-return
+	   (per-thread magazine) pool. The origin-return pool scales far better under many concurrent IO-completion
+	   threads; this flag is a rollback/kill-switch. */
+	"UseLegacyBufferPool": false,
+
 	/* Percentage of log memory that is kept mutable */
 	"MutablePercent" : 90,
 

libs/server/Metrics/Info/GarnetInfoMetrics.cs

@@ -140,6 +140,7 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper)
                 new("gc_heap_bytes", gcMemoryInfo.HeapSizeBytes.ToString()),
                 new("gc_managed_memory_bytes_excluding_heap", gcAvailableMemory.ToString()),
                 new("gc_fragmented_bytes", gcMemoryInfo.FragmentedBytes.ToString()),
+                new("native_allocator_bytes", Tsavorite.core.NativeMemoryTracker.Bytes.ToString()),
                 new("store_index_size", store_index_size.ToString()),
                 new("store_mainlog_memory_size", store_mainlog_memory_size.ToString()),
                 new("store_readcache_memory_size", store_readcache_memory_size.ToString()),

libs/server/Servers/GarnetServerOptions.cs

@@ -21,6 +21,23 @@ public class GarnetServerOptions : ServerOptions
         /// </summary>
         public bool DisableObjects = false;
 
+        /// <summary>
+        /// Which memory surfaces use a native (off-managed-heap) allocator. Resolved from the
+        /// <c>--native-allocator</c> mode (off | full) and installed at startup via
+        /// <see cref="NativeAllocatorInitializer"/>. The direct-VM surfaces (log pages / hash index / recovery
+        /// frames) call the OS virtual-memory APIs directly and are always available; no native library is required.
+        /// </summary>
+        public NativeAllocatorSurfaces NativeAllocatorSurfaces = NativeAllocatorSurfaces.None;
+
+        /// <summary>
+        /// Select the per-level <c>ConcurrentQueue</c> <see cref="SectorAlignedBufferPool"/> instead of the
+        /// default origin-return (per-thread magazine) pool. The origin-return pool returns each buffer to the
+        /// thread that allocated it, scaling with concurrent IO-completion threads under a per-pool byte budget.
+        /// Installed at startup by toggling <see cref="SectorAlignedBufferPool.UseOriginReturn"/> before any pool
+        /// is created.
+        /// </summary>
+        public bool UseLegacyBufferPool = false;
+
         /// <summary>
         /// Enable cluster.
         /// </summary>