Skip to content

Commit 85e8f60

Browse files
Merge branch 'main' into users/kmontrose/VADDUpdates
2 parents 6497b46 + fcdb5fc commit 85e8f60

43 files changed

Lines changed: 1428 additions & 395 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,32 +222,32 @@ jobs:
222222
os: [ ubuntu-latest, windows-latest ]
223223
framework: [ 'net8.0', 'net10.0' ]
224224
configuration: [ 'Debug', 'Release' ]
225-
test: [ 'Tsavorite.test', 'Tsavorite.test.recordops', 'Tsavorite.test.session', 'Tsavorite.test.session.context', 'Tsavorite.test.hlog', 'Tsavorite.test.recovery' ]
225+
test: [ 'Tsavorite.test', 'Tsavorite.test.recordops', 'Tsavorite.test.session', 'Tsavorite.test.session.context', 'Tsavorite.test.hlog', 'Tsavorite.test.recovery', 'Tsavorite.test.epoch' ]
226226
if: needs.changes.outputs.tsavorite == 'true'
227227
steps:
228228
- name: Check out code
229229
uses: actions/checkout@v6
230230
- name: Set environment variable for Linux
231231
run: echo "RunAzureTests=yes" >> $GITHUB_ENV
232-
if: ${{ matrix.os == 'ubuntu-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }}
232+
if: ${{ matrix.os == 'ubuntu-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' && matrix.test != 'Tsavorite.test.epoch' }}
233233
- name: Set environment variable for Windows
234234
run: echo ("RunAzureTests=yes") >> $env:GITHUB_ENV
235-
if: ${{ matrix.os == 'windows-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }}
235+
if: ${{ matrix.os == 'windows-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' && matrix.test != 'Tsavorite.test.epoch' }}
236236
- name: Setup .NET
237237
uses: actions/setup-dotnet@v5
238238
- name: Setup Node.js for Azurite
239-
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }}
239+
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' && matrix.test != 'Tsavorite.test.epoch' }}
240240
uses: actions/setup-node@v6
241241
with:
242242
node-version: 22
243243
- name: Cache Azurite
244-
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }}
244+
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' && matrix.test != 'Tsavorite.test.epoch' }}
245245
uses: actions/cache@v5
246246
with:
247247
path: ${{ runner.os == 'Windows' && '%APPDATA%\npm-cache' || '~/.npm' }}
248248
key: azurite-${{ runner.os }}
249249
- name: Install and Run Azurite
250-
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }}
250+
if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' && matrix.test != 'Tsavorite.test.epoch' }}
251251
shell: bash
252252
run: |
253253
npm install -g azurite
@@ -271,6 +271,7 @@ jobs:
271271
'Tsavorite.test.session.context' = 'libs/storage/Tsavorite/cs/test/test.session.context'
272272
'Tsavorite.test.hlog' = 'libs/storage/Tsavorite/cs/test/test.hlog'
273273
'Tsavorite.test.recovery' = 'libs/storage/Tsavorite/cs/test/test.recovery'
274+
'Tsavorite.test.epoch' = 'libs/storage/Tsavorite/cs/test/test.epoch'
274275
}
275276
$dir = $tsavoriteDirMap['${{ matrix.test }}']
276277
echo "TSAVORITE_TEST_DIR=$dir" >> $env:GITHUB_ENV

Garnet.slnx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@
7777
<File Path="Version.props" />
7878
</Folder>
7979
<Folder Name="/test/">
80-
<Project Path="libs/storage/Tsavorite/cs/test/test.epoch/Tsavorite.test.epoch.csproj" />
8180
<Project Path="test/Garnet.fuzz/Garnet.fuzz.csproj" />
8281
<Project Path="test/standalone/BfTreeInterop.test/BfTreeInterop.test.csproj" />
8382
<Project Path="test/standalone/Garnet.test/Garnet.test.csproj" />

benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBench.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public static GarnetServerOptions GetServerOptions(Options options)
2828
CommitFrequencyMs = options.CommitFrequencyMs,
2929
AofPhysicalSublogCount = options.AofPhysicalSublogCount,
3030
AofReplayTaskCount = options.AofReplayTaskCount,
31-
ReplicationOffsetMaxLag = 0,
31+
AofReplayMaxLagBytes = 0,
3232
CheckpointDir = OperatingSystem.IsLinux() ? "/tmp" : null,
3333
};
3434
return serverOptions;

benchmark/Resp.benchmark/OfflineBench/AOFBench/AofGen.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ public AofGen(Options options)
234234
CommitFrequencyMs = -1,
235235
FastAofTruncate = true,
236236
EnableCluster = true,
237-
ReplicationOffsetMaxLag = 0,
237+
AofReplayMaxLagBytes = 0,
238238
AofPhysicalSublogCount = options.AofPhysicalSublogCount
239239
};
240240
aofServerOptions.GetAofSettings(0, out var logSettings);

benchmark/Resp.benchmark/OfflineBench/GarnetServerInstance.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) Microsoft Corporation.
1+
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT license.
33

44
using System.Net;
@@ -27,7 +27,7 @@ public static GarnetServerOptions GetServerOptions(Options options)
2727
CommitFrequencyMs = options.CommitFrequencyMs,
2828
AofPhysicalSublogCount = options.AofPhysicalSublogCount,
2929
AofReplayTaskCount = options.AofReplayTaskCount,
30-
ReplicationOffsetMaxLag = 0,
30+
AofReplayMaxLagBytes = 0,
3131
CheckpointDir = OperatingSystem.IsLinux() ? "/tmp" : null,
3232
};
3333
return serverOptions;

libs/client/ClientSession/GarnetClientSessionReplicationExtensions.cs

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -548,19 +548,16 @@ public void SetClusterSyncHeader(string sourceNodeId)
548548
}
549549

550550
/// <summary>
551-
/// Issue CLUSTER ADVANCE_TIME
551+
/// Issue an in-band CLUSTER ADVANCE_TIME pulse for one physical sublog. No response is
552+
/// expected, so the pulse remains ordered with APPENDLOG traffic on this connection.
552553
/// </summary>
554+
/// <param name="physicalSublogIdx"></param>
553555
/// <param name="sequenceNumber"></param>
554-
/// <param name="aofAddress"></param>
555-
/// <returns></returns>
556556
/// <seealso cref="M:Garnet.cluster.ClusterSession.NetworkClusterAdvanceTime"/>
557-
public Task<string> ExecuteClusterAdvanceTime(long sequenceNumber, Span<byte> aofAddress)
557+
public void ExecuteClusterAdvanceTime(int physicalSublogIdx, long sequenceNumber)
558558
{
559-
var tcs = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
560-
tcsQueue.Enqueue(tcs);
561559
var curr = offset;
562-
var argCount = 2;
563-
var arraySize = 2 + argCount;
560+
const int arraySize = 4;
564561

565562
while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end))
566563
{
@@ -586,24 +583,20 @@ public Task<string> ExecuteClusterAdvanceTime(long sequenceNumber, Span<byte> ao
586583
offset = curr;
587584

588585
//3
589-
while (!RespWriteUtils.TryWriteArrayItem(sequenceNumber, ref curr, end))
586+
while (!RespWriteUtils.TryWriteArrayItem(physicalSublogIdx, ref curr, end))
590587
{
591588
Flush();
592589
curr = offset;
593590
}
594591
offset = curr;
595592

596593
//4
597-
while (!RespWriteUtils.TryWriteBulkString(aofAddress, ref curr, end))
594+
while (!RespWriteUtils.TryWriteArrayItem(sequenceNumber, ref curr, end))
598595
{
599596
Flush();
600597
curr = offset;
601598
}
602599
offset = curr;
603-
604-
Flush();
605-
Interlocked.Increment(ref numCommands);
606-
return tcs.Task;
607600
}
608601

609602
/// <summary>

libs/cluster/Server/ClusterProvider.cs

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) Microsoft Corporation.
1+
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT license.
33

44
using System;
@@ -239,7 +239,9 @@ public MetricsItem[] GetReplicationInfo()
239239
{
240240
var (address, port) = config.GetLocalNodePrimaryAddress();
241241
var primaryLinkStatus = clusterManager.GetPrimaryLinkStatus(config);
242-
var replicationOffsetLag = storeWrapper.appendOnlyFile.Log.TailAddress.AggregateDiff(replicationManager.ReplicationOffset);
242+
var replicationOffsetAccLag = storeWrapper.appendOnlyFile.Log.TailAddress.AggregateDiff(replicationManager.ReplicationOffset);
243+
var replicationOffsetVectorLag = storeWrapper.appendOnlyFile.Log.TailAddress.Diff(replicationManager.ReplicationOffset);
244+
var rcm = storeWrapper.appendOnlyFile.readConsistencyManager;
243245
replicationInfo.Add(new("master_host", address));
244246
replicationInfo.Add(new("master_port", port.ToString()));
245247
replicationInfo.Add(primaryLinkStatus[0]);
@@ -250,8 +252,11 @@ public MetricsItem[] GetReplicationInfo()
250252
replicationInfo.Add(new("slave_read_only", "1"));
251253
replicationInfo.Add(new("replica_announced", "1"));
252254
replicationInfo.Add(new("master_sync_last_io_seconds_ago", replicationManager.LastPrimarySyncSeconds.ToString()));
253-
replicationInfo.Add(new("replication_offset_lag", replicationOffsetLag.ToString()));
254-
replicationInfo.Add(new("replication_offset_max_lag", storeWrapper.runtimeConfig.GetInt(ServerConfigType.REPLICATION_OFFSET_MAX_LAG).ToString()));
255+
replicationInfo.Add(new("replication_offset_vector_lag", replicationOffsetVectorLag.ToString()));
256+
replicationInfo.Add(new("replication_offset_acc_lag", replicationOffsetAccLag.ToString()));
257+
replicationInfo.Add(new("aof_replay_max_lag_bytes", storeWrapper.serverOptions.AofReplayMaxLagBytes.ToString()));
258+
replicationInfo.Add(new("physical_sublog_max_sequence_vector", rcm == null ? "-1" : rcm.GetPhysicalSublogMaxSequenceVector()));
259+
replicationInfo.Add(new("physical_sublog_max_drift_sequence_vector", rcm == null ? "-1" : rcm.GetPhysicalSublogMaxDriftSequenceVector()));
255260
}
256261
else
257262
{

libs/cluster/Server/Failover/ReplicaFailoverSession.cs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,6 @@ private async Task<bool> TakeOverAsPrimaryAsync()
152152
if (clusterProvider.serverOptions.AofPhysicalSublogCount > 1)
153153
{
154154
clusterProvider.storeWrapper.appendOnlyFile.ResetSequenceNumberGenerator();
155-
await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false);
156155
}
157156

158157
// Initialize checkpoint history
@@ -161,10 +160,6 @@ private async Task<bool> TakeOverAsPrimaryAsync()
161160

162161
_ = clusterProvider.BumpAndWaitForEpochTransitionAsync().ConfigureAwait(false);
163162

164-
// Stop advance time task when reconfiguring node to be replica
165-
if (clusterProvider.storeWrapper.serverOptions.AofPhysicalSublogCount > 1)
166-
await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false);
167-
168163
// Resume all background maintenance that were possibly shutdown when this node became a replica
169164
clusterProvider.storeWrapper.StartPrimaryTasks();
170165
}

libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriver.cs

Lines changed: 14 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -69,23 +69,28 @@ public AofAddress PreviousAddress
6969
}
7070

7171
/// <summary>
72-
/// Return previous address for a specific sublog without copying the full AofAddress struct
72+
/// Return previous address for a specific sublog without copying the full AofAddress struct.
73+
/// This is the address actually shipped (Consume'd) to the replica, used to bound AOF
74+
/// truncation conservatively.
7375
/// </summary>
7476
/// <param name="physicalSublogIdx">Index of the physical sublog.</param>
7577
/// <returns>The previous address of the specified sublog's sync task.</returns>
7678
public long GetPreviousAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].PreviousAddress;
7779

7880
/// <summary>
79-
/// Return start address for a specific sublog without copying the full AofAddress struct
81+
/// Return the shipped watermark address for a specific sublog (the monotonic high-water this
82+
/// replica's sync task has shipped/scanned past), used to drive the backpressure gate.
8083
/// </summary>
8184
/// <param name="physicalSublogIdx">Index of the physical sublog.</param>
82-
/// <returns>The start address of the specified sublog's sync task.</returns>
83-
public long GetStartAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].StartAddress;
85+
/// <returns>The shipped watermark address of the specified sublog's sync task.</returns>
86+
public long GetShippedWatermarkAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].ShippedWatermarkAddress;
8487

8588
/// <summary>
86-
/// Replica endpoint
89+
/// Return start address for a specific sublog without copying the full AofAddress struct
8790
/// </summary>
88-
readonly IPEndPoint endPoint;
91+
/// <param name="physicalSublogIdx">Index of the physical sublog.</param>
92+
/// <returns>The start address of the specified sublog's sync task.</returns>
93+
public long GetStartAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].StartAddress;
8994

9095
public AofSyncDriver(
9196
ClusterProvider clusterProvider,
@@ -100,13 +105,12 @@ public AofSyncDriver(
100105
this.aofSyncDriverStore = aofSyncDriverStore;
101106
this.localNodeId = localNodeId;
102107
this.remoteNodeId = remoteNodeId;
103-
this.endPoint = endPoint;
104108
cts = new();
105109
this.logger = logger;
106110

107111
aofSyncTasks = new AofSyncTask[clusterProvider.serverOptions.AofPhysicalSublogCount];
108112
for (var physicalSublogIdx = 0; physicalSublogIdx < aofSyncTasks.Length; physicalSublogIdx++)
109-
aofSyncTasks[physicalSublogIdx] = new AofSyncTask(clusterProvider, physicalSublogIdx, endPoint, startAddress[physicalSublogIdx], localNodeId, remoteNodeId, cts, logger);
113+
aofSyncTasks[physicalSublogIdx] = new AofSyncTask(clusterProvider, aofSyncDriverStore, physicalSublogIdx, endPoint, startAddress[physicalSublogIdx], localNodeId, remoteNodeId, cts, logger);
110114
}
111115

112116
/// <summary>
@@ -152,10 +156,9 @@ public async Task RunAsync()
152156
}
153157
else
154158
{
155-
var tasks = new Task[aofSyncTasks.Length + 1];
156-
tasks[0] = AdvancePhysicalSublogTimeAsync();
159+
var tasks = new Task[aofSyncTasks.Length];
157160
for (var i = 0; i < aofSyncTasks.Length; i++)
158-
tasks[i + 1] = aofSyncTasks[i].RunAofSyncTaskAsync(this);
161+
tasks[i] = aofSyncTasks[i].RunAofSyncTaskAsync(this);
159162

160163
_ = await Task.WhenAny(tasks).ConfigureAwait(false);
161164
}
@@ -174,60 +177,6 @@ public async Task RunAsync()
174177
}
175178
}
176179

177-
/// <summary>
178-
/// Advance physical sublog time background task.
179-
/// </summary>
180-
/// <returns></returns>
181-
/// <exception cref="GarnetException"></exception>
182-
/// <seealso cref="T:Garnet.cluster.ReplicationManager.AdvanceTime"/>
183-
async Task AdvancePhysicalSublogTimeAsync()
184-
{
185-
var enteredMonitor = false;
186-
var client = new GarnetClientSession(
187-
endPoint,
188-
clusterProvider.replicationManager.GetAofSyncNetworkBufferSettings,
189-
clusterProvider.replicationManager.GetNetworkPool,
190-
tlsOptions: this.clusterProvider.serverOptions.TlsOptions?.TlsClientOptions,
191-
authUsername: this.clusterProvider.ClusterUsername,
192-
authPassword: this.clusterProvider.ClusterPassword,
193-
logger: logger);
194-
195-
try
196-
{
197-
enteredMonitor = activeWorkerMonitor.TryEnter();
198-
if (!enteredMonitor)
199-
throw new GarnetException($"Failed to acquire read lock at {nameof(AdvancePhysicalSublogTimeAsync)}");
200-
201-
// Connect to replica
202-
await client.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, cts.Token).ConfigureAwait(false);
203-
204-
var appendOnlyFile = clusterProvider.storeWrapper.appendOnlyFile;
205-
var previousTailAddress = AofAddress.Create(appendOnlyFile.Log.Size, 0);
206-
207-
while (!cts.IsCancellationRequested)
208-
{
209-
await Task.Delay(clusterProvider.storeWrapper.runtimeConfig.GetInt(ServerConfigType.AOF_TAIL_WITNESS_FREQ), cts.Token).ConfigureAwait(false);
210-
var currentTailAddress = appendOnlyFile.Log.TailAddress;
211-
var newWrites = previousTailAddress.AnyLesser(currentTailAddress);
212-
213-
if (newWrites)
214-
{
215-
var sequenceNumber = appendOnlyFile.GetLargerThanMaximumSequenceNumber();
216-
_ = await client.ExecuteClusterAdvanceTime(sequenceNumber, currentTailAddress.Span).
217-
WaitAsync(clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token).
218-
ConfigureAwait(false);
219-
previousTailAddress.MonotonicUpdate(ref currentTailAddress);
220-
}
221-
}
222-
}
223-
finally
224-
{
225-
if (enteredMonitor)
226-
_ = activeWorkerMonitor.Exit();
227-
client?.Dispose();
228-
}
229-
}
230-
231180
#region DisklesSyncInterface
232181
public async Task ConnectClientsAsync()
233182
{

0 commit comments

Comments
 (0)