fix(reminders): observe completed retries before timeout (#10916)

Author: ReubenBond

src/Orleans.Reminders.TestKit/ReminderTableRetryPolicy.cs

@@ -147,14 +147,23 @@ internal static async Task<T> ExecuteUntilAsync<T>(
             try
             {
                 var attempt = operation();
-                remaining = timeout - stopwatch.Elapsed;
-                if (remaining <= TimeSpan.Zero)
+                T value;
+                if (attempt.IsCompleted)
                 {
-                    lastException = new TimeoutException("The retry timeout elapsed while starting the operation.");
-                    ThrowTimeout();
+                    value = await attempt;
+                }
+                else
+                {
+                    remaining = timeout - stopwatch.Elapsed;
+                    if (remaining <= TimeSpan.Zero)
+                    {
+                        lastException = new TimeoutException("The retry timeout elapsed while starting the operation.");
+                        ThrowTimeout();
+                    }
+
+                    value = await attempt.WaitAsync(remaining, cancellationToken);
                 }
 
-                var value = await attempt.WaitAsync(remaining, cancellationToken);
                 lastException = null;
                 lastObservation = describe(value);
                 if (succeeded(value))

test/Orleans.Reminders.TestKit.Tests/ReminderTableRetryPolicyTests.cs

@@ -153,7 +153,34 @@ public async Task UniformPolicy_HardBoundsTheFirstAttempt()
     }
 
     [Fact]
-    public async Task UniformPolicy_EnforcesDeadlineAfterStartingFirstAttempt()
+    public async Task UniformPolicy_ObservesCompletedFirstAttemptAfterDeadline()
+    {
+        var invocations = 0;
+
+        var result = await ReminderTableRetryPolicy.ExecuteUntilAsync(
+            () =>
+            {
+                invocations++;
+                Thread.Sleep(TimeSpan.FromMilliseconds(20));
+                return Task.FromResult("late-success");
+            },
+            _ => true,
+            "DelayedStart",
+            "ReadGuarantee",
+            "ReadRow",
+            "a completed result",
+            value => value,
+            "read convergence",
+            TimeSpan.FromMilliseconds(1),
+            TimeSpan.FromMilliseconds(5),
+            TestContext.Current.CancellationToken);
+
+        Assert.Equal("late-success", result);
+        Assert.Equal(1, invocations);
+    }
+
+    [Fact]
+    public async Task UniformPolicy_ObservesCompletedFaultAfterDeadline()
     {
         var invocations = 0;
 
@@ -163,23 +190,23 @@ public async Task UniformPolicy_EnforcesDeadlineAfterStartingFirstAttempt()
                 {
                     invocations++;
                     Thread.Sleep(TimeSpan.FromMilliseconds(20));
-                    return Task.FromResult("late-success");
+                    return Task.FromException<string>(new InvalidOperationException("late-contention"));
                 },
                 _ => true,
                 "DelayedStart",
-                "ReadGuarantee",
-                "ReadRow",
-                "a result within the deadline",
+                "MutationGuarantee",
+                "UpsertRow",
+                "a completed mutation",
                 value => value,
-                "read convergence",
+                "mutation retry",
                 TimeSpan.FromMilliseconds(1),
                 TimeSpan.FromMilliseconds(5),
                 TestContext.Current.CancellationToken));
 
         Assert.Equal(1, invocations);
         Assert.Contains("attempts=1", exception.Message, StringComparison.Ordinal);
         Assert.Contains("Last observation: <no completed attempt>", exception.Message, StringComparison.Ordinal);
-        Assert.Contains("Last exception: System.TimeoutException", exception.Message, StringComparison.Ordinal);
+        Assert.Contains("Last exception: System.InvalidOperationException: late-contention", exception.Message, StringComparison.Ordinal);
     }
 
     private sealed class TestRunner(IReminderTable table, string providerName)