diff --git a/Orleans.slnx b/Orleans.slnx index d2395ae9857..df8ccda67ca 100644 --- a/Orleans.slnx +++ b/Orleans.slnx @@ -41,6 +41,7 @@ + @@ -148,6 +149,7 @@ + diff --git a/docs/site/src/content/docs/grains/durable-messaging.md b/docs/site/src/content/docs/grains/durable-messaging.md new file mode 100644 index 00000000000..20e94524e6e --- /dev/null +++ b/docs/site/src/content/docs/grains/durable-messaging.md @@ -0,0 +1,128 @@ +--- +title: Durable messaging +description: Understand the durable inbox and outbox guarantees, recovery model, and operating limits. +ms.date: 08/20/2026 +ms.topic: conceptual +--- + +# Durable messaging + +The `Microsoft.Orleans.DurableMessaging` package provides a grain-scoped inbox and +outbox built on Orleans Journaling and Durable Jobs. It is intended for application +messages whose state effects and outgoing messages must survive activation loss. + +## Message and routing model + +Each identifies its sender, target, +route, message ID, optional correlation, optional +`ReplyTo` grain ID, and an opaque serialized body. A receiver evaluates registered +handlers in registration order. Handlers can select envelopes by exact route, route +prefix, correlation hierarchy, or arbitrary metadata, and typed handlers deserialize +the body only when selected. The receiving grain verifies that the envelope target +matches its own identity before deduplication or persistence. + +The preview correlation key type now belongs to this package as +. Draft consumers of +`Orleans.HierarchicalKey` should update their namespace import. Its serialized alias +and member identifiers remain unchanged, so envelopes written by the earlier draft +remain readable. + +`ReplyTo` is general message metadata. Applications decide which route and body to use +for a follow-up message. + +## Commit and delivery guarantees + +Durable Messaging has the following boundaries: + +- Calling stages an envelope in the + grain journal. Before journal capture, Durable Messaging allocates a stable job ID and + durably schedules the outbox job. The envelope, job ownership, and other journaled + grain effects then become durable in one commit. The job polls safely while the + envelope is provisional, and dispatch starts only after that commit succeeds. +- Sending an equivalent envelope with the same `MessageId` more than once is idempotent, + whether the original is provisional or durable. Reusing that ID with different routing, + correlation, body, or request-context content throws without changing the outbox. +- A failed inbox-handler attempt restores the last durable journal version before + retry or dead-letter accounting commits. The failed attempt's staged effects and + outgoing envelopes are discarded at that boundary. +- Inbox handlers stage journaled effects and outgoing envelopes. Durable Messaging + commits those changes together with inbox completion after the handler returns; + handlers cannot create an earlier journal commit or delete boundary. +- Deleting the grain journal discards staged inbox and outbox work and clears the + corresponding volatile pump bookkeeping before a later write begins. +- A receiver allocates a stable ownership token and places it in a scheduled inbox job + before committing both the envelope and ownership, and returns `Accepted` only after + both are durable. +- Transport is **at-least-once**. A crash after receiver acceptance but before durable + outbox removal can send the same envelope again. +- The receiver deduplicates by `(SenderId, MessageId)`. Duplicate deliveries converge + on one set of handler effects while that deduplication record is retained. After + + expires, the same envelope can be accepted and processed again. The expired record and + replay acceptance are committed atomically. +- Delivery is **unordered**. Inbox and outbox storage are dictionaries, and retries can + reorder envelopes. Applications which require ordering must carry sequence numbers + and make their handlers converge on application-defined order. + +The inbox and outbox use independent Durable Jobs. A blocked inbox handler on one grain +doesn't stop another grain's outbox. Monotonic ownership generations fence job +callbacks. Scheduling uses an internal stable physical job ID for each grain, pump, and +ownership generation, so retrying an ambiguous response while the original schedule is +active returns that job instead of creating another one. Completed-generation +tombstones let delayed duplicates terminate. A job which wakes before its ownership +commit or activation recovery is visible polls the same attempt instead of completing. +After recovery, a scheduled generation with no committed owner and no work is a +confirmed orphan and completes, so Durable Jobs removes it. If recovered work has no +matching owner, recovery schedules and commits a new generation before the old +generation terminates. Callbacks for the recovered and replacement generations both poll +until replacement ownership commits, preserving the existing durable wake-up if +scheduling or persistence must retry. Ownership-clear write failures restore the +preceding generation, so the current job remains responsible. Pump callbacks execute as +non-interleaving grain timer turns so that infrastructure writes can't commit +provisional state from a concurrently running handler. + +## Backpressure, retries, and dead letters + +The inbox rejects new, nonduplicate envelopes with `Backpressured` when it reaches +. The +sender retains and retries the envelope. Handler failures restore the preceding durable +state before retry accounting is committed. Messages move to the appropriate inbox or +outbox dead-letter collection after their configured attempt or age limit. Use + to inspect those records. +After an operator or application has handled a record, remove it with + +or +so dead-letter storage remains bounded by the application's retention policy. +Removal is staged in the grain transaction and becomes durable with the grain's next +journal write. + +Malformed typed bodies are isolated during handler deserialization and follow the same +retry and dead-letter path; they don't prevent later envelopes from being recovered. +A successfully decoded null body is delivered as null. Typed handler parameters are +explicitly null-capable and handlers which require a non-null body must validate it. + +## Deployment requirements + +Configure Durable Jobs storage and Journaling storage before enabling Durable +Messaging. Grains which use Durable Messaging derive from +; its activation lifecycle initializes the +journaled state manager and materializes the inbox and outbox participants before +message recovery begins. Durable Messaging selects the built-in `orleans-binary` +journal format so opaque envelope bodies and request-context slices recover exactly. +Durable Messaging grains use non-reentrant execution: they don't apply `Reentrant`, +`MayInterleave`, `AlwaysInterleave`, or `StatelessWorker`. A single non-interleaving +activation owns each grain journal and pump, so infrastructure writes cannot commit +provisional application state or compete with another activation for the same ownership. +The Journaling implementation must provide + and accept + so Durable Messaging +receives commit and recovery notifications. Activation reports a +durable-messaging-specific diagnostic when observer registration is unsupported. Use +shared, production-grade storage for multi-silo deployments. In-memory Durable Jobs and +journal storage are suitable only for development and tests. + +Capacity and retention settings bound storage growth and define the effectively-once +window. Monitor inbox depth, outbox depth, retry failures, dead letters, and oldest +pending-message age. Keep deduplication retention longer than the maximum expected +outbox retry age. The `orleans-durable-messaging-orphaned-jobs-reclaimed` counter +identifies terminal cleanup of schedule-before-commit crash remnants. diff --git a/docs/site/src/content/docs/resources/nuget-packages.md b/docs/site/src/content/docs/resources/nuget-packages.md index 30dca3704f7..526be1cdbb7 100644 --- a/docs/site/src/content/docs/resources/nuget-packages.md +++ b/docs/site/src/content/docs/resources/nuget-packages.md @@ -77,8 +77,9 @@ Memory persistence distributes records across cluster storage grains but isn't d | [Microsoft.Orleans.Reminders.Redis](https://www.nuget.org/packages/Microsoft.Orleans.Reminders.Redis) | Redis reminders. | | [Microsoft.Orleans.DurableJobs](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs) | Distributed scheduling for durable one-time jobs. | | [Microsoft.Orleans.DurableJobs.AzureStorage](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs.AzureStorage) | Azure Blob Storage for durable jobs. | +| [Microsoft.Orleans.DurableMessaging](https://www.nuget.org/packages/Microsoft.Orleans.DurableMessaging) | Experimental grain-scoped durable inbox and outbox messaging. | -Use reminders for recurring durable callbacks and durable jobs for scheduled one-time work. Grain timers are activation-scoped and use the core runtime rather than a provider package. +Use reminders for recurring durable callbacks and durable jobs for scheduled one-time work. Durable Messaging composes Durable Jobs with Journaling for recoverable, at-least-once grain messages. Grain timers are activation-scoped and use the core runtime rather than a provider package. ## Streams and broadcast channels diff --git a/docs/site/src/content/docs/toc.yml b/docs/site/src/content/docs/toc.yml index ac0f03b8883..51fcc4f7228 100644 --- a/docs/site/src/content/docs/toc.yml +++ b/docs/site/src/content/docs/toc.yml @@ -112,6 +112,8 @@ items: href: grains/journaling/operations.md - name: Samples href: grains/journaling/samples.md + - name: Durable messaging + href: grains/durable-messaging.md - name: Transactions href: grains/transactions.md - name: Serialization and code generation diff --git a/docs/site/src/data/external-link-allowlist.json b/docs/site/src/data/external-link-allowlist.json index e691c21bb91..911ba0af130 100644 --- a/docs/site/src/data/external-link-allowlist.json +++ b/docs/site/src/data/external-link-allowlist.json @@ -9,6 +9,7 @@ "https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html": "AWS serves this public CLI configuration page to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://docs.aws.amazon.com/sdk-for-net/v4/developer-guide/creds-assign.html": "AWS serves this public SDK credential-resolution page to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://docs.aws.amazon.com/streams/latest/dev/introduction.html": "AWS serves this public Kinesis overview to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", + "https://www.nuget.org/packages/Microsoft.Orleans.DurableMessaging": "The new Durable Messaging package is documented but not yet published; remove this entry and its unpublished API-package entry after publication.", "https://en.wikipedia.org/wiki/Kalman_filter": "Wikipedia serves this public article to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://www.f5.com/company/blog/nginx/nginx-power-of-two-choices-load-balancing-algorithm": "The F5 site serves this canonical NGINX article to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403." } diff --git a/docs/site/src/data/unpublished-api-packages.json b/docs/site/src/data/unpublished-api-packages.json index 64d185c77e2..f08c10bac7c 100644 --- a/docs/site/src/data/unpublished-api-packages.json +++ b/docs/site/src/data/unpublished-api-packages.json @@ -1,4 +1,6 @@ { "description": "Generated API assemblies which are not currently published as standalone NuGet packages.", - "packages": {} + "packages": { + "Microsoft.Orleans.DurableMessaging": "The new package is awaiting its first NuGet publication." + } } diff --git a/src/Orleans.DurableJobs/IDurableJobReceiverExtension.cs b/src/Orleans.DurableJobs/IDurableJobReceiverExtension.cs index 3116374239c..29049864296 100644 --- a/src/Orleans.DurableJobs/IDurableJobReceiverExtension.cs +++ b/src/Orleans.DurableJobs/IDurableJobReceiverExtension.cs @@ -30,7 +30,7 @@ internal sealed partial class DurableJobReceiverExtension : IDurableJobReceiverE private readonly IGrainContext _grain; private readonly DurableJobReceiverExtensionShared _shared; private readonly IDurableJobHandlerLookup _featureHandlers; - private readonly Dictionary<(string JobId, long ExecutionGeneration, int DequeueCount), JobAttemptState> _jobAttempts = []; + private readonly Dictionary<(string ShardId, string JobId, long ExecutionGeneration, int DequeueCount), JobAttemptState> _jobAttempts = []; public DurableJobReceiverExtension( IGrainContext grain, @@ -145,7 +145,7 @@ private async Task ExecuteHandlerAsync( } private ValueTask GetJobStatusAsync( - (string JobId, long ExecutionGeneration, int DequeueCount) key, + (string ShardId, string JobId, long ExecutionGeneration, int DequeueCount) key, IJobRunContext context, JobAttemptState state, bool newJob, @@ -181,7 +181,7 @@ private ValueTask GetJobStatusAsync( return ValueTask.FromCanceled(new CancellationToken(canceled: true)); async ValueTask LongPollGetJobStatusAsync( - (string JobId, long ExecutionGeneration, int DequeueCount) key, + (string ShardId, string JobId, long ExecutionGeneration, int DequeueCount) key, IJobRunContext context, JobAttemptState state, CancellationToken attemptCancellationToken) @@ -218,7 +218,7 @@ async ValueTask LongPollGetJobStatusAsync( } private DurableJobRunResult GetSuccessfulResult( - (string JobId, long ExecutionGeneration, int DequeueCount) key, + (string ShardId, string JobId, long ExecutionGeneration, int DequeueCount) key, JobAttemptState state) { var result = state.Task.Result; @@ -239,7 +239,7 @@ private DurableJobRunResult GetSuccessfulResult( return result; } - private void RemoveJobAttempt((string JobId, long ExecutionGeneration, int DequeueCount) key, JobAttemptState state) + private void RemoveJobAttempt((string ShardId, string JobId, long ExecutionGeneration, int DequeueCount) key, JobAttemptState state) { if (_jobAttempts.TryGetValue(key, out var current) && ReferenceEquals(current, state)) { @@ -247,8 +247,8 @@ private void RemoveJobAttempt((string JobId, long ExecutionGeneration, int Deque } } - private static (string JobId, long ExecutionGeneration, int DequeueCount) GetExecutionKey(IJobRunContext context) - => (context.Job.Id, context.Job.ExecutionGeneration, context.DequeueCount); + private static (string ShardId, string JobId, long ExecutionGeneration, int DequeueCount) GetExecutionKey(IJobRunContext context) + => (context.Job.ShardId, context.Job.Id, context.Job.ExecutionGeneration, context.DequeueCount); internal sealed class TestAccessor(DurableJobReceiverExtension extension) { diff --git a/src/Orleans.DurableJobs/InMemoryJobQueue.cs b/src/Orleans.DurableJobs/InMemoryJobQueue.cs index 53d5572c966..5005917cfb7 100644 --- a/src/Orleans.DurableJobs/InMemoryJobQueue.cs +++ b/src/Orleans.DurableJobs/InMemoryJobQueue.cs @@ -101,6 +101,23 @@ public bool RemoveJob(string jobId) } } + public bool TryGetJob(string jobId, out DurableJob? job) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jobId); + lock (_syncLock) + { + if (_jobsIdToBucket.TryGetValue(jobId, out var bucket) + && bucket.TryGetJob(jobId, out var entry)) + { + job = entry.Job; + return true; + } + + job = null; + return false; + } + } + /// /// Returns whether the queue still contains the supplied durable job. /// diff --git a/src/Orleans.DurableJobs/JobShard.cs b/src/Orleans.DurableJobs/JobShard.cs index c3e2de8555d..d2909b9f151 100644 --- a/src/Orleans.DurableJobs/JobShard.cs +++ b/src/Orleans.DurableJobs/JobShard.cs @@ -231,18 +231,19 @@ public async Task TryStartAttemptAsync( throw new ArgumentOutOfRangeException(nameof(request), "Scheduled time is out of shard bounds."); } - var jobId = Guid.NewGuid().ToString(); - var job = new DurableJob + if (request.JobId is { } requestedJobId + && _jobQueue.TryGetJob(requestedJobId, out var existingJob)) { - Id = jobId, - TargetGrainId = request.Target, - Name = request.JobName, - DueTime = request.DueTime, - ShardId = Id, - Metadata = request.Metadata, - TraceParent = request.TraceParent, - TraceState = request.TraceState, - }; + if (!request.Matches(existingJob!)) + { + throw new InvalidOperationException( + $"Durable job ID '{requestedJobId}' is already scheduled with different properties."); + } + + return existingJob; + } + + var job = request.CreateJob(Id); await PersistAddJobAsync(job, cancellationToken); _jobQueue.Enqueue(job, 0); diff --git a/src/Orleans.DurableJobs/JournaledJobShardState.cs b/src/Orleans.DurableJobs/JournaledJobShardState.cs index 7f3c8d939c6..153da33a9fe 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardState.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardState.cs @@ -78,17 +78,19 @@ private JournaledJobShardState( throw new ArgumentOutOfRangeException(nameof(request), "Scheduled time is out of shard bounds."); } - var job = new DurableJob + if (request.JobId is { } requestedJobId + && _jobQueue.TryGetJob(requestedJobId, out var existingJob)) { - Id = Guid.NewGuid().ToString(), - TargetGrainId = request.Target, - Name = request.JobName, - DueTime = request.DueTime, - ShardId = Id, - Metadata = request.Metadata, - TraceParent = request.TraceParent, - TraceState = request.TraceState, - }; + if (!request.Matches(existingJob!)) + { + throw new InvalidOperationException( + $"Durable job ID '{requestedJobId}' is already scheduled with different properties."); + } + + return existingJob; + } + + var job = request.CreateJob(Id); Write(DurableJobShardJournalRecord.ForSchedule(job)); ApplySchedule(job); diff --git a/src/Orleans.DurableJobs/LocalDurableJobManager.cs b/src/Orleans.DurableJobs/LocalDurableJobManager.cs index 82584639c8d..215253e8159 100644 --- a/src/Orleans.DurableJobs/LocalDurableJobManager.cs +++ b/src/Orleans.DurableJobs/LocalDurableJobManager.cs @@ -175,6 +175,7 @@ private static ScheduleJobRequest EnsureScheduleRequestHasTraceContext(ScheduleJ return new ScheduleJobRequest { + JobId = request.JobId, Target = request.Target, JobName = request.JobName, DueTime = request.DueTime, @@ -634,7 +635,7 @@ public void AddWritableShard(DateTimeOffset shardKey, IJobShard shard, int strip } private WritableShardKey GetWritableShardKey(ScheduleJobRequest request) - => new(GetShardStartTime(request.DueTime), GetShardStripe()); + => new(GetShardStartTime(request.DueTime), GetShardStripe(request.JobId)); private IDictionary CreateShardMetadata(WritableShardKey shardKey) { @@ -655,13 +656,18 @@ private DateTimeOffset GetShardStartTime(DateTimeOffset scheduledTime) return new DateTimeOffset(bucketTicks, TimeSpan.Zero); } - private int GetShardStripe() + private int GetShardStripe(string? stableJobId) { if (_options.ShardStripeCount <= 1) { return 0; } + if (stableJobId is not null) + { + return (int)(StableHash.ComputeHash(stableJobId) % (uint)_options.ShardStripeCount); + } + // Round-robin assignment. Stripe selection is a write-side fan-out knob only: // the persisted job location is the shard id, not (StartTime, Stripe), so consistency // across calls/silos is not required and round-robin distributes evenly under any input skew. diff --git a/src/Orleans.DurableJobs/Orleans.DurableJobs.csproj b/src/Orleans.DurableJobs/Orleans.DurableJobs.csproj index e838ed729a7..f13cf88e736 100644 --- a/src/Orleans.DurableJobs/Orleans.DurableJobs.csproj +++ b/src/Orleans.DurableJobs/Orleans.DurableJobs.csproj @@ -30,7 +30,7 @@ + - diff --git a/src/Orleans.DurableJobs/ScheduleJobRequest.cs b/src/Orleans.DurableJobs/ScheduleJobRequest.cs index 7b40da79ac9..4c2fa8d19f8 100644 --- a/src/Orleans.DurableJobs/ScheduleJobRequest.cs +++ b/src/Orleans.DurableJobs/ScheduleJobRequest.cs @@ -9,6 +9,16 @@ namespace Orleans.DurableJobs; /// public readonly struct ScheduleJobRequest { + /// + /// Gets an optional stable job identifier. + /// + /// + /// Repeating a request with the same identifier, target, name, and metadata is idempotent within + /// its due-time shard. The first request determines the due time and trace context. + /// A conflicting request using the same identifier is rejected. + /// + internal string? JobId { get; init; } + /// /// Gets the grain identifier of the target grain that will receive the durable job. /// @@ -41,6 +51,57 @@ public readonly struct ScheduleJobRequest /// public string? TraceState { get; init; } - internal void Validate() => + internal void Validate() + { ArgumentException.ThrowIfNullOrWhiteSpace(JobName, nameof(JobName)); + if (JobId is not null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(JobId, nameof(JobId)); + } + } + + internal DurableJob CreateJob(string shardId) => + new() + { + Id = JobId ?? Guid.NewGuid().ToString(), + TargetGrainId = Target, + Name = JobName, + DueTime = DueTime, + ShardId = shardId, + Metadata = Metadata, + TraceParent = TraceParent, + TraceState = TraceState + }; + + internal bool Matches(DurableJob job) => + job.Id == JobId + && job.TargetGrainId == Target + && string.Equals(job.Name, JobName, StringComparison.Ordinal) + && MetadataEquals(job.Metadata, Metadata); + + private static bool MetadataEquals( + IReadOnlyDictionary? left, + IReadOnlyDictionary? right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left is null || right is null || left.Count != right.Count) + { + return false; + } + + foreach (var (key, value) in left) + { + if (!right.TryGetValue(key, out var candidate) + || !string.Equals(value, candidate, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } } diff --git a/src/Orleans.DurableMessaging/Configuration/DurableInboxOptions.cs b/src/Orleans.DurableMessaging/Configuration/DurableInboxOptions.cs new file mode 100644 index 00000000000..1fe16b7f2bf --- /dev/null +++ b/src/Orleans.DurableMessaging/Configuration/DurableInboxOptions.cs @@ -0,0 +1,192 @@ +using System; + +namespace Orleans.DurableMessaging.Configuration; + +/// +/// Configuration options for the durable inbox messaging system. +/// +/// +/// +/// These options control the behavior of the durable inbox, including capacity limits, +/// deduplication tracking, retry behavior, and pump batch sizes. +/// +/// +/// Transport is at-least-once. Deduplication provides effectively-once handler effects only +/// while the processed-message record is retained. Configuration values affect memory usage, +/// throughput, and recovery characteristics. +/// +/// +public class DurableInboxOptions +{ + internal const int MaximumBackoffExponent = 6; + internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; + + // The runtime multiplies this base delay by at most MaximumBackoffMultiplier. + // Keep the expanded delay within the timer implementation's uint-millisecond limit. + private static readonly TimeSpan MaxSupportedRetryDelay = + TimeSpan.FromTicks( + TimeSpan.FromMilliseconds(uint.MaxValue - 1).Ticks + / MaximumBackoffMultiplier); + + /// + /// Gets or sets the maximum number of pending messages in the inbox. + /// When this limit is reached, new message deliveries will return DeliveryResult.Backpressured(). + /// + /// + /// + /// A lower value (e.g., 100) provides stronger backpressure but may reduce throughput. + /// A higher value (e.g., 10,000) allows more buffering but increases memory usage and recovery time. + /// + /// + /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable + /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. + /// + /// + /// + /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. + /// + public int MaxCapacity { get; set; } = 1000; + + /// + /// Gets or sets the time window for tracking processed messages to prevent duplicates. + /// Messages that were processed within this window will be rejected with DeliveryResult.Duplicate(). + /// + /// + /// + /// A longer window (e.g., 30 days) provides stronger deduplication guarantees but increases + /// memory usage and storage I/O. A shorter window (e.g., 1 hour) reduces overhead but may + /// allow duplicate processing if retries are delayed. + /// + /// + /// Processed message tracking uses composite key (SenderId, MessageId) with timestamps. + /// Expired entries are removed atomically when a replay is accepted and are also eligible for + /// compaction during inbox pump maintenance. + /// + /// + /// Consider your retry policies when setting this value. For example, if senders retry for + /// up to 24 hours, set the window to at least 48 hours to ensure deduplication coverage. + /// + /// + /// + /// The deduplication window. Must be greater than zero. Defaults to 7 days. + /// + public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets the base delay between retry attempts when delivery encounters backpressure. + /// + /// + /// + /// When the target inbox is at capacity and returns DeliveryResult.Backpressured(), + /// the outbox delivery pump applies exponential backoff from this duration before retrying. + /// + /// + /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, + /// but may increase CPU usage during sustained backpressure. A longer delay (e.g., 5 seconds) + /// reduces retry overhead but increases latency for message delivery. + /// + /// + /// For high-throughput scenarios where quick recovery from backpressure is important, + /// consider values between 100-500ms. For less time-sensitive workloads, 1-5 seconds is appropriate. + /// + /// + /// + /// The base backpressure retry delay. Must be greater than zero. Defaults to 1 second. + /// + public TimeSpan BackpressureRetryDelay { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. + /// + public int MaxProcessingAttempts { get; set; } = 5; + + /// + /// Gets or sets the maximum number of attempts before an outbox message is dead-lettered. + /// + public int MaxDeliveryAttempts { get; set; } = 100; + + /// + /// Gets or sets the maximum age of an outbox message. + /// + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets the maximum number of inbox messages processed by one durable job attempt. + /// + public int InboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets the maximum number of outbox messages processed by one durable job attempt. + /// + public int OutboxBatchSize { get; set; } = 32; + + /// + /// Validates the configuration values and throws if any are invalid. + /// + /// + /// Thrown if is less than or equal to zero, + /// or if is less than or equal to , + /// or if a retry, retention, or batch option is outside its supported range. + /// + /// + /// This method is typically called by the dependency injection container during service registration + /// to ensure configuration values are valid before the system starts. + /// + public void Validate() + { + if (MaxCapacity <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxCapacity), MaxCapacity, "MaxCapacity must be greater than zero."); + } + + if (DeduplicationWindow <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); + } + + if (BackpressureRetryDelay <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(BackpressureRetryDelay), BackpressureRetryDelay, "BackpressureRetryDelay must be greater than TimeSpan.Zero."); + } + if (BackpressureRetryDelay > MaxSupportedRetryDelay) + { + throw new ArgumentOutOfRangeException( + nameof(BackpressureRetryDelay), + BackpressureRetryDelay, + $"BackpressureRetryDelay must be less than or equal to {MaxSupportedRetryDelay}."); + } + + if (MaxProcessingAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); + } + + if (MaxDeliveryAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxDeliveryAttempts), MaxDeliveryAttempts, "MaxDeliveryAttempts must be greater than zero."); + } + + if (MaxOutboxRetryAge <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); + } + + if (MaxOutboxRetryAge >= DeduplicationWindow) + { + throw new ArgumentOutOfRangeException( + nameof(MaxOutboxRetryAge), + MaxOutboxRetryAge, + "MaxOutboxRetryAge must be less than DeduplicationWindow."); + } + + if (InboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(InboxBatchSize), InboxBatchSize, "InboxBatchSize must be greater than zero."); + } + + if (OutboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(OutboxBatchSize), OutboxBatchSize, "OutboxBatchSize must be greater than zero."); + } + } +} diff --git a/src/Orleans.DurableMessaging/CorrelationHandler.cs b/src/Orleans.DurableMessaging/CorrelationHandler.cs new file mode 100644 index 00000000000..4157442fabb --- /dev/null +++ b/src/Orleans.DurableMessaging/CorrelationHandler.cs @@ -0,0 +1,165 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.DurableMessaging; + +/// +/// Base class for handlers that match messages based on correlation key hierarchy. +/// +/// +/// +/// simplifies implementing handlers that respond to messages +/// with a that matches or is a descendant of +/// a specific correlation key. This enables hierarchical workflow routing where a parent +/// workflow can handle messages from all child workflows. +/// +/// +/// The handler matches when the envelope's correlation key: +/// +/// Exactly matches the configured correlation key, or +/// Is a descendant (child, grandchild, etc.) of the configured correlation key +/// +/// +/// +/// For example, if the correlation key is "workflow/order-123", this handler will match messages with: +/// +/// "workflow/order-123" (exact match) +/// "workflow/order-123/payment" (child) +/// "workflow/order-123/payment/verify" (grandchild) +/// +/// +/// +/// For exact route key matching, use . For prefix-based routing, +/// use . +/// +/// +/// Handler Precedence: When registering multiple handlers, more specific handlers +/// (like ) should be registered before generic handlers +/// to ensure correct dispatch order. First-match-wins semantics apply. +/// +/// +/// +/// +/// public class OrderWorkflowHandler : CorrelationHandler +/// { +/// private readonly string _orderId; +/// +/// public OrderWorkflowHandler(string orderId) +/// : base(HierarchicalKey.Create($"workflow/order-{orderId}")) +/// { +/// _orderId = orderId; +/// } +/// +/// protected override async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken ct) +/// { +/// // This handler receives messages from: +/// // - The main order workflow ("workflow/order-123") +/// // - Child workflows like payment ("workflow/order-123/payment") +/// // - Grandchild workflows like verification ("workflow/order-123/payment/verify") +/// +/// // Deserialize the message +/// if (!context.Envelope.Data.TryGetBody<WorkflowEvent>(out var workflowEvent)) +/// { +/// throw new InvalidOperationException("Failed to deserialize WorkflowEvent"); +/// } +/// +/// // Process based on correlation hierarchy +/// if (CorrelationKey.Equals(context.Envelope.CorrelationKey)) +/// { +/// // Main workflow message +/// await HandleMainWorkflow(workflowEvent, ct); +/// } +/// else +/// { +/// // Child workflow message - use correlation key to identify which child +/// await HandleChildWorkflow(context.Envelope.CorrelationKey, workflowEvent, ct); +/// } +/// } +/// } +/// +/// // Registration +/// var orderId = "123"; +/// var handler = new OrderWorkflowHandler(orderId); +/// inbox.RegisterHandler(handler); +/// +/// +public abstract class CorrelationHandler : IInboxHandler +{ + private readonly HierarchicalKey _correlationKey; + + /// + /// Initializes a new instance of the class. + /// + /// The correlation key to match. The handler will match this key and all descendants. + /// Thrown when is null. + protected CorrelationHandler(HierarchicalKey correlationKey) + { + ArgumentNullException.ThrowIfNull(correlationKey); + _correlationKey = correlationKey; + } + + /// + /// Gets the correlation key that this handler matches. + /// + protected HierarchicalKey CorrelationKey => _correlationKey; + + /// + /// Determines whether this handler can handle a message based on correlation key hierarchy. + /// + /// The handler context containing the envelope. + /// + /// true if the envelope's correlation key matches or is a descendant of this handler's correlation key; + /// otherwise, false. + /// + /// + /// + /// This implementation checks if the envelope's correlation key equals the configured correlation key + /// or if the configured correlation key is an ancestor of the envelope's correlation key using + /// . + /// + /// + /// Returns false if the envelope has no correlation key (null). + /// + /// + /// Note: returns true for exact matches + /// (a key is considered an ancestor of itself), so this handler will match both the configured + /// correlation key and all its descendants. + /// + /// + public bool CanHandle(IInboxHandlerContext context) + { + return _correlationKey.IsAncestorOf(context.Envelope.CorrelationKey); + } + + /// + /// Handles a message that matches the configured correlation key or is a descendant. + /// + /// Handler context containing the envelope and methods for sending messages. + /// Cancellation token. + /// A representing the asynchronous operation. + /// + /// + /// This method is only called when returns true, meaning the + /// envelope's correlation key matches or is a descendant of the configured correlation key. + /// + /// + /// Derived classes can use to compare against + /// context.Envelope.CorrelationKey to determine if this is an exact match or a child workflow. + /// + /// + /// Derived classes should handle business logic errors gracefully (e.g., log and send error + /// response) rather than throwing exceptions. Unhandled exceptions will be logged and may + /// prevent the message from being marked as processed. + /// + /// + protected abstract ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); + + /// + /// Explicit interface implementation that delegates to the protected method. + /// + ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + return HandleAsync(context, cancellationToken); + } +} diff --git a/src/Orleans.DurableMessaging/DeliveryResult.cs b/src/Orleans.DurableMessaging/DeliveryResult.cs new file mode 100644 index 00000000000..a363d96df8f --- /dev/null +++ b/src/Orleans.DurableMessaging/DeliveryResult.cs @@ -0,0 +1,56 @@ +using Orleans.Serialization; + +namespace Orleans.DurableMessaging; + +/// +/// Result of attempting to deliver a message to an inbox. +/// Struct for future extensibility (can add fields without breaking changes). +/// +[GenerateSerializer, Alias("Orleans.DurableMessaging.DeliveryResult")] +public readonly struct DeliveryResult +{ + /// + /// The status of the delivery attempt. + /// + [Id(0)] + public DeliveryStatus Status { get; init; } + + /// + /// Optional diagnostic message (e.g., reason for rejection). + /// + [Id(2)] + public string? Message { get; init; } + + /// + /// Creates a result indicating the message was accepted and persisted to inbox. + /// + public static DeliveryResult Accepted() => new() { Status = DeliveryStatus.Accepted }; + + /// + /// Creates a result indicating the message was a duplicate. + /// + public static DeliveryResult Duplicate() => new() { Status = DeliveryStatus.Duplicate }; + + /// + /// Creates a result indicating the inbox is at capacity. + /// + public static DeliveryResult Backpressured() => new() { Status = DeliveryStatus.Backpressured }; + + /// + /// Creates a result indicating no handler was found for the route key. + /// + public static DeliveryResult RouteNotFound(string routeKey) => new() + { + Status = DeliveryStatus.RouteNotFound, + Message = $"No handler for route '{routeKey}'" + }; + + /// + /// Creates a result indicating the message was dead-lettered. + /// + public static DeliveryResult DeadLettered(string reason) => new() + { + Status = DeliveryStatus.DeadLettered, + Message = reason + }; +} diff --git a/src/Orleans.DurableMessaging/DeliveryStatus.cs b/src/Orleans.DurableMessaging/DeliveryStatus.cs new file mode 100644 index 00000000000..f45f4fc9957 --- /dev/null +++ b/src/Orleans.DurableMessaging/DeliveryStatus.cs @@ -0,0 +1,32 @@ +namespace Orleans.DurableMessaging; + +/// +/// Status codes for delivery attempts. +/// +public enum DeliveryStatus +{ + /// + /// Message was accepted and persisted to inbox. + /// + Accepted = 0, + + /// + /// Message was a duplicate (already processed or in inbox). + /// + Duplicate = 1, + + /// + /// Inbox is at capacity; sender should retry later. + /// + Backpressured = 2, + + /// + /// No handler registered for the specified RouteKey. + /// + RouteNotFound = 3, + + /// + /// The message was moved to the receiver's dead-letter store. + /// + DeadLettered = 6 +} diff --git a/src/Orleans.DurableMessaging/DurableEnvelope.cs b/src/Orleans.DurableMessaging/DurableEnvelope.cs new file mode 100644 index 00000000000..b01d9f314b9 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableEnvelope.cs @@ -0,0 +1,263 @@ +using System; +using Orleans; + +namespace Orleans.DurableMessaging; + +/// +/// Envelope for durable inbox/outbox messages. +/// Body and request-context values are stored as opaque byte slices for deferred deserialization. +/// +/// +/// +/// The DurableEnvelope provides a non-generic, polymorphic wrapper for durable messages between grains. +/// It uses deferred deserialization via to prevent serialization errors +/// from crashing grains during recovery. +/// +/// +/// Messages are uniquely identified by the composite key (SenderId, MessageId) for deduplication tracking. +/// The RouteKey field enables multiplexing multiple message types to different handlers within a single grain, +/// following Orleans' established patterns for streaming (subscriptionId) and transactions (resourceId). +/// +/// +/// CorrelationKey establishes hierarchical relationships between related messages +/// (for example, "transfer-123/debit" and "transfer-123/credit"). ReplyTo carries a +/// general destination for follow-up messages. +/// +/// +/// +/// +/// // Creating an envelope using DurableEnvelopeBuilder: +/// var envelope = context.CreateEnvelope() +/// .To(targetGrain, "payment/process") +/// .WithBody(new PaymentRequest { Amount = 100.00m }) +/// .WithCorrelationKey("order-12345/payment") +/// .WithReplyTo(context.GrainId) +/// .Build(); +/// +/// context.Send(envelope); +/// +/// +[GenerateSerializer, Alias("Orleans.DurableMessaging.DurableEnvelope")] +public readonly struct DurableEnvelope +{ + /// + /// Unique identifier for this message instance, used for deduplication. + /// + /// + /// Combined with , this forms the composite deduplication key + /// (SenderId, MessageId) that prevents duplicate message processing. The MessageId + /// is typically generated as a new GUID when the message is created. + /// + [Id(0)] + public required Guid MessageId { get; init; } + + /// + /// Identity of the sending grain. + /// + /// + /// Used in combination with for deduplication tracking. + /// When processing a message, the inbox checks if (SenderId, MessageId) has already + /// been processed. This provides effectively-once handler effects while the deduplication + /// record is retained; transport remains at-least-once. + /// + [Id(1)] + public required GrainId SenderId { get; init; } + + /// + /// Identity of the target grain. + /// + /// + /// Specifies the destination grain for this message. The inbox extension on the + /// receiver grain will validate that a handler is registered for the specified + /// before accepting the message. + /// + [Id(2)] + public required GrainId ReceiverId { get; init; } + + /// + /// Routing key for handler dispatch. Analogous to subscriptionId/resourceId in other extensions. + /// + /// + /// + /// The RouteKey enables multiplexing multiple message types to different handlers within + /// a single grain's inbox, following established Orleans patterns: + /// + /// + /// Streaming uses subscriptionId to route stream items to subscription handlers + /// Transactions use resourceId to route operations to transactional resources + /// Inbox/Outbox uses RouteKey to route messages to registered handlers + /// + /// + /// Handlers are registered using IDurableInbox.RegisterHandler(string routeKey, IInboxHandler handler). + /// If no handler is registered for the specified RouteKey, delivery returns DeliveryResult.RouteNotFound(). + /// + /// + /// + /// + /// // Register handlers for different message types + /// inbox.RegisterHandler("payment/process", new PaymentHandler()); + /// inbox.RegisterHandler("order/confirm", new OrderConfirmationHandler()); + /// inbox.RegisterHandler("refund/initiate", new RefundHandler()); + /// + /// // Messages are routed based on RouteKey + /// var envelope = builder.To(targetGrain, "payment/process").WithBody(request).Build(); + /// + /// + [Id(3)] + public required string RouteKey { get; init; } + + /// + /// Optional hierarchical correlation key for request/response pairing. + /// Supports parent/child relationships for correlated sub-requests. + /// + /// + /// + /// The CorrelationKey provides a hierarchical, human-readable identifier for tracking + /// related messages across distributed request/response flows. Unlike a GUID, hierarchical + /// keys support parent/child relationships and are easy to trace in logs and distributed traces. + /// + /// + /// The representation provides stable segment-boundary + /// comparisons for application-defined correlation hierarchies. + /// + /// + /// + /// + /// // Simple correlation + /// var envelope = builder + /// .WithCorrelationKey("order-12345") + /// .Build(); + /// + /// // Hierarchical correlation for orchestrated operations + /// var transferKey = HierarchicalKey.Create("transfer-abc"); + /// + /// // Child operations inherit correlation hierarchy + /// var debitEnvelope = builder + /// .WithCorrelationKey(transferKey.CreateChildKey("debit")) // "transfer-abc/debit" + /// .Build(); + /// + /// var creditEnvelope = builder + /// .WithCorrelationKey(transferKey.CreateChildKey("credit")) // "transfer-abc/credit" + /// .Build(); + /// + /// // Handlers can check relationships + /// if (envelope.CorrelationKey?.IsChildOf(transferKey) == true) + /// { + /// // This message is part of the transfer-abc operation + /// } + /// + /// + [Id(4)] + public HierarchicalKey? CorrelationKey { get; init; } + + /// + /// Optional destination for follow-up messages. + /// A reference can be created from this GrainId as needed. + /// + /// + /// + /// For durable request/response patterns, the ReplyTo field specifies the grain that should + /// receive the response. Unlike observer references (which have lifecycle and serialization issues), + /// storing the GrainId provides a stable, durable reference that can be used to create a grain + /// reference when needed. + /// + /// + /// The reply message should use the same to enable matching + /// requests with their responses. + /// + /// + /// + /// + /// // Sender creates request with ReplyTo + /// var request = builder + /// .To(targetGrain, "payment/process") + /// .WithBody(new PaymentRequest { Amount = 100.00m }) + /// .WithCorrelationKey("order-12345") + /// .WithReplyTo(context.GrainId) // Specify where to send response + /// .Build(); + /// + /// context.Send(request); + /// + /// // Handler sends reply + /// public async ValueTask HandleAsync(PaymentRequest request, IInboxHandlerContext context, CancellationToken ct) + /// { + /// var result = await ProcessPayment(request); + /// + /// if (context.Envelope.ReplyTo is { } replyTo) + /// { + /// var response = context.CreateEnvelope() + /// .To(replyTo, "payment/response") + /// .WithBody(result) + /// .WithCorrelationKey(context.Envelope.CorrelationKey) // Preserve correlation + /// .Build(); + /// + /// context.Send(response); + /// } + /// } + /// + /// + [Id(5)] + public GrainId? ReplyTo { get; init; } + + /// + /// Opaque data containing the serialized body and request context. + /// Uses deferred deserialization to prevent serialization errors from crashing grains. + /// + /// + /// + /// The Data field stores the message body and request context as opaque byte slices in a shared + /// managed buffer. This provides several + /// critical benefits: + /// + /// + /// Deferred deserialization: Body and context values are only deserialized when accessed + /// Error isolation: Deserialization failures don't crash grains; they return false from Try* methods + /// Recovery safety: Grains can recover even if message types are no longer available + /// Slice-based access: All values share one envelope buffer + /// Per-key context access: Individual context values can be retrieved independently + /// + /// + /// Access the body using Data.TryGetBody<T>() and context values using + /// Data.TryGetContextValue<T>(key). For forwarding messages without deserialization, + /// use Data.GetBodyBytes() or Data.TryGetContextBytes(key). + /// + /// + /// + /// + /// // Accessing the message body with type safety + /// if (envelope.Data.TryGetBody<PaymentRequest>(out var request)) + /// { + /// // Successfully deserialized as PaymentRequest + /// await ProcessPayment(request); + /// } + /// else + /// { + /// // Type mismatch, corruption, or missing type + /// // Grain doesn't crash - can log, skip, or dead-letter + /// _logger.LogWarning("Failed to deserialize message body"); + /// } + /// + /// // Accessing specific context values + /// if (envelope.Data.TryGetContextValue<string>("TraceId", out var traceId)) + /// { + /// // Use trace ID for distributed tracing + /// } + /// + /// // Forwarding without deserialization + /// var bodyBytes = envelope.Data.GetBodyBytes(); + /// ForwardToAnotherSystem(bodyBytes); + /// + /// + [Id(6)] + public required DurableEnvelopeData Data { get; init; } + + /// + /// Timestamp when the message was created. + /// + /// + /// Used for diagnostics, monitoring, and potential message expiration policies. + /// The timestamp is typically set to DateTimeOffset.UtcNow when the envelope is built. + /// + [Id(7)] + public DateTimeOffset CreatedAt { get; init; } +} diff --git a/src/Orleans.DurableMessaging/DurableEnvelopeBuilder.cs b/src/Orleans.DurableMessaging/DurableEnvelopeBuilder.cs new file mode 100644 index 00000000000..c7526d29a4e --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableEnvelopeBuilder.cs @@ -0,0 +1,336 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using Orleans; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; + +namespace Orleans.DurableMessaging; + +/// +/// Builder for creating durable envelopes with fluent configuration. +/// Use to set the message body, then to create the envelope. +/// Context values are serialized independently (MigrationContext pattern) for per-key access. +/// +/// +/// +/// The builder implements to serialize into one pooled staging buffer. +/// Building the envelope copies that data into a managed buffer with offset/length indices for each value. +/// +/// +/// Usage example: +/// +/// var envelope = context.CreateEnvelope() +/// .To(targetGrain, "transfer.debit") +/// .WithBody(new DebitRequest { Amount = 100m }) +/// .WithCorrelationKey("transfer-123/debit") +/// .WithReplyTo(context.GrainId) +/// .WithContextValue("trace-id", "abc-123") +/// .Build(); +/// context.Send(envelope); +/// +/// +/// +public sealed class DurableEnvelopeBuilder : IBufferWriter +{ + // Internal properties injected by IInboxHandlerContext implementation + internal SerializerSessionPool SessionPool { get; init; } = null!; + internal GrainId SenderId { get; init; } + + private GrainId _receiverId; + private string _routeKey = string.Empty; + private HierarchicalKey? _correlationKey; + private GrainId? _replyTo; + + // MigrationContext-style keyed context storage + private Dictionary? _contextIndices; + private ArrayBufferWriter _buffer = new(); + private (int Offset, int Length) _bodySlice; + private bool _bodyWritten; + private bool _built; + + internal DurableEnvelopeBuilder() + { + } + + /// + /// Initializes a builder for a message sent by the specified grain. + /// + /// The serializer session pool used to encode the message. + /// The identity of the sending grain. + public DurableEnvelopeBuilder(SerializerSessionPool sessionPool, GrainId senderId) + { + ArgumentNullException.ThrowIfNull(sessionPool); + SessionPool = sessionPool; + SenderId = senderId; + } + + /// + /// Sets the target grain and route key for this envelope. + /// + /// The target grain to receive the message. + /// The route key for handler dispatch (e.g., "transfer.debit"). + /// This builder for chaining. + /// Thrown if is null. + /// + /// Thrown if is the default grain id or is empty or whitespace. + /// + /// + /// + /// builder.To(targetGrain, "account.debit"); + /// + /// + public DurableEnvelopeBuilder To(GrainId target, string routeKey) + { + ThrowIfBuilt(); + ArgumentNullException.ThrowIfNull(routeKey); + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + if (target.IsDefault) + { + throw new ArgumentException("The target grain id must not be the default value.", nameof(target)); + } + + _receiverId = target; + _routeKey = routeKey; + return this; + } + + /// + /// Sets the message body. This serializes the body immediately into the shared buffer. + /// Can be called before or after - order doesn't matter. + /// + /// The type of the message body. + /// The message body to serialize. + /// This builder for chaining. + /// Thrown if the body has already been set. + /// + /// + /// builder.WithBody(new DebitRequest { Amount = 100m, AccountId = "acct-123" }); + /// + /// + public DurableEnvelopeBuilder WithBody(T body) + { + ThrowIfBuilt(); + if (_bodyWritten) + { + throw new InvalidOperationException("Body has already been set."); + } + + var startOffset = _buffer.WrittenCount; + using var session = SessionPool.GetSession(); + var writer = Writer.Create((IBufferWriter)this, session); + SessionPool.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), body); + writer.Commit(); + _bodySlice = (startOffset, _buffer.WrittenCount - startOffset); + _bodyWritten = true; + + return this; + } + + /// + /// Sets the hierarchical correlation key for request/response tracking. + /// + /// The correlation key (e.g., "transfer-123/debit"). + /// This builder for chaining. + /// + /// + /// // Parent request + /// builder.WithCorrelationKey(HierarchicalKey.Create("transfer-123")); + /// + /// // Child request + /// var parentKey = HierarchicalKey.Create("transfer-123"); + /// builder.WithCorrelationKey(parentKey.CreateChildKey("debit")); + /// + /// + public DurableEnvelopeBuilder WithCorrelationKey(HierarchicalKey correlationKey) + { + ThrowIfBuilt(); + _correlationKey = correlationKey; + return this; + } + + /// + /// Sets the hierarchical correlation key for request/response tracking (string convenience overload). + /// + /// The correlation key as a string (e.g., "transfer-123/debit"). + /// This builder for chaining. + /// Thrown if is null. + /// Thrown if contains invalid segments. + /// + /// + /// builder.WithCorrelationKey("transfer-123/debit"); + /// + /// + public DurableEnvelopeBuilder WithCorrelationKey(string correlationKey) + { + ThrowIfBuilt(); + ArgumentNullException.ThrowIfNull(correlationKey); + _correlationKey = HierarchicalKey.Create(correlationKey); + return this; + } + + /// + /// Sets the destination for follow-up messages. + /// + /// The grain to receive the reply. + /// This builder for chaining. + /// Thrown if is the default grain id. + /// + /// + /// // Request with reply-to + /// builder + /// .To(workerGrain, "process") + /// .WithReplyTo(context.GrainId) + /// .WithBody(request); + /// + /// // Reply in handler + /// if (context.Envelope.ReplyTo is { } replyTo) + /// { + /// var reply = context.CreateEnvelope() + /// .To(replyTo, "process.reply") + /// .WithCorrelationKey(context.Envelope.CorrelationKey) + /// .WithBody(response) + /// .Build(); + /// context.Send(reply); + /// } + /// + /// + public DurableEnvelopeBuilder WithReplyTo(GrainId replyTo) + { + ThrowIfBuilt(); + if (replyTo.IsDefault) + { + throw new ArgumentException("The reply-to grain id must not be the default value.", nameof(replyTo)); + } + + _replyTo = replyTo; + return this; + } + + /// + /// Adds a typed request context value. Each value is serialized independently + /// into the shared buffer (MigrationContext pattern), allowing per-key retrieval. + /// Can be called before or after - order doesn't matter. + /// + /// The type of the context value. + /// The context key (e.g., "trace-id", "tenant-id"). + /// The context value to serialize. + /// This builder for chaining. + /// Thrown if is null. + /// Thrown if is empty or whitespace. + /// Thrown if the key has already been set. + /// + /// + /// builder + /// .WithContextValue("trace-id", "abc-123") + /// .WithContextValue("tenant-id", "tenant-456") + /// .WithContextValue("user-id", userId); + /// + /// + public DurableEnvelopeBuilder WithContextValue(string key, T value) + { + ThrowIfBuilt(); + ArgumentNullException.ThrowIfNull(key); + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + _contextIndices ??= new(StringComparer.Ordinal); + + if (_contextIndices.ContainsKey(key)) + { + throw new InvalidOperationException($"Context key '{key}' has already been set."); + } + + var startOffset = _buffer.WrittenCount; + using var session = SessionPool.GetSession(); + var writer = Writer.Create((IBufferWriter)this, session); + SessionPool.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), value); + writer.Commit(); + _contextIndices[key] = (startOffset, _buffer.WrittenCount - startOffset); + + return this; + } + + /// + /// Builds the durable envelope from the configured values. + /// + /// A new with the configured values. + /// + /// Thrown if the body has not been set via , + /// or if the target and route key have not been set via . + /// + /// + /// + /// var envelope = context.CreateEnvelope() + /// .To(targetGrain, "transfer.debit") + /// .WithBody(new DebitRequest { Amount = 100m }) + /// .Build(); + /// + /// + public DurableEnvelope Build() + { + if (_built) + { + throw new InvalidOperationException("This builder has already produced an envelope."); + } + + if (!_bodyWritten) + { + throw new InvalidOperationException("Message body must be set via WithBody()."); + } + + if (string.IsNullOrEmpty(_routeKey)) + { + throw new InvalidOperationException("Target and route key must be set via To()."); + } + + var buffer = _buffer.WrittenSpan.ToArray(); + _buffer = new ArrayBufferWriter(); + var data = new DurableEnvelopeData(SessionPool); + data.Initialize(buffer, _bodySlice, _contextIndices); + _built = true; + + return new DurableEnvelope + { + MessageId = Guid.NewGuid(), + SenderId = SenderId, + ReceiverId = _receiverId, + RouteKey = _routeKey, + CorrelationKey = _correlationKey, + ReplyTo = _replyTo, + Data = data, + CreatedAt = DateTimeOffset.UtcNow + }; + } + + /// + /// Resets the builder for reuse. This is typically called by pooling infrastructure. + /// + internal void Reset() + { + _receiverId = default; + _routeKey = string.Empty; + _correlationKey = null; + _replyTo = null; + _contextIndices = null; + _buffer = new ArrayBufferWriter(); + _bodySlice = default; + _bodyWritten = false; + _built = false; + } + + private void ThrowIfBuilt() + { + if (_built) + { + throw new InvalidOperationException("This builder has already produced an envelope."); + } + } + + // IBufferWriter implementation for serialization + void IBufferWriter.Advance(int count) => _buffer.Advance(count); + Memory IBufferWriter.GetMemory(int sizeHint) => _buffer.GetMemory(sizeHint); + Span IBufferWriter.GetSpan(int sizeHint) => _buffer.GetSpan(sizeHint); +} diff --git a/src/Orleans.DurableMessaging/DurableEnvelopeData.cs b/src/Orleans.DurableMessaging/DurableEnvelopeData.cs new file mode 100644 index 00000000000..d13d212ef3c --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableEnvelopeData.cs @@ -0,0 +1,171 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; + +namespace Orleans.DurableMessaging; + +/// +/// Opaque data storage for envelope body and request context. +/// Modeled after MigrationContext's deferred serialization pattern with keyed indices. +/// Body and all request-context values share the same underlying managed buffer. +/// +/// +/// This design enables: +/// +/// Deferred deserialization: Body and context values are only deserialized when accessed +/// Slice-based access: Body and all context values share one envelope buffer +/// Error isolation: Deserialization failures don't crash the grain; they can be handled gracefully +/// Per-key context access: Individual context values can be retrieved independently +/// +/// +[GenerateSerializer, Alias("Orleans.DurableMessaging.DurableEnvelopeData")] +public sealed class DurableEnvelopeData +{ + [NonSerialized] + private readonly SerializerSessionPool? _sessionPool; + + /// + /// Shared buffer containing body and all request context values. + /// + [Id(0)] + private byte[] _buffer = []; + + /// + /// Offset and length of the body within the buffer. + /// + [Id(1)] + private (int Offset, int Length) _bodySlice; + + /// + /// Keyed indices for request context values within the buffer. + /// Each key maps to its own (Offset, Length) slice, allowing independent deserialization. + /// + [Id(2), Immutable] + private Dictionary? _contextIndices; + + /// + /// Initializes a new instance of the class. + /// + /// The serializer session pool for serialization/deserialization. + [GeneratedActivatorConstructor] + internal DurableEnvelopeData(SerializerSessionPool sessionPool) + { + _sessionPool = sessionPool; + } + + /// + /// Gets the keys of all stored request context values. + /// + public IEnumerable ContextKeys => _contextIndices?.Keys ?? Enumerable.Empty(); + + /// + /// Returns true if a request context value exists for the specified key. + /// + /// The context key to check. + /// True if the key exists in the context; otherwise, false. + public bool HasContextKey(string key) => _contextIndices?.ContainsKey(key) ?? false; + + /// + /// Attempts to deserialize the body as the specified type. + /// Returns false if deserialization fails (type mismatch, corruption, etc.). + /// + /// The type to deserialize the body as. + /// The deserialized value, which can be , or default if deserialization fails. + /// True if deserialization succeeded; otherwise, false. + public bool TryGetBody([MaybeNull] out T value) + { + if (_sessionPool is null || _bodySlice.Length == 0) + { + value = default; + return false; + } + + try + { + var slice = new ReadOnlySequence(_buffer, _bodySlice.Offset, _bodySlice.Length); + using var session = _sessionPool.GetSession(); + var reader = Reader.Create(slice, session); + var field = reader.ReadFieldHeader(); + value = _sessionPool.CodecProvider.GetCodec().ReadValue(ref reader, field); + return true; + } + catch + { + value = default; + return false; + } + } + + /// + /// Attempts to deserialize a specific request context value. + /// Returns false if the key doesn't exist or deserialization fails. + /// + /// The type to deserialize the context value as. + /// The context key to retrieve. + /// The deserialized value, which can be , or default if not found or deserialization fails. + /// True if the key exists and deserialization succeeded; otherwise, false. + public bool TryGetContextValue(string key, [MaybeNull] out T value) + { + if (_sessionPool is null || _contextIndices is null || !_contextIndices.TryGetValue(key, out var slice)) + { + value = default; + return false; + } + + try + { + var buffer = new ReadOnlySequence(_buffer, slice.Offset, slice.Length); + using var session = _sessionPool.GetSession(); + var reader = Reader.Create(buffer, session); + var field = reader.ReadFieldHeader(); + value = _sessionPool.CodecProvider.GetCodec().ReadValue(ref reader, field); + return true; + } + catch + { + value = default; + return false; + } + } + + /// + /// Gets the raw body bytes for forwarding without deserialization. + /// + /// A read-only sequence containing the raw body bytes. + public ReadOnlySequence GetBodyBytes() + => new(_buffer, _bodySlice.Offset, _bodySlice.Length); + + /// + /// Gets the raw bytes for a specific context key for forwarding without deserialization. + /// + /// The context key to retrieve. + /// The raw bytes for the context value, or default if not found. + /// True if the key exists; otherwise, false. + public bool TryGetContextBytes(string key, out ReadOnlySequence value) + { + if (_contextIndices is not null && _contextIndices.TryGetValue(key, out var slice)) + { + value = new ReadOnlySequence(_buffer, slice.Offset, slice.Length); + return true; + } + + value = default; + return false; + } + + internal void Initialize( + byte[] buffer, + (int Offset, int Length) bodySlice, + Dictionary? contextIndices) + { + _buffer = buffer; + _bodySlice = bodySlice; + _contextIndices = contextIndices; + } + +} diff --git a/src/Orleans.DurableMessaging/DurableInbox.cs b/src/Orleans.DurableMessaging/DurableInbox.cs new file mode 100644 index 00000000000..7085d37963b --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableInbox.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +/// +/// Durable inbox implementation for receiving and processing messages. +/// Uses IDurableDictionary for persistent storage with deduplication support. +/// +internal sealed class DurableInbox : IDurableInbox +{ + private readonly IDurableDictionary<(GrainId SenderId, Guid MessageId), DurableEnvelope> _inbox; + private readonly List _handlers; + private readonly Dictionary _exactRouteHandlers; + private readonly int _capacity; + + /// + /// Creates a new DurableInbox instance. + /// + /// Durable dictionary for storing unprocessed messages. + /// Maximum inbox capacity (default: 1000). + public DurableInbox( + IDurableDictionary<(GrainId SenderId, Guid MessageId), DurableEnvelope> inbox, + int capacity = 1000) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity); + + _inbox = inbox; + _handlers = new List(); + _exactRouteHandlers = new Dictionary(StringComparer.Ordinal); + _capacity = capacity; + } + + internal DurableInbox( + IDurableDictionary<(GrainId SenderId, Guid MessageId), DurableEnvelope> inbox, + IEnumerable handlers, + int capacity) + : this(inbox, capacity) + { + foreach (var handler in handlers) + { + RegisterHandler(handler); + } + } + + /// + /// Number of unprocessed messages. + /// + public int Count => _inbox.Count; + + /// + /// Maximum capacity. When reached, DeliverAsync returns Backpressured. + /// + public int Capacity => _capacity; + + /// + /// Gets all pending messages (no ordering guarantee). + /// + public IEnumerable Messages => _inbox.Values; + + /// + /// Tries to get a specific message by its key. + /// + /// The sender grain ID. + /// The message ID. + /// The envelope if found. + /// True if the message exists in the inbox; otherwise, false. + public bool TryGetMessage(GrainId senderId, Guid messageId, [MaybeNullWhen(false)] out DurableEnvelope envelope) + { + var key = (senderId, messageId); + return _inbox.TryGetValue(key, out envelope); + } + + /// + /// Registers a handler that will be evaluated using its CanHandle method. + /// Handlers are evaluated in registration order (first-match-wins). + /// + /// The handler implementation. + public void RegisterHandler(IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(handler); + + _handlers.Add(handler); + + } + + /// + /// Tries to find a handler for the given context by calling CanHandle on registered handlers. + /// Returns the first handler that returns true from CanHandle. + /// + /// The inbox handler context containing envelope metadata. + /// The handler if found; otherwise, null. + /// True if a handler was found; otherwise, false. + internal bool TryFindHandler(IInboxHandlerContext context, [MaybeNullWhen(false)] out IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(context); + + foreach (var candidate in _handlers) + { + if (candidate.CanHandle(context)) + { + handler = candidate; + return true; + } + } + + handler = null; + return false; + } + + /// + /// Registers a handler for a specific route. + /// + /// The route key to handle. + /// The handler implementation. + public void RegisterHandler(string routeKey, IInboxHandler handler) + { + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + ArgumentNullException.ThrowIfNull(handler); + + if (!_exactRouteHandlers.TryAdd(routeKey, handler)) + { + throw new InvalidOperationException($"A handler is already registered for exact route '{routeKey}'."); + } + + var wrappedHandler = new ExactRouteKeyHandlerWrapper(routeKey, handler); + _handlers.Add(wrappedHandler); + + } + + /// + /// Checks if a route has a registered handler. + /// + /// The route key to check. + /// True if a handler is registered for this route; otherwise, false. + /// Thrown if is null, empty, or whitespace. + public bool HasHandler(string routeKey) + { + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + return _exactRouteHandlers.ContainsKey(routeKey); + } + + /// + /// Tries to get a handler for a specific route. + /// + /// The route key to get the handler for. + /// The handler if found. + /// True if a handler is registered for this route; otherwise, false. + /// Thrown if is null, empty, or whitespace. + public bool TryGetHandler(string routeKey, [MaybeNullWhen(false)] out IInboxHandler handler) + { + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + return _exactRouteHandlers.TryGetValue(routeKey, out handler); + } + +} + +/// +/// Internal wrapper that adapts exact route registration to the capability-based handler contract. +/// +internal sealed class ExactRouteKeyHandlerWrapper : IInboxHandler +{ + private readonly string _routeKey; + private readonly IInboxHandler _innerHandler; + + public ExactRouteKeyHandlerWrapper(string routeKey, IInboxHandler innerHandler) + { + _routeKey = routeKey; + _innerHandler = innerHandler; + } + + public bool CanHandle(IInboxHandlerContext context) + { + return context.Envelope.RouteKey == _routeKey; + } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + return _innerHandler.HandleAsync(context, cancellationToken); + } +} diff --git a/src/Orleans.DurableMessaging/DurableInboxExtension.cs b/src/Orleans.DurableMessaging/DurableInboxExtension.cs new file mode 100644 index 00000000000..5426bb0608f --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableInboxExtension.cs @@ -0,0 +1,1228 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.DurableMessaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Orleans.Serialization.TypeSystem; +using Orleans.Timers; + +namespace Orleans.DurableMessaging; + +/// +/// Implementation of durable inbox extension for grain message delivery. +/// Handles message persistence, deduplication, and processing. +/// +internal sealed partial class DurableInboxExtension : + IDurableInboxExtension, + IDurableJobFeatureHandler, + IJournaledStateObserver, + ILifecycleObserver, + IDisposable +{ + internal const string JobName = "orleans.messaging.inbox-drain"; + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + private readonly IGrainContext _grainContext; + private readonly IGrainFactory _grainFactory; + private readonly ITimerRegistry _timerRegistry; + private readonly IJournaledStateManager _stateManager; + private readonly SerializerSessionPool _sessionPool; + private readonly ILogger _logger; + private readonly DurableMessagingInstruments _instruments; + private readonly DurableInbox _durableInbox; + private readonly IDictionary<(GrainId SenderId, Guid MessageId), DurableEnvelope> _inboxDict; + private readonly IDictionary<(GrainId SenderId, Guid MessageId), DateTimeOffset> _processed; + private readonly IDictionary<(GrainId SenderId, Guid MessageId), InboxMessageState> _messageStates; + private readonly IDictionary<(GrainId SenderId, Guid MessageId), InboxDeadLetter> _deadLetters; + private readonly IDurableValue _jobId; + private readonly IDurableValue _completedJobId; + private readonly IDurableValue _jobSequence; + private readonly IDurableOutbox _outbox; + private readonly ILocalDurableJobManager _jobManager; + private readonly TimeProvider _timeProvider; + private readonly TimeProvider _jobTimeProvider; + private readonly HashSet<(GrainId SenderId, Guid MessageId)> _provisionalAcceptances = []; + private readonly HashSet _localDrainJobIds = new(StringComparer.Ordinal); + private readonly DurableMessagingPumpResults _pumpResults; + private readonly int _maxCapacity; + private readonly TimeSpan _deduplicationWindow; + private readonly int _maxProcessingAttempts; + private readonly int _batchSize; + private readonly TimeSpan _retryDelay; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly CancellationTokenSource _shutdownCts = new(); + private int _metricsActive; + private int _reportedDepth; + private int _handlerExecutionDepth; + private DateTimeOffset? _pendingJobDueTime; + private bool _provisionalScheduleConfirmed; + private string _ownershipEpoch = Guid.NewGuid().ToString("N"); + private long _stateGeneration; + private bool _recoveryCompleted; + + /// + /// Creates a new inbox extension instance. + /// + /// The grain context for this extension. + /// State manager for atomic persistence. + /// Serializer session pool for envelope creation. + /// Logger for diagnostics. + /// Journaling metrics. + /// The grain's durable inbox (shared with grain DI). + /// Durable dictionary for inbox messages. + /// Durable dictionary for processed message tracking. + /// Durable outbox for sending response messages. + /// Durable messaging options. + public DurableInboxExtension( + IGrainContext grainContext, + IGrainFactory grainFactory, + ITimerRegistry timerRegistry, + IJournaledStateManager stateManager, + SerializerSessionPool sessionPool, + ILogger logger, + DurableMessagingInstruments instruments, + DurableInbox durableInbox, + IDictionary<(GrainId SenderId, Guid MessageId), DurableEnvelope> inboxDict, + IDictionary<(GrainId SenderId, Guid MessageId), DateTimeOffset> processed, + IDictionary<(GrainId SenderId, Guid MessageId), InboxMessageState> messageStates, + IDictionary<(GrainId SenderId, Guid MessageId), InboxDeadLetter> deadLetters, + IDurableValue jobId, + IDurableValue completedJobId, + IDurableValue jobSequence, + IDurableOutbox outbox, + ILocalDurableJobManager jobManager, + IDurableJobHandlerRegistry jobHandlers, + DurableMessagingPumpResults pumpResults, + TimeProvider timeProvider, + TimeProvider jobTimeProvider, + DurableInboxOptions options) + { + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(grainFactory); + ArgumentNullException.ThrowIfNull(timerRegistry); + ArgumentNullException.ThrowIfNull(stateManager); + ArgumentNullException.ThrowIfNull(sessionPool); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(instruments); + ArgumentNullException.ThrowIfNull(durableInbox); + ArgumentNullException.ThrowIfNull(inboxDict); + ArgumentNullException.ThrowIfNull(processed); + ArgumentNullException.ThrowIfNull(messageStates); + ArgumentNullException.ThrowIfNull(deadLetters); + ArgumentNullException.ThrowIfNull(jobId); + ArgumentNullException.ThrowIfNull(completedJobId); + ArgumentNullException.ThrowIfNull(jobSequence); + ArgumentNullException.ThrowIfNull(outbox); + ArgumentNullException.ThrowIfNull(jobManager); + ArgumentNullException.ThrowIfNull(jobHandlers); + ArgumentNullException.ThrowIfNull(pumpResults); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(jobTimeProvider); + ArgumentNullException.ThrowIfNull(options); + _grainContext = grainContext; + _grainFactory = grainFactory; + _timerRegistry = timerRegistry; + _stateManager = stateManager; + _sessionPool = sessionPool; + _logger = logger; + _instruments = instruments; + _durableInbox = durableInbox; + _inboxDict = inboxDict; + _processed = processed; + _messageStates = messageStates; + _deadLetters = deadLetters; + _jobId = jobId; + _completedJobId = completedJobId; + _jobSequence = jobSequence; + _outbox = outbox; + _jobManager = jobManager; + _pumpResults = pumpResults; + _timeProvider = timeProvider; + _jobTimeProvider = jobTimeProvider; + _maxCapacity = options.MaxCapacity; + _deduplicationWindow = options.DeduplicationWindow; + _maxProcessingAttempts = options.MaxProcessingAttempts; + _batchSize = options.InboxBatchSize; + _retryDelay = options.BackpressureRetryDelay; + DurableMessagingStateManagerCapabilities.RegisterObserver(stateManager, this); + jobHandlers.Register(this); + grainContext.ObservableLifecycle.Subscribe( + RuntimeTypeNameFormatter.Format(GetType()), + GrainLifecycleStage.Activate, + this); + } + + /// + /// Gets the number of messages currently in the inbox. + /// + public int Count => _inboxDict.Count; + + /// + /// Gets the inbox capacity limit. + /// + public int Capacity => _maxCapacity; + + /// + /// Registers a handler for a specific route key. + /// Delegates to the shared durable inbox that is injected into grains. + /// + /// The route key to handle. + /// The handler implementation. + public void RegisterHandler(string routeKey, IInboxHandler handler) + { + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + ArgumentNullException.ThrowIfNull(handler); + + _durableInbox.RegisterHandler(routeKey, handler); + LogHandlerRegistered(_logger, routeKey, _grainContext.GrainId); + } + + /// + /// Checks if a handler is registered for the specified route key. + /// + /// The route key to check. + /// True if a handler is registered; otherwise, false. + public bool HasHandler(string routeKey) => _durableInbox.HasHandler(routeKey); + + /// + /// Tries to get a handler for the specified route key. + /// + /// The route key to get the handler for. + /// The handler if found. + /// True if a handler is registered; otherwise, false. + public bool TryGetHandler(string routeKey, [MaybeNullWhen(false)] out IInboxHandler handler) => _durableInbox.TryGetHandler(routeKey, out handler); + + /// + /// Delivers a message to this grain's durable inbox. + /// + /// The message envelope. + /// Cancellation token. + /// Result indicating delivery/processing status. + public async ValueTask DeliverAsync( + DurableEnvelope envelope, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (envelope.ReceiverId != _grainContext.GrainId) + { + throw new ArgumentException( + $"The envelope receiver '{envelope.ReceiverId}' does not match this grain '{_grainContext.GrainId}'.", + nameof(envelope)); + } + + EnsureMetricsActive(); + var key = (envelope.SenderId, envelope.MessageId); + var result = DeliveryResult.Accepted(); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + var replaceExpiredDedupeRecord = false; + if (_processed.TryGetValue(key, out var processedAt)) + { + replaceExpiredDedupeRecord = DurableMessagingTime.IsExpired( + _timeProvider.GetUtcNow(), + processedAt, + _deduplicationWindow); + if (!replaceExpiredDedupeRecord) + { + LogDuplicateMessageDetected( + _logger, + envelope.MessageId, + envelope.SenderId, + envelope.ReceiverId, + envelope.RouteKey, + envelope.CorrelationKey?.ToString()); + _instruments.OnInboxMessageReceived(_grainContext.GrainId.Type.ToString(), envelope.RouteKey, "duplicate"); + return DeliveryResult.Duplicate(); + } + } + + if (_inboxDict.ContainsKey(key)) + { + LogDuplicateMessageInInbox( + _logger, + envelope.MessageId, + envelope.SenderId, + envelope.ReceiverId, + envelope.RouteKey, + envelope.CorrelationKey?.ToString()); + _instruments.OnInboxMessageReceived(_grainContext.GrainId.Type.ToString(), envelope.RouteKey, "duplicate"); + await EnsureJobScheduledUnderGateAsync(CancellationToken.None).ConfigureAwait(true); + ScheduleLocalDrain(); + result = DeliveryResult.Duplicate(); + } + else + { + var wasDurablyEmpty = GetDurableInboxCount() == 0; + if (_inboxDict.Count >= _maxCapacity) + { + LogBackpressureRejection( + _logger, + _inboxDict.Count, + _maxCapacity, + _grainContext.GrainId, + envelope.MessageId, + envelope.SenderId, + envelope.RouteKey, + envelope.CorrelationKey?.ToString()); + _instruments.OnInboxMessageReceived(_grainContext.GrainId.Type.ToString(), envelope.RouteKey, "backpressured"); + return DeliveryResult.Backpressured(); + } + + var selectionContext = new InboxHandlerSelectionContext(envelope, _grainContext.GrainId); + if (!_durableInbox.TryFindHandler(selectionContext, out _)) + { + LogRouteNotFound( + _logger, + envelope.RouteKey, + _grainContext.GrainId, + envelope.MessageId, + envelope.SenderId, + envelope.CorrelationKey?.ToString()); + _instruments.OnInboxMessageReceived(_grainContext.GrainId.Type.ToString(), envelope.RouteKey, "route_not_found"); + return DeliveryResult.RouteNotFound(envelope.RouteKey); + } + + if (replaceExpiredDedupeRecord) + { + _processed.Remove(key); + } + + _inboxDict[key] = envelope; + _messageStates[key] = new InboxMessageState(); + _provisionalAcceptances.Add(key); + _provisionalScheduleConfirmed = false; + var stateGeneration = Volatile.Read(ref _stateGeneration); + UpdateInboxDepth(1); + var committed = false; + try + { + await EnsureJobScheduledUnderGateAsync( + CancellationToken.None, + persistState: false, + replaceExisting: wasDurablyEmpty, + includeProvisional: true).ConfigureAwait(true); + _provisionalScheduleConfirmed = true; + ValidateAcceptanceState(key, stateGeneration); + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + ValidateAcceptanceState(key, stateGeneration); + committed = true; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + finally + { + _provisionalAcceptances.Remove(key); + if (_provisionalAcceptances.Count == 0) + { + _provisionalScheduleConfirmed = false; + } + } + + if (committed && string.IsNullOrEmpty(_jobId.Value)) + { + await EnsureJobScheduledUnderGateAsync(CancellationToken.None).ConfigureAwait(true); + } + + ScheduleLocalDrain(); + + LogMessageAccepted( + _logger, + envelope.MessageId, + envelope.SenderId, + envelope.ReceiverId, + envelope.RouteKey, + envelope.CorrelationKey?.ToString()); + _instruments.OnInboxMessageReceived(_grainContext.GrainId.Type.ToString(), envelope.RouteKey, "accepted"); + } + } + finally + { + _gate.Release(); + } + + return result; + } + + private async ValueTask EnsureJobScheduledUnderGateAsync( + CancellationToken cancellationToken, + bool persistState = true, + bool replaceExisting = false, + bool includeProvisional = false) + { + var messageCount = includeProvisional ? _inboxDict.Count : GetDurableInboxCount(); + if (messageCount == 0 || (!replaceExisting && !string.IsNullOrEmpty(_jobId.Value))) + { + return; + } + + var previousJobId = _jobId.Value; + var jobId = replaceExisting || string.IsNullOrEmpty(previousJobId) + ? DurableMessagingJobOwnership.NextId(_ownershipEpoch, _jobSequence) + : previousJobId; + var dueTime = _pendingJobDueTime ??= _jobTimeProvider.GetUtcNow(); + _jobId.Value = jobId; + try + { + await _jobManager.ScheduleJobAsync( + new ScheduleJobRequest + { + JobId = DurableMessagingJobOwnership.CreateJobId(JobName, _grainContext.GrainId, jobId), + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = dueTime, + Metadata = DurableMessagingJobOwnership.CreateMetadata(jobId) + }, + cancellationToken).ConfigureAwait(true); + } + catch + { + _jobId.Value = previousJobId; + throw; + } + + if (persistState) + { + await _stateManager.WriteStateAsync(cancellationToken).ConfigureAwait(true); + } + } + + public void OnWriteStarted() + { + } + + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + return ValidatePersistenceBoundary(cancellationToken); + } + + public void OnWriteRequested() => ValidatePersistenceRequest(); + + public void OnDeleteRequested() => ValidatePersistenceRequest(); + + public ValueTask OnDeletePreparingAsync(CancellationToken cancellationToken) + { + return ValidatePersistenceBoundary(cancellationToken); + } + + public void OnDeleteCompleted() + { + Interlocked.Increment(ref _stateGeneration); + _ownershipEpoch = Guid.NewGuid().ToString("N"); + _provisionalAcceptances.Clear(); + _provisionalScheduleConfirmed = false; + _pendingJobDueTime = null; + ReconcileInboxDepth(); + } + + public ValueTask OnWriteFinalizingAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_provisionalAcceptances.Count > 0 && !_provisionalScheduleConfirmed) + { + return ValueTask.FromException( + new InvalidOperationException( + "Journaled state cannot be captured while durable inbox acceptance is waiting for job scheduling.")); + } + + return default; + } + + private ValueTask ValidatePersistenceBoundary(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Volatile.Read(ref _handlerExecutionDepth) != 0 + ? ValueTask.FromException( + new InvalidOperationException( + "Journaled state cannot be committed or deleted from inside a durable inbox handler. " + + "Handler effects, outgoing messages, and inbox completion are committed atomically after the handler returns.")) + : default; + } + + private void ValidatePersistenceRequest() + { + if (Volatile.Read(ref _handlerExecutionDepth) != 0) + { + throw new InvalidOperationException( + "Journaled state cannot be committed or deleted from inside a durable inbox handler. " + + "Handler effects, outgoing messages, and inbox completion are committed atomically after the handler returns."); + } + } + + public void OnWriteCompleted() + { + _pendingJobDueTime = null; + } + + public void OnRecoveryCompleted() + { + Interlocked.Increment(ref _stateGeneration); + _ownershipEpoch = Guid.NewGuid().ToString("N"); + _recoveryCompleted = true; + _provisionalAcceptances.Clear(); + _provisionalScheduleConfirmed = false; + ReconcileInboxDepth(); + } + + public void OnRecoveryStarted() + { + Interlocked.Increment(ref _stateGeneration); + _recoveryCompleted = false; + } + + public void OnRecoveryRequested() + { + Interlocked.Increment(ref _stateGeneration); + _recoveryCompleted = false; + } + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + var hasStableOwnership = DurableMessagingJobOwnership.TryGetOwnershipId( + context.Job, + out var ownershipId); + if (!string.Equals(_jobId.Value, ownershipId, StringComparison.Ordinal)) + { + if (!hasStableOwnership) + { + return DurableJobRunResult.Completed; + } + + var disposition = DurableMessagingJobOwnership.ResolveMismatch( + _recoveryCompleted, + !string.IsNullOrEmpty(_jobId.Value), + DurableMessagingJobOwnership.IsCompleted(_completedJobId.Value, ownershipId), + _inboxDict.Count > 0); + if (disposition == OwnershipMismatchDisposition.ReclaimOrphan) + { + LogOrphanedJobReclaimed(_logger, ownershipId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return DurableJobRunResult.Completed; + } + + if (disposition == OwnershipMismatchDisposition.CompleteStale) + { + return DurableJobRunResult.Completed; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (_localDrainJobIds.Contains(ownershipId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var key = new DurableMessagingPumpExecutionKey(JobName, context.Job.Id, context.RunId); + if (_pumpResults.TryTake(key, out var result, out var exception)) + { + if (exception is not null) + { + throw exception; + } + + return result!; + } + + if (_pumpResults.TryStart(key, cancellationToken, out var execution)) + { + var state = new PumpTimerState( + this, + execution, + ownershipId, + hasStableOwnership, + cancellationToken); + state.Handle.Attach(_timerRegistry.RegisterGrainTimer( + _grainContext, + static (state, timerCancellation) => state.RunAsync(timerCancellation), + state, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) + { + Interleave = false, + KeepAlive = true + })); + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + private async Task RunPumpTimerAsync( + DurableMessagingPumpExecution execution, + string ownershipId, + bool hasStableOwnership, + CancellationToken jobCancellation, + CancellationToken timerCancellation) + { + if (!_pumpResults.TryBegin(execution)) + { + return; + } + + DurableJobRunResult? result = null; + Exception? failure = null; + try + { + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + jobCancellation, + timerCancellation, + _shutdownCts.Token); + result = await ExecuteJobCoreAsync( + ownershipId, + clearOwnershipWhenEmpty: true, + hasStableOwnership, + linkedCancellation.Token); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + if (failure is null) + { + _pumpResults.Complete(execution, result!); + } + else + { + _pumpResults.Fail(execution, failure); + } + } + } + + internal async ValueTask ExecuteJobCoreAsync( + string jobId, + bool clearOwnershipWhenEmpty, + bool hasStableOwnership, + CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (string.IsNullOrEmpty(_jobId.Value)) + { + if (hasStableOwnership + && !DurableMessagingJobOwnership.IsCompleted(_completedJobId.Value, jobId)) + { + if (_inboxDict.Count == 0) + { + LogOrphanedJobReclaimed(_logger, jobId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return DurableJobRunResult.Completed; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + else if (GetDurableInboxCount() == 0) + { + return DurableJobRunResult.Completed; + } + else + { + _jobId.Value = jobId; + await _stateManager.WriteStateAsync(cancellationToken).ConfigureAwait(true); + } + } + else if (!string.Equals(_jobId.Value, jobId, StringComparison.Ordinal)) + { + return DurableJobRunResult.Completed; + } + } + finally + { + _gate.Release(); + } + + await ProcessPendingMessagesAsync(cancellationToken).ConfigureAwait(true); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + if (!string.Equals(_jobId.Value, jobId, StringComparison.Ordinal)) + { + return DurableJobRunResult.Completed; + } + + CompactProcessedMessages(); + if (GetDurableInboxCount() == 0) + { + if (_inboxDict.Count > 0) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!clearOwnershipWhenEmpty) + { + return DurableJobRunResult.Completed; + } + + _completedJobId.Value = jobId; + _jobId.Value = null; + try + { + await _stateManager.WriteStateAsync(cancellationToken).ConfigureAwait(true); + return DurableJobRunResult.Completed; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + return DurableJobRunResult.RescheduleAt( + DurableMessagingTime.AddClamped(_jobTimeProvider.GetUtcNow(), _retryDelay)); + } + } + + var nextAttempt = GetNextAttemptAt(); + var delay = nextAttempt - _timeProvider.GetUtcNow(); + return DurableJobRunResult.RescheduleAt( + DurableMessagingTime.AddClamped( + _jobTimeProvider.GetUtcNow(), + delay > TimeSpan.Zero ? delay : TimeSpan.Zero)); + } + finally + { + _gate.Release(); + } + } + + private async Task ProcessPendingMessagesAsync(CancellationToken cancellationToken) + { + var now = _timeProvider.GetUtcNow(); + var pending = _inboxDict + .Where(pair => + !_provisionalAcceptances.Contains(pair.Key) + && (!_messageStates.TryGetValue(pair.Key, out var state) + || state.NextAttemptAt is null + || state.NextAttemptAt <= now)) + .Take(_batchSize) + .Select(static pair => pair.Value) + .ToList(); + + foreach (var envelope in pending) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await ProcessMessageAsync(envelope, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + LogProcessingError(_logger, exception, envelope.MessageId, envelope.SenderId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + } + } + } + + private DateTimeOffset GetNextAttemptAt() + { + var now = _timeProvider.GetUtcNow(); + var attempts = _inboxDict.Keys + .Select(key => _messageStates.TryGetValue(key, out var state) ? state.NextAttemptAt : null) + .ToList(); + return attempts.Any(value => value is null || value <= now) + ? now + : attempts.Min()!.Value; + } + + private int GetDurableInboxCount() => + _inboxDict.Keys.Count(key => !_provisionalAcceptances.Contains(key)); + + private void CompactProcessedMessages() + { + var now = _timeProvider.GetUtcNow(); + foreach (var entry in _processed + .Where(pair => DurableMessagingTime.IsExpired(now, pair.Value, _deduplicationWindow)) + .ToList()) + { + _processed.Remove(entry.Key); + } + } + + /// + /// Processes a single message by invoking its handler. + /// + private async Task ProcessMessageAsync(DurableEnvelope envelope, CancellationToken cancellationToken) + { + var key = (envelope.SenderId, envelope.MessageId); + var grainTypeName = _grainContext.GrainId.Type.ToString(); + var stopwatch = Stopwatch.StartNew(); + if (!_inboxDict.ContainsKey(key)) + { + return; + } + + var selectionContext = new InboxHandlerSelectionContext(envelope, _grainContext.GrainId); + var stateGeneration = Volatile.Read(ref _stateGeneration); + IInboxHandler? handler = null; + Exception? handlerException = null; + try + { + _durableInbox.TryFindHandler(selectionContext, out handler); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + catch (Exception exception) + { + handlerException = exception; + } + + if (handlerException is null && handler is null) + { + if (!await DeadLetterAsync( + key, + envelope, + "No compatible handler is registered.", + stateGeneration).ConfigureAwait(true)) + { + return; + } + + stopwatch.Stop(); + _instruments.OnInboxMessageProcessed(grainTypeName, envelope.RouteKey, "dead_lettered"); + _instruments.OnInboxProcessingDuration(stopwatch.Elapsed, grainTypeName, envelope.RouteKey); + return; + } + + if (handlerException is null) + { + var context = new InboxHandlerContext(envelope, _grainContext.GrainId, _outbox, _sessionPool); + Interlocked.Increment(ref _handlerExecutionDepth); + try + { + await handler!.HandleAsync(context, cancellationToken).ConfigureAwait(true); + cancellationToken.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + catch (Exception exception) + { + handlerException = exception; + } + finally + { + Interlocked.Decrement(ref _handlerExecutionDepth); + } + } + + if (handlerException is not null) + { + if (Volatile.Read(ref _stateGeneration) != stateGeneration) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + return; + } + + LogHandlerException(_logger, handlerException, envelope.MessageId, envelope.SenderId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + var deadLettered = await RecordProcessingFailureAsync(key, handlerException).ConfigureAwait(true); + stopwatch.Stop(); + _instruments.OnInboxMessageProcessed(grainTypeName, envelope.RouteKey, deadLettered ? "dead_lettered" : "retry"); + _instruments.OnInboxProcessingDuration(stopwatch.Elapsed, grainTypeName, envelope.RouteKey); + return; + } + + await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(true); + try + { + if (Volatile.Read(ref _stateGeneration) != stateGeneration) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + return; + } + + if (_inboxDict.ContainsKey(key)) + { + RemoveMessage(key); + _messageStates.Remove(key); + _processed[key] = _timeProvider.GetUtcNow(); + try + { + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + } + } + finally + { + _gate.Release(); + } + + stopwatch.Stop(); + _instruments.OnInboxMessageProcessed(grainTypeName, envelope.RouteKey, "success"); + _instruments.OnInboxProcessingDuration(stopwatch.Elapsed, grainTypeName, envelope.RouteKey); + LogMessageProcessed(_logger, envelope.MessageId, envelope.SenderId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + } + + private void ValidateAcceptanceState( + (GrainId SenderId, Guid MessageId) key, + long expectedGeneration) + { + if (Volatile.Read(ref _stateGeneration) != expectedGeneration + || !_inboxDict.ContainsKey(key) + || string.IsNullOrEmpty(_jobId.Value)) + { + throw new InvalidOperationException( + "Durable inbox acceptance was interrupted by state recovery or deletion."); + } + } + + private async ValueTask RecordProcessingFailureAsync( + (GrainId SenderId, Guid MessageId) key, + Exception exception) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(true); + try + { + if (!_inboxDict.TryGetValue(key, out var recoveredEnvelope)) + { + return false; + } + + if (!_messageStates.TryGetValue(key, out var state)) + { + state = new InboxMessageState(); + } + + state.AttemptCount++; + state.LastError = exception.ToString(); + if (state.AttemptCount >= _maxProcessingAttempts) + { + try + { + await DeadLetterUnderGateAsync(key, recoveredEnvelope, exception.Message, state.AttemptCount).ConfigureAwait(true); + return true; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + } + + var exponent = Math.Min( + state.AttemptCount - 1, + DurableInboxOptions.MaximumBackoffExponent); + state.NextAttemptAt = DurableMessagingTime.AddClamped( + _timeProvider.GetUtcNow(), + TimeSpan.FromTicks(_retryDelay.Ticks * (1L << exponent))); + _messageStates[key] = state; + try + { + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + return false; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + } + finally + { + _gate.Release(); + } + } + + private async ValueTask DeadLetterAsync( + (GrainId SenderId, Guid MessageId) key, + DurableEnvelope envelope, + string reason, + long expectedGeneration) + { + await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(true); + try + { + if (Volatile.Read(ref _stateGeneration) != expectedGeneration + || !_inboxDict.ContainsKey(key)) + { + return false; + } + + var attemptCount = _messageStates.TryGetValue(key, out var state) ? state.AttemptCount : 0; + try + { + await DeadLetterUnderGateAsync(key, envelope, reason, attemptCount).ConfigureAwait(true); + return true; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + throw; + } + } + finally + { + _gate.Release(); + } + } + + private async ValueTask DeadLetterUnderGateAsync( + (GrainId SenderId, Guid MessageId) key, + DurableEnvelope envelope, + string reason, + int attemptCount) + { + _deadLetters[key] = new InboxDeadLetter + { + Envelope = envelope, + DeadLetteredAt = _timeProvider.GetUtcNow(), + Reason = reason, + AttemptCount = attemptCount + }; + RemoveMessage(key); + _messageStates.Remove(key); + _processed[key] = _timeProvider.GetUtcNow(); + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + + internal async Task ResumeProcessingAsync(bool replaceExisting, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + EnsureMetricsActive(); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + await EnsureJobScheduledUnderGateAsync(cancellationToken, replaceExisting: replaceExisting).ConfigureAwait(true); + ScheduleLocalDrain(); + } + + finally + { + _gate.Release(); + } + } + + public Task OnStart(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + DurableMessagingActivationValidator.Validate(_grainContext); + return ResumeProcessingAsync(replaceExisting: true, cancellationToken); + } + + public Task OnStop(CancellationToken cancellationToken) + { + StopProcessing(); + return Task.CompletedTask; + } + + internal void StopProcessing() + { + _shutdownCts.Cancel(); + if (Interlocked.Exchange(ref _metricsActive, 0) != 0) + { + _instruments.OnInboxDepthChanged(-Interlocked.Exchange(ref _reportedDepth, 0)); + } + } + + public void Dispose() + { + StopProcessing(); + _shutdownCts.Dispose(); + } + + private void EnsureMetricsActive() + { + if (Interlocked.Exchange(ref _metricsActive, 1) == 0) + { + Volatile.Write(ref _reportedDepth, _inboxDict.Count); + _instruments.OnInboxDepthChanged(_inboxDict.Count); + } + } + + private void UpdateInboxDepth(int delta) + { + if (Volatile.Read(ref _metricsActive) != 0) + { + Interlocked.Add(ref _reportedDepth, delta); + _instruments.OnInboxDepthChanged(delta); + } + } + + private void ReconcileInboxDepth() + { + if (Volatile.Read(ref _metricsActive) == 0) + { + return; + } + + var count = _inboxDict.Count; + var delta = count - Interlocked.Exchange(ref _reportedDepth, count); + if (delta != 0) + { + _instruments.OnInboxDepthChanged(delta); + } + } + + private bool RemoveMessage((GrainId SenderId, Guid MessageId) key) + { + if (!_inboxDict.Remove(key)) + { + return false; + } + + UpdateInboxDepth(-1); + + return true; + } + + // Structured logging using LoggerMessage source generator + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Registered handler for route '{RouteKey}' on grain {GrainId}")] + private static partial void LogHandlerRegistered(ILogger logger, string routeKey, GrainId grainId); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Duplicate message {MessageId} from {SenderId} to {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogDuplicateMessageDetected(ILogger logger, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Duplicate message {MessageId} from {SenderId} already in inbox for {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogDuplicateMessageInInbox(ILogger logger, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Inbox at capacity ({Count}/{Capacity}) for grain {GrainId}, rejecting message {MessageId} from {SenderId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogBackpressureRejection(ILogger logger, int count, int capacity, GrainId grainId, Guid messageId, GrainId senderId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "No handler registered for route '{RouteKey}' on grain {GrainId}, rejecting message {MessageId} from {SenderId} (CorrelationKey: {CorrelationKey})")] + private static partial void LogRouteNotFound(ILogger logger, string routeKey, GrainId grainId, Guid messageId, GrainId senderId, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Accepted message {MessageId} from {SenderId} to {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogMessageAccepted(ILogger logger, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error processing message {MessageId} from {SenderId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogProcessingError(ILogger logger, Exception exception, Guid messageId, GrainId senderId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Handler for route '{RouteKey}' not found during processing of message {MessageId} (CorrelationKey: {CorrelationKey})")] + private static partial void LogHandlerNotFoundDuringProcessing(ILogger logger, string routeKey, Guid messageId, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Processed message {MessageId} from {SenderId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogMessageProcessed(ILogger logger, Guid messageId, GrainId senderId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Handler threw exception for message {MessageId} from {SenderId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogHandlerException(ILogger logger, Exception exception, Guid messageId, GrainId senderId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Reclaimed orphaned inbox job ownership {OwnershipId} for grain {GrainId}")] + private static partial void LogOrphanedJobReclaimed(ILogger logger, string ownershipId, GrainId grainId); + + private sealed class PumpTimerState( + DurableInboxExtension owner, + DurableMessagingPumpExecution execution, + string ownershipId, + bool hasStableOwnership, + CancellationToken jobCancellation) + { + public OneShotTimerHandle Handle { get; } = new(); + + public async Task RunAsync(CancellationToken timerCancellation) + { + try + { + await owner.RunPumpTimerAsync( + execution, + ownershipId, + hasStableOwnership, + jobCancellation, + timerCancellation); + } + finally + { + Handle.Complete(); + } + } + } + + private void ScheduleLocalDrain() + { + if (_jobId.Value is not { Length: > 0 } jobId || GetDurableInboxCount() == 0) + { + return; + } + + if (!_localDrainJobIds.Add(jobId)) + { + return; + } + + var state = new LocalDrainTimerState(this, jobId); + state.Handle.Attach(_timerRegistry.RegisterGrainTimer( + _grainContext, + static (state, cancellationToken) => state.RunAsync(cancellationToken), + state, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) + { + Interleave = false, + KeepAlive = true + })); + } + + private sealed class LocalDrainTimerState(DurableInboxExtension owner, string jobId) + { + public OneShotTimerHandle Handle { get; } = new(); + + public async Task RunAsync(CancellationToken cancellationToken) + { + try + { + _ = await owner.ExecuteJobCoreAsync( + jobId, + clearOwnershipWhenEmpty: false, + hasStableOwnership: false, + cancellationToken); + } + finally + { + owner._localDrainJobIds.Remove(jobId); + Handle.Complete(); + } + } + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessageState.cs b/src/Orleans.DurableMessaging/DurableMessageState.cs new file mode 100644 index 00000000000..516d43a3850 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessageState.cs @@ -0,0 +1,65 @@ +using System; + +namespace Orleans.DurableMessaging; + +[GenerateSerializer, Alias("Orleans.DurableMessaging.InboxMessageState")] +internal sealed class InboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + +} + +[GenerateSerializer, Alias("Orleans.DurableMessaging.OutboxMessageState")] +internal sealed class OutboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + [Id(3)] + public DateTimeOffset? EnqueuedAt { get; set; } +} + +[GenerateSerializer, Alias("Orleans.DurableMessaging.InboxDeadLetter")] +internal sealed class InboxDeadLetter +{ + [Id(0)] + public required DurableEnvelope Envelope { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} + +[GenerateSerializer, Alias("Orleans.DurableMessaging.OutboxDeadLetter")] +internal sealed class OutboxDeadLetter +{ + [Id(0)] + public required DurableEnvelope Envelope { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingActivationValidator.cs b/src/Orleans.DurableMessaging/DurableMessagingActivationValidator.cs new file mode 100644 index 00000000000..50435804063 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingActivationValidator.cs @@ -0,0 +1,41 @@ +using Orleans.Concurrency; + +namespace Orleans.DurableMessaging; + +internal static class DurableMessagingActivationValidator +{ + public static void Validate(IGrainContext grainContext) + { + var grain = grainContext.GrainInstance + ?? throw new InvalidOperationException("Durable Messaging activation requires an initialized grain instance."); + var grainType = grain.GetType(); + if (grainType.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)) + { + throw new InvalidOperationException( + $"Durable Messaging requires one activation per grain identity, but grain type '{grainType}' is a stateless worker."); + } + + if (grainType.IsDefined(typeof(ReentrantAttribute), inherit: true) + || grainType.IsDefined(typeof(MayInterleaveAttribute), inherit: true)) + { + throw new InvalidOperationException( + $"Durable Messaging requires non-reentrant grain execution, but grain type '{grainType}' enables interleaving."); + } + + var grainInterfaces = grainType + .GetInterfaces() + .Where(static type => typeof(IGrain).IsAssignableFrom(type)) + .ToArray(); + var interleavableMethod = grainInterfaces + .SelectMany(static type => type.GetInterfaces().Append(type)) + .Distinct() + .SelectMany(static type => type.GetMethods()) + .FirstOrDefault(static method => method.IsDefined(typeof(AlwaysInterleaveAttribute), inherit: true)); + if (interleavableMethod is not null) + { + throw new InvalidOperationException( + $"Durable Messaging grain type '{grainType}' implements interleavable method " + + $"'{interleavableMethod.DeclaringType}.{interleavableMethod.Name}'."); + } + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingGrainParticipant.cs b/src/Orleans.DurableMessaging/DurableMessagingGrainParticipant.cs new file mode 100644 index 00000000000..46aa5274bd7 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingGrainParticipant.cs @@ -0,0 +1,16 @@ +using Orleans.Journaling; + +namespace Orleans.DurableMessaging; + +internal sealed class DurableMessagingGrainParticipant( + IDurableInbox inbox, + IDurableOutbox outbox, + DurableInboxExtension extension) : IJournaledGrainParticipant +{ + public void Initialize() + { + _ = inbox; + _ = outbox; + _ = extension; + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingInstruments.cs b/src/Orleans.DurableMessaging/DurableMessagingInstruments.cs new file mode 100644 index 00000000000..a685648d0cf --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingInstruments.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Threading; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +internal sealed class DurableMessagingInstruments(OrleansInstruments instruments) +{ + private const string MillisecondsUnit = "ms"; + private const string GrainTypeTagName = "grain_type"; + private const string RouteKeyTagName = "route_key"; + private const string StatusTagName = "status"; + + private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-durable-messaging-inbox-messages-received"); + private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-durable-messaging-inbox-messages-processed"); + private readonly Counter _outboxMessagesSent = instruments.Meter.CreateCounter("orleans-durable-messaging-outbox-messages-sent"); + private readonly Counter _outboxMessagesDelivered = instruments.Meter.CreateCounter("orleans-durable-messaging-outbox-messages-delivered"); + private readonly Counter _orphanedJobsReclaimed = instruments.Meter.CreateCounter("orleans-durable-messaging-orphaned-jobs-reclaimed"); + private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-durable-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly Histogram _outboxDeliveryDuration = instruments.Meter.CreateHistogram("orleans-durable-messaging-outbox-delivery-duration", MillisecondsUnit); + private readonly DepthTracker _inboxDepth = new(instruments.Meter, "orleans-durable-messaging-inbox-depth"); + private readonly DepthTracker _outboxDepth = new(instruments.Meter, "orleans-durable-messaging-outbox-depth"); + + internal static DurableMessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); + + internal void OnInboxDepthChanged(int delta) => _inboxDepth.Adjust(delta); + + internal void OnOutboxDepthChanged(int delta) => _outboxDepth.Adjust(delta); + + internal void OnInboxMessageReceived(string grainType, string routeKey, string status) => + Add(_inboxMessagesReceived, grainType, routeKey, status); + + internal void OnInboxMessageProcessed(string grainType, string routeKey, string status) => + Add(_inboxMessagesProcessed, grainType, routeKey, status); + + internal void OnInboxProcessingDuration(TimeSpan duration, string grainType, string routeKey) => + Record(_inboxProcessingDuration, duration, grainType, routeKey); + + internal void OnOutboxMessageSent(string grainType, string routeKey) + { + if (_outboxMessagesSent.Enabled) + { + _outboxMessagesSent.Add(1, CreateTags(grainType, routeKey)); + } + } + + internal void OnOutboxMessageDelivered(string grainType, string routeKey, string status) => + Add(_outboxMessagesDelivered, grainType, routeKey, status); + + internal void OnOutboxDeliveryDuration(TimeSpan duration, string grainType, string routeKey) => + Record(_outboxDeliveryDuration, duration, grainType, routeKey); + + internal void OnOrphanedJobReclaimed(string grainType, string jobName) + { + if (_orphanedJobsReclaimed.Enabled) + { + _orphanedJobsReclaimed.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new("job_name", jobName) + ]); + } + } + + private static void Add(Counter counter, string grainType, string routeKey, string status) + { + if (counter.Enabled) + { + counter.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new(RouteKeyTagName, routeKey), + new(StatusTagName, status) + ]); + } + } + + private static void Record(Histogram histogram, TimeSpan duration, string grainType, string routeKey) + { + if (histogram.Enabled) + { + histogram.Record(Math.Max(0, duration.TotalMilliseconds), CreateTags(grainType, routeKey)); + } + } + + private static KeyValuePair[] CreateTags(string grainType, string routeKey) => + [ + new(GrainTypeTagName, grainType), + new(RouteKeyTagName, routeKey) + ]; + + private sealed class DirectMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) => new(options); + + public void Dispose() + { + } + } + + private sealed class DepthTracker + { + private readonly ObservableGauge _gauge; + private long _value; + + public DepthTracker(Meter meter, string name) + { + _gauge = meter.CreateObservableGauge(name, () => Volatile.Read(ref _value)); + } + + public void Adjust(int delta) => Interlocked.Add(ref _value, delta); + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingJobOwnership.cs b/src/Orleans.DurableMessaging/DurableMessagingJobOwnership.cs new file mode 100644 index 00000000000..9b1d322e8db --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingJobOwnership.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using Orleans.DurableJobs; +using Orleans.Journaling; + +namespace Orleans.DurableMessaging; + +internal static class DurableMessagingJobOwnership +{ + private const string MetadataKey = "orleans.messaging.ownership-id"; + + public static IReadOnlyDictionary CreateMetadata(string ownershipId) => + new Dictionary(1, StringComparer.Ordinal) + { + [MetadataKey] = ownershipId + }; + + public static string CreateJobId(string jobName, GrainId target, string ownershipId) => + $"{Encode(Encoding.UTF8.GetBytes(jobName))}." + + $"{Encode(target.Type.Value.Value.Span)}." + + $"{Encode(target.Key.Value.Span)}." + + $"{Encode(Encoding.UTF8.GetBytes(ownershipId))}"; + + public static bool TryGetOwnershipId(DurableJob job, out string ownershipId) + { + if (job.Metadata is not null + && job.Metadata.TryGetValue(MetadataKey, out var value) + && !string.IsNullOrEmpty(value)) + { + ownershipId = value; + return true; + } + + ownershipId = job.Id; + return false; + } + + public static string NextId(string epoch, IDurableValue sequence) + { + ArgumentException.ThrowIfNullOrWhiteSpace(epoch); + sequence.Value++; + return $"{epoch}:{sequence.Value.ToString(CultureInfo.InvariantCulture)}"; + } + + public static bool IsCompleted(string? completedOwnershipId, string ownershipId) + { + if (string.Equals(completedOwnershipId, ownershipId, StringComparison.Ordinal)) + { + return true; + } + + return TryParse(completedOwnershipId, out var completedEpoch, out var completed) + && TryParse(ownershipId, out var currentEpoch, out var current) + && string.Equals(completedEpoch, currentEpoch, StringComparison.Ordinal) + && current <= completed; + } + + private static string Encode(ReadOnlySpan value) => Convert.ToHexString(value); + + private static bool TryParse(string? value, out string epoch, out long sequence) + { + sequence = 0; + var separator = value?.LastIndexOf(':') ?? -1; + if (separator <= 0 + || !long.TryParse( + value.AsSpan(separator + 1), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence)) + { + epoch = string.Empty; + return false; + } + + epoch = value![..separator]; + return true; + } + + public static OwnershipMismatchDisposition ResolveMismatch( + bool recoveryCompleted, + bool hasCurrentOwner, + bool ownershipCompleted, + bool hasWork) + { + if (!recoveryCompleted) + { + return OwnershipMismatchDisposition.WaitForRecovery; + } + + if (hasCurrentOwner || ownershipCompleted) + { + return OwnershipMismatchDisposition.CompleteStale; + } + + return hasWork + ? OwnershipMismatchDisposition.WaitForReplacement + : OwnershipMismatchDisposition.ReclaimOrphan; + } +} + +internal enum OwnershipMismatchDisposition +{ + WaitForRecovery, + WaitForReplacement, + CompleteStale, + ReclaimOrphan +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingPumpResults.cs b/src/Orleans.DurableMessaging/DurableMessagingPumpResults.cs new file mode 100644 index 00000000000..100e04c51b7 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingPumpResults.cs @@ -0,0 +1,354 @@ +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +internal readonly record struct DurableMessagingPumpExecutionKey(string JobName, string JobId, string RunId); + +internal readonly record struct DurableMessagingPumpExecution(DurableMessagingPumpExecutionKey Key, long Generation); + +internal sealed class DurableMessagingPumpResults +{ + private const int DefaultMaxRetainedEntries = 65_536; + private static readonly TimeSpan DefaultRetentionPeriod = TimeSpan.FromMinutes(10); + + private readonly object _lock = new(); + private readonly Dictionary _entries = []; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _completedRetentionPeriod; + private readonly TimeSpan _abandonedRetentionPeriod; + private readonly TimeSpan _cleanupInterval; + private readonly int _maxRetainedEntries; + private DateTimeOffset _nextCleanup; + private long _generation; + + internal DurableMessagingPumpResults() + : this(TimeProvider.System, DefaultRetentionPeriod, DefaultRetentionPeriod, DefaultMaxRetainedEntries) + { + } + + internal DurableMessagingPumpResults( + TimeProvider timeProvider, + TimeSpan completedRetentionPeriod, + TimeSpan abandonedRetentionPeriod, + int maxRetainedEntries) + { + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(completedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(abandonedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxRetainedEntries); + + _timeProvider = timeProvider; + _completedRetentionPeriod = completedRetentionPeriod; + _abandonedRetentionPeriod = abandonedRetentionPeriod; + _maxRetainedEntries = maxRetainedEntries; + _cleanupInterval = TimeSpan.FromTicks(Math.Max( + TimeSpan.FromSeconds(1).Ticks, + Math.Min(TimeSpan.FromMinutes(1).Ticks, Math.Min(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks) / 4))); + _nextCleanup = DurableMessagingTime.AddClamped(timeProvider.GetUtcNow(), _cleanupInterval); + } + + public bool TryStart( + DurableMessagingPumpExecutionKey key, + CancellationToken cancellationToken, + out DurableMessagingPumpExecution execution) + { + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + removed = Prune(now, force: _entries.Count >= _maxRetainedEntries); + if (_entries.ContainsKey(key)) + { + execution = default; + } + else + { + if (_entries.Count >= _maxRetainedEntries) + { + var candidate = _entries + .Where(static pair => pair.Value.State != EntryState.Running) + .OrderBy(static pair => pair.Value.State == EntryState.Completed ? pair.Value.CompletedAt : pair.Value.CreatedAt) + .FirstOrDefault(); + if (candidate.Value is not null && _entries.Remove(candidate.Key)) + { + (removed ??= []).Add(candidate.Value); + } + } + + if (_entries.Count >= _maxRetainedEntries) + { + execution = default; + } + else + { + execution = new(key, ++_generation); + _entries.Add(key, new Entry(execution.Generation, now)); + } + } + } + + DisposeRegistrations(removed); + if (execution == default) + { + return false; + } + + if (!cancellationToken.CanBeCanceled) + { + return true; + } + + var registration = cancellationToken.UnsafeRegister( + static state => + { + var cancellation = (CancellationState)state!; + cancellation.Owner.CancelWaiting(cancellation.Execution, cancellation.Token); + }, + new CancellationState(this, execution, cancellationToken)); + + var disposeRegistration = false; + lock (_lock) + { + if (_entries.TryGetValue(key, out var current) && current.Generation == execution.Generation) + { + current.CancellationRegistration = registration; + } + else + { + disposeRegistration = true; + } + } + + if (disposeRegistration) + { + registration.Dispose(); + } + + return true; + } + + public bool TryBegin(DurableMessagingPumpExecution execution) + { + CancellationTokenRegistration registration = default; + lock (_lock) + { + if (!_entries.TryGetValue(execution.Key, out var entry) + || entry.Generation != execution.Generation + || entry.State != EntryState.Waiting) + { + return false; + } + + entry.State = EntryState.Running; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + + registration.Dispose(); + return true; + } + + public void Complete(DurableMessagingPumpExecution execution, DurableJobRunResult result) + { + ArgumentNullException.ThrowIfNull(result); + Finish(execution, result, exception: null); + } + + public void Fail(DurableMessagingPumpExecution execution, Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + Finish(execution, result: null, exception); + } + + public bool TryTake( + DurableMessagingPumpExecutionKey key, + out DurableJobRunResult? result, + out Exception? exception) + { + Entry? removedEntry = null; + List? pruned; + lock (_lock) + { + pruned = Prune(_timeProvider.GetUtcNow(), force: false); + if (!_entries.TryGetValue(key, out var entry) || entry.State != EntryState.Completed) + { + result = null; + exception = null; + } + else + { + _entries.Remove(key); + removedEntry = entry; + result = entry.Result; + exception = entry.Exception; + } + } + + DisposeRegistrations(pruned); + if (removedEntry is null) + { + return false; + } + + removedEntry.CancellationRegistration.Dispose(); + return true; + } + + private void Finish( + DurableMessagingPumpExecution execution, + DurableJobRunResult? result, + Exception? exception) + { + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State != EntryState.Completed) + { + entry.Result = result; + entry.Exception = exception; + entry.State = EntryState.Completed; + entry.CompletedAt = now; + } + + removed = Prune(now, force: _entries.Count > _maxRetainedEntries); + } + + DisposeRegistrations(removed); + } + + private void CancelWaiting(DurableMessagingPumpExecution execution, CancellationToken cancellationToken) + { + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State == EntryState.Waiting) + { + entry.Exception = new OperationCanceledException(cancellationToken); + entry.State = EntryState.Completed; + entry.CompletedAt = now; + } + } + } + + private List? Prune(DateTimeOffset now, bool force) + { + if (!force && now < _nextCleanup) + { + return null; + } + + _nextCleanup = DurableMessagingTime.AddClamped(now, _cleanupInterval); + List? removed = null; + foreach (var pair in _entries.ToArray()) + { + var entry = pair.Value; + var expired = entry.State switch + { + EntryState.Completed => now - entry.CompletedAt >= _completedRetentionPeriod, + EntryState.Waiting => now - entry.CreatedAt >= _abandonedRetentionPeriod, + _ => false + }; + if (expired && _entries.Remove(pair.Key)) + { + (removed ??= []).Add(entry); + } + } + + if (_entries.Count <= _maxRetainedEntries) + { + return removed; + } + + foreach (var pair in _entries + .Where(static pair => pair.Value.State != EntryState.Running) + .OrderBy(static pair => pair.Value.State == EntryState.Completed ? pair.Value.CompletedAt : pair.Value.CreatedAt) + .ToArray()) + { + if (_entries.Count <= _maxRetainedEntries) + { + break; + } + + if (_entries.Remove(pair.Key)) + { + (removed ??= []).Add(pair.Value); + } + } + + return removed; + } + + private static void DisposeRegistrations(List? entries) + { + if (entries is null) + { + return; + } + + foreach (var entry in entries) + { + entry.CancellationRegistration.Dispose(); + } + } + + private sealed class Entry(long generation, DateTimeOffset createdAt) + { + public long Generation { get; } = generation; + public DateTimeOffset CreatedAt { get; } = createdAt; + public DateTimeOffset CompletedAt { get; set; } + public EntryState State { get; set; } + public DurableJobRunResult? Result { get; set; } + public Exception? Exception { get; set; } + public CancellationTokenRegistration CancellationRegistration { get; set; } + } + + private sealed record CancellationState( + DurableMessagingPumpResults Owner, + DurableMessagingPumpExecution Execution, + CancellationToken Token); + + private enum EntryState + { + Waiting, + Running, + Completed + } +} + +internal sealed class OneShotTimerHandle +{ + private readonly object _lock = new(); + private IGrainTimer? _timer; + private bool _completed; + + public void Attach(IGrainTimer timer) + { + lock (_lock) + { + if (_completed) + { + timer.Dispose(); + } + else + { + _timer = timer; + } + } + } + + public void Complete() + { + lock (_lock) + { + _completed = true; + _timer?.Dispose(); + _timer = null; + } + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingStateManagerCapabilities.cs b/src/Orleans.DurableMessaging/DurableMessagingStateManagerCapabilities.cs new file mode 100644 index 00000000000..070c42a2c91 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingStateManagerCapabilities.cs @@ -0,0 +1,20 @@ +using Orleans.Journaling; + +namespace Orleans.DurableMessaging; + +internal static class DurableMessagingStateManagerCapabilities +{ + public static void RegisterObserver(IJournaledStateManager stateManager, IJournaledStateObserver observer) + { + try + { + stateManager.RegisterObserver(observer); + } + catch (NotSupportedException exception) + { + throw new InvalidOperationException( + "Durable messaging requires IJournaledStateManager observer support through IJournaledStateManager.RegisterObserver.", + exception); + } + } +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingStateNames.cs b/src/Orleans.DurableMessaging/DurableMessagingStateNames.cs new file mode 100644 index 00000000000..59856bea9c8 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingStateNames.cs @@ -0,0 +1,20 @@ +namespace Orleans.DurableMessaging; + +internal static class DurableMessagingStateNames +{ + private const string Prefix = "__orleans.durable-messaging."; + + public const string Inbox = Prefix + "inbox"; + public const string InboxProcessed = Prefix + "inbox-processed"; + public const string InboxMessageState = Prefix + "inbox-message-state"; + public const string InboxDeadLetters = Prefix + "inbox-dead-letters"; + public const string InboxJobId = Prefix + "inbox-job-id"; + public const string InboxCompletedJobId = Prefix + "inbox-completed-job-id"; + public const string InboxJobSequence = Prefix + "inbox-job-sequence"; + public const string Outbox = Prefix + "outbox"; + public const string OutboxMessageState = Prefix + "outbox-message-state"; + public const string OutboxDeadLetters = Prefix + "outbox-dead-letters"; + public const string OutboxJobId = Prefix + "outbox-job-id"; + public const string OutboxCompletedJobId = Prefix + "outbox-completed-job-id"; + public const string OutboxJobSequence = Prefix + "outbox-job-sequence"; +} diff --git a/src/Orleans.DurableMessaging/DurableMessagingTime.cs b/src/Orleans.DurableMessaging/DurableMessagingTime.cs new file mode 100644 index 00000000000..e6bf5b43ce9 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableMessagingTime.cs @@ -0,0 +1,14 @@ +namespace Orleans.DurableMessaging; + +internal static class DurableMessagingTime +{ + public static bool IsExpired(DateTimeOffset now, DateTimeOffset timestamp, TimeSpan retention) => + now - timestamp >= retention; + + public static DateTimeOffset AddClamped(DateTimeOffset timestamp, TimeSpan duration) + { + var utcTicks = timestamp.UtcDateTime.Ticks; + var remainingTicks = DateTimeOffset.MaxValue.Ticks - utcTicks; + return new DateTimeOffset(utcTicks + Math.Min(duration.Ticks, remainingTicks), TimeSpan.Zero); + } +} diff --git a/src/Orleans.DurableMessaging/DurableOutbox.cs b/src/Orleans.DurableMessaging/DurableOutbox.cs new file mode 100644 index 00000000000..b44bc2cfef2 --- /dev/null +++ b/src/Orleans.DurableMessaging/DurableOutbox.cs @@ -0,0 +1,1223 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.DurableMessaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.TypeSystem; +using Orleans.Timers; + +namespace Orleans.DurableMessaging; + +/// +/// Durable outbox implementation which composes journaled dictionaries and provides background delivery capability. +/// Implements to start pumping messages when the grain activates. +/// +/// +/// +/// This implementation uses a background task to pump messages from the outbox to target grains. +/// The pumping task is started when the grain activates (via lifecycle subscription) and is +/// also scheduled whenever messages become durable through journal commit notifications. +/// +/// +/// IMPORTANT: Messages are only sent AFTER they have been durably persisted. This ensures that +/// if the grain crashes after Send() but before WriteStateAsync() completes, the message won't +/// be lost and can be recovered and resent on reactivation. +/// +/// +/// Messages that fail due to backpressure remain in the outbox and will be retried by the +/// background pump. This design avoids blocking the grain for extended periods, maintaining +/// Orleans' non-blocking grain model. +/// +/// +internal sealed partial class DurableOutbox : IDurableOutbox, IDurableJobFeatureHandler, ILifecycleObserver, IJournaledStateObserver +{ + internal const string JobName = "orleans.messaging.outbox-flush"; + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + private readonly IJournaledStateManager _stateManager; + private readonly IDurableDictionary _messages; + private readonly IGrainFactory _grainFactory; + private readonly IGrainContext _grainContext; + private readonly ITimerRegistry _timerRegistry; + private readonly ILogger _logger; + private readonly DurableMessagingInstruments _instruments; + private readonly TimeSpan _backpressureRetryDelay; + private readonly TimeSpan _maxRetryAge; + private readonly int _maxDeliveryAttempts; + private readonly int _batchSize; + private readonly IDurableDictionary _messageStates; + private readonly IDurableDictionary _deadLetters; + private readonly IDurableValue _jobId; + private readonly IDurableValue _completedJobId; + private readonly IDurableValue _jobSequence; + private readonly ILocalDurableJobManager _jobManager; + private readonly TimeProvider _jobTimeProvider; + private readonly DurableMessagingPumpResults _pumpResults; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly SemaphoreSlim _deliveryGate = new(1, 1); + private readonly CancellationTokenSource _shutdown = new(); + + /// + /// Set of message IDs that have been added to the outbox but not yet durably persisted. + /// Messages in this set will be skipped by the delivery pump until they become durable. + /// + private readonly HashSet _pendingMessageIds = []; + private readonly HashSet _committingMessageIds = []; + private string? _committingOwnershipId; + private DateTimeOffset? _pendingJobDueTime; + private DateTimeOffset? _replacementOwnershipDueTime; + private DateTimeOffset? _scheduledOwnershipDueTime; + private string? _replacementOwnershipId; + private string? _scheduledOwnershipId; + private string? _durableOwnershipId; + private bool _jobScheduleConfirmed; + private bool _recoveryCompleted; + private int _ensureJobScheduledQueued; + private string _ownershipEpoch = Guid.NewGuid().ToString("N"); + private long _stateGeneration; + private long? _activeDeliveryGeneration; + + private int _metricsActive; + private int _reportedDepth; + + /// + /// Creates a new DurableOutbox instance. + /// + /// State manager for durable storage. + /// Durable dictionary containing pending messages. + /// Grain factory for accessing target grains. + /// The grain context for lifecycle subscription. + /// Logger for diagnostics. + /// Journaling metrics. + /// Durable inbox options containing backpressure retry delay. + public DurableOutbox( + IJournaledStateManager manager, + [FromKeyedServices(DurableMessagingStateNames.Outbox)] IDurableDictionary messages, + IGrainFactory grainFactory, + IGrainContext grainContext, + ITimerRegistry timerRegistry, + ILogger logger, + DurableMessagingInstruments instruments, + [FromKeyedServices(DurableMessagingStateNames.OutboxMessageState)] IDurableDictionary messageStates, + [FromKeyedServices(DurableMessagingStateNames.OutboxDeadLetters)] IDurableDictionary deadLetters, + [FromKeyedServices(DurableMessagingStateNames.OutboxJobId)] IDurableValue jobId, + [FromKeyedServices(DurableMessagingStateNames.OutboxCompletedJobId)] IDurableValue completedJobId, + [FromKeyedServices(DurableMessagingStateNames.OutboxJobSequence)] IDurableValue jobSequence, + ILocalDurableJobManager jobManager, + IDurableJobHandlerRegistry jobHandlers, + DurableMessagingPumpResults pumpResults, + [FromKeyedServices(DurableJobTimeProviderNames.DurableJobs)] TimeProvider jobTimeProvider, + IOptions options) + { + ArgumentNullException.ThrowIfNull(manager); + ArgumentNullException.ThrowIfNull(messages); + ArgumentNullException.ThrowIfNull(grainFactory); + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(timerRegistry); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(instruments); + ArgumentNullException.ThrowIfNull(messageStates); + ArgumentNullException.ThrowIfNull(deadLetters); + ArgumentNullException.ThrowIfNull(jobId); + ArgumentNullException.ThrowIfNull(completedJobId); + ArgumentNullException.ThrowIfNull(jobSequence); + ArgumentNullException.ThrowIfNull(jobManager); + ArgumentNullException.ThrowIfNull(jobHandlers); + ArgumentNullException.ThrowIfNull(pumpResults); + ArgumentNullException.ThrowIfNull(jobTimeProvider); + ArgumentNullException.ThrowIfNull(options); + _stateManager = manager; + _messages = messages; + _grainFactory = grainFactory; + _grainContext = grainContext; + _timerRegistry = timerRegistry; + _logger = logger; + _instruments = instruments; + _messageStates = messageStates; + _deadLetters = deadLetters; + _jobId = jobId; + _completedJobId = completedJobId; + _jobSequence = jobSequence; + _jobManager = jobManager; + _pumpResults = pumpResults; + _jobTimeProvider = jobTimeProvider; + _backpressureRetryDelay = options.Value.BackpressureRetryDelay; + _maxRetryAge = options.Value.MaxOutboxRetryAge; + _maxDeliveryAttempts = options.Value.MaxDeliveryAttempts; + _batchSize = options.Value.OutboxBatchSize; + jobHandlers.Register(this); + DurableMessagingStateManagerCapabilities.RegisterObserver(manager, this); + + // Subscribe to the grain lifecycle to start pumping on activation + var lifecycle = grainContext.ObservableLifecycle; + lifecycle.Subscribe(RuntimeTypeNameFormatter.Format(GetType()), GrainLifecycleStage.Activate, this); + } + + /// + /// Gets all pending outbound messages (no ordering guarantee). + /// + public int Count => _messages.Count; + + /// + public IEnumerable Messages => _messages.Values; + + /// + /// Enqueues a fully-built envelope for delivery (non-generic). + /// + /// The envelope to send. + /// + /// The message is persisted atomically with grain state when + /// is called. The background pump will + /// deliver the message to the target grain ONLY AFTER the message has been durably persisted. + /// + public void Send(DurableEnvelope envelope) + { + EnsureMetricsActive(); + if (_messages.TryGetValue(envelope.MessageId, out var existingEnvelope)) + { + if (!AreEquivalent(existingEnvelope, envelope)) + { + throw new InvalidOperationException( + $"The durable outbox already contains a different envelope with message ID '{envelope.MessageId}'."); + } + + return; + } + + var startsNewBatch = Count == 0 && _pendingMessageIds.Count == 0; + + // Track this message as pending (not yet durable) + _pendingMessageIds.Add(envelope.MessageId); + + // Store envelope keyed by MessageId for O(1) lookup during removal + _messages.Add(envelope.MessageId, envelope); + if (startsNewBatch) + { + _jobId.Value = DurableMessagingJobOwnership.NextId(_ownershipEpoch, _jobSequence); + _pendingJobDueTime = _jobTimeProvider.GetUtcNow(); + _jobScheduleConfirmed = false; + } + + _messageStates[envelope.MessageId] = new OutboxMessageState + { + EnqueuedAt = _jobTimeProvider.GetUtcNow() + }; + UpdateOutboxDepth(1); + + // Record metric for message sent + var grainType = _grainContext.GrainId.Type.ToString(); + _instruments.OnOutboxMessageSent(grainType, envelope.RouteKey); + + // Durable scheduling is completed by OnWritePreparingAsync before this state can commit. + // Delivery remains fenced by _pendingMessageIds until the commit completes. + } + + private static bool AreEquivalent(DurableEnvelope left, DurableEnvelope right) + { + if (left.MessageId != right.MessageId + || left.SenderId != right.SenderId + || left.ReceiverId != right.ReceiverId + || !string.Equals(left.RouteKey, right.RouteKey, StringComparison.Ordinal) + || !Equals(left.CorrelationKey, right.CorrelationKey) + || !Nullable.Equals(left.ReplyTo, right.ReplyTo) + || left.CreatedAt != right.CreatedAt) + { + return false; + } + + if (ReferenceEquals(left.Data, right.Data)) + { + return true; + } + + if (left.Data is null || right.Data is null + || !SequenceEqual(left.Data.GetBodyBytes(), right.Data.GetBodyBytes())) + { + return false; + } + + var leftContextKeys = left.Data.ContextKeys.ToHashSet(StringComparer.Ordinal); + var rightContextKeys = right.Data.ContextKeys.ToHashSet(StringComparer.Ordinal); + if (!leftContextKeys.SetEquals(rightContextKeys)) + { + return false; + } + + foreach (var key in leftContextKeys) + { + if (!left.Data.TryGetContextBytes(key, out var leftContext) + || !right.Data.TryGetContextBytes(key, out var rightContext) + || !SequenceEqual(leftContext, rightContext)) + { + return false; + } + } + + return true; + } + + private static bool SequenceEqual(ReadOnlySequence left, ReadOnlySequence right) + { + if (left.Length != right.Length) + { + return false; + } + + return left.IsSingleSegment && right.IsSingleSegment + ? left.FirstSpan.SequenceEqual(right.FirstSpan) + : left.ToArray().AsSpan().SequenceEqual(right.ToArray()); + } + + /// + /// Called immediately before journaled state is captured for a write. + /// Snapshots the pending message and ownership set included in that write so + /// releases only state which became durable. + /// + public void OnWriteStarted() + { + _committingMessageIds.Clear(); + _committingMessageIds.UnionWith(_pendingMessageIds); + _committingOwnershipId = _jobId.Value; + } + + public async ValueTask OnWriteFinalizingAsync(CancellationToken cancellationToken) + { + if (_activeDeliveryGeneration is { } deliveryGeneration + && deliveryGeneration != Volatile.Read(ref _stateGeneration)) + { + throw new InvalidOperationException( + "Outbox delivery was interrupted by state recovery or deletion."); + } + + if (_pendingMessageIds.Count == 0 || _pendingJobDueTime is not { } dueTime || _jobScheduleConfirmed) + { + return; + } + + var jobId = _jobId.Value; + if (string.IsNullOrEmpty(jobId)) + { + throw new InvalidOperationException("Pending outbox messages require stable durable job ownership."); + } + + await _jobManager.ScheduleJobAsync( + new ScheduleJobRequest + { + JobId = DurableMessagingJobOwnership.CreateJobId(JobName, _grainContext.GrainId, jobId), + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = dueTime, + Metadata = DurableMessagingJobOwnership.CreateMetadata(jobId) + }, + cancellationToken).ConfigureAwait(true); + _jobScheduleConfirmed = true; + _scheduledOwnershipId = jobId; + _scheduledOwnershipDueTime = dueTime; + } + + public void OnWriteCompleted() + { + _durableOwnershipId = _committingOwnershipId; + _pendingMessageIds.ExceptWith(_committingMessageIds); + if (_committingMessageIds.Count > 0) + { + _pendingJobDueTime = null; + } + + _committingMessageIds.Clear(); + _committingOwnershipId = null; + if (string.Equals(_scheduledOwnershipId, _jobId.Value, StringComparison.Ordinal)) + { + _scheduledOwnershipId = null; + _scheduledOwnershipDueTime = null; + } + } + + public void OnDeleteCompleted() + { + Interlocked.Increment(ref _stateGeneration); + _ownershipEpoch = Guid.NewGuid().ToString("N"); + _pendingMessageIds.Clear(); + _committingMessageIds.Clear(); + _committingOwnershipId = null; + _pendingJobDueTime = null; + _replacementOwnershipDueTime = null; + _scheduledOwnershipDueTime = null; + _replacementOwnershipId = null; + _scheduledOwnershipId = null; + _durableOwnershipId = null; + _jobScheduleConfirmed = false; + ReconcileOutboxDepth(); + } + + public void OnRecoveryCompleted() + { + _durableOwnershipId = _jobId.Value; + _ownershipEpoch = Guid.NewGuid().ToString("N"); + _recoveryCompleted = true; + _pendingMessageIds.Clear(); + _committingMessageIds.Clear(); + _committingOwnershipId = null; + _replacementOwnershipDueTime = null; + _replacementOwnershipId = null; + if (Count > 0 && _scheduledOwnershipId is { } scheduledOwnershipId) + { + _jobId.Value = scheduledOwnershipId; + _jobScheduleConfirmed = true; + } + else + { + _pendingJobDueTime = null; + _scheduledOwnershipDueTime = null; + _scheduledOwnershipId = null; + _jobScheduleConfirmed = false; + } + + ReconcileOutboxDepth(); + if (Count > 0) + { + QueueEnsureJobScheduled(replaceExisting: true); + } + } + + public void OnRecoveryStarted() + { + Interlocked.Increment(ref _stateGeneration); + _recoveryCompleted = false; + } + + public void OnRecoveryRequested() + { + Interlocked.Increment(ref _stateGeneration); + _recoveryCompleted = false; + } + + /// + /// Removes a message after successful delivery. + /// + /// The unique identifier of the message to remove. + /// True if the message was found and removed; otherwise, false. + public bool RemoveMessage(Guid messageId) + { + _pendingMessageIds.Remove(messageId); + _messageStates.Remove(messageId); + var removed = _messages.Remove(messageId); + if (removed) + { + UpdateOutboxDepth(-1); + } + + return removed; + } + + /// + /// Tries to get a specific outbox message. + /// + /// The unique identifier of the message. + /// When this method returns, contains the envelope if found; otherwise, the default value. + /// True if the message was found; otherwise, false. + public bool TryGetMessage(Guid messageId, [MaybeNullWhen(false)] out DurableEnvelope envelope) + { + return _messages.TryGetValue(messageId, out envelope); + } + + /// + /// Triggers delivery of all durable pending messages in the outbox (single attempt). + /// + /// Cancellation token. + /// A task representing the delivery operation. + /// + /// This method makes a SINGLE attempt to deliver each durable pending message. Messages that + /// are still pending (not yet durably persisted) are skipped. Messages that fail due to + /// backpressure remain in the outbox and will be retried by the background pump. + /// + public async Task DeliverPendingMessagesAsync(CancellationToken cancellationToken = default) + { + await _deliveryGate.WaitAsync(cancellationToken).ConfigureAwait(true); + var stateGeneration = Volatile.Read(ref _stateGeneration); + _activeDeliveryGeneration = stateGeneration; + try + { + if (Count == 0) + { + return; + } + + var now = _jobTimeProvider.GetUtcNow(); + var pending = _messages.Values + .Where(envelope => + !_pendingMessageIds.Contains(envelope.MessageId) + && IsReadyForAttempt(envelope, now)) + .Take(_batchSize) + .ToList(); + + if (pending.Count == 0) + { + LogNoDurableMessages(_logger, Count); + return; + } + + LogDeliveringMessages(_logger, pending.Count); + + var grainTypeName = _grainContext.GrainId.Type.ToString(); + var deliveredCount = 0; + var backpressuredCount = 0; + var failedCount = 0; + var batchDirty = false; + + try + { + foreach (var envelope in pending) + { + var stopwatch = Stopwatch.StartNew(); + var messageNow = _jobTimeProvider.GetUtcNow(); + if (_messageStates.TryGetValue(envelope.MessageId, out var existingState) + && existingState.EnqueuedAt is { } enqueuedAt + && DurableMessagingTime.IsExpired(messageNow, enqueuedAt, _maxRetryAge)) + { + batchDirty = true; + DeadLetterExpiredMessage(envelope, existingState, messageNow); + failedCount++; + continue; + } + + try + { + var targetGrain = _grainFactory.GetGrain(envelope.ReceiverId); + var result = await targetGrain.DeliverAsync( + envelope, + cancellationToken).ConfigureAwait(true); + if (Volatile.Read(ref _stateGeneration) != stateGeneration) + { + return; + } + + stopwatch.Stop(); + switch (result.Status) + { + case DeliveryStatus.Accepted: + case DeliveryStatus.Duplicate: + case DeliveryStatus.DeadLettered: + batchDirty = true; + RemoveMessage(envelope.MessageId); + deliveredCount++; + LogMessageDelivered( + _logger, + envelope.MessageId, + envelope.SenderId, + envelope.ReceiverId, + envelope.RouteKey, + result.Status, + envelope.CorrelationKey?.ToString()); + _instruments.OnOutboxMessageDelivered(grainTypeName, envelope.RouteKey, result.Status.ToString().ToLowerInvariant()); + break; + case DeliveryStatus.Backpressured: + batchDirty = true; + RecordDeliveryFailure(envelope, "The receiver is backpressured."); + backpressuredCount++; + LogDeliveryBackpressured(_logger, envelope.MessageId, envelope.ReceiverId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + _instruments.OnOutboxMessageDelivered(grainTypeName, envelope.RouteKey, "backpressured"); + break; + case DeliveryStatus.RouteNotFound: + batchDirty = true; + RecordDeliveryFailure(envelope, result.Message ?? "The receiver has no compatible route."); + failedCount++; + LogDeliveryRouteNotFound( + _logger, + envelope.MessageId, + envelope.SenderId, + envelope.ReceiverId, + envelope.RouteKey, + envelope.CorrelationKey?.ToString(), + result.Message ?? "(no message)"); + _instruments.OnOutboxMessageDelivered(grainTypeName, envelope.RouteKey, "route_not_found"); + break; + default: + batchDirty = true; + RecordDeliveryFailure(envelope, $"Unexpected delivery status '{result.Status}'."); + failedCount++; + LogUnexpectedDeliveryStatus(_logger, result.Status, envelope.MessageId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + break; + } + + _instruments.OnOutboxDeliveryDuration(stopwatch.Elapsed, grainTypeName, envelope.RouteKey); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + if (Volatile.Read(ref _stateGeneration) != stateGeneration) + { + return; + } + + stopwatch.Stop(); + batchDirty = true; + RecordDeliveryFailure(envelope, ex.ToString()); + failedCount++; + LogDeliveryError(_logger, ex, envelope.MessageId, envelope.SenderId, envelope.ReceiverId, envelope.RouteKey, envelope.CorrelationKey?.ToString()); + _instruments.OnOutboxMessageDelivered(grainTypeName, envelope.RouteKey, "error"); + _instruments.OnOutboxDeliveryDuration(stopwatch.Elapsed, grainTypeName, envelope.RouteKey); + } + } + + if (batchDirty) + { + cancellationToken.ThrowIfCancellationRequested(); + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + batchDirty = false; + } + } + catch + { + if (batchDirty) + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + } + + throw; + } + + LogDeliveryComplete(_logger, deliveredCount, backpressuredCount, failedCount, Count); + } + finally + { + _activeDeliveryGeneration = null; + _deliveryGate.Release(); + } + } + + private void RecordDeliveryFailure(DurableEnvelope envelope, string error) + { + if (!_messageStates.TryGetValue(envelope.MessageId, out var state)) + { + state = new OutboxMessageState(); + } + + state.AttemptCount++; + state.LastError = error; + var now = _jobTimeProvider.GetUtcNow(); + state.EnqueuedAt ??= now; + if (state.AttemptCount >= _maxDeliveryAttempts + || DurableMessagingTime.IsExpired(now, state.EnqueuedAt.Value, _maxRetryAge)) + { + _deadLetters[envelope.MessageId] = new OutboxDeadLetter + { + Envelope = envelope, + DeadLetteredAt = now, + Reason = error, + AttemptCount = state.AttemptCount + }; + RemoveMessage(envelope.MessageId); + return; + } + + var exponent = Math.Min( + state.AttemptCount - 1, + DurableInboxOptions.MaximumBackoffExponent); + var delay = TimeSpan.FromTicks(_backpressureRetryDelay.Ticks * (1L << exponent)); + state.NextAttemptAt = DurableMessagingTime.AddClamped(now, delay); + _messageStates[envelope.MessageId] = state; + } + + private void DeadLetterExpiredMessage( + DurableEnvelope envelope, + OutboxMessageState state, + DateTimeOffset now) + { + _deadLetters[envelope.MessageId] = new OutboxDeadLetter + { + Envelope = envelope, + DeadLetteredAt = now, + Reason = $"The message exceeded the maximum retry age of {_maxRetryAge}.", + AttemptCount = state.AttemptCount + }; + RemoveMessage(envelope.MessageId); + } + + /// + /// Called when the grain activates. Starts the background pump if there are pending durable messages. + /// + public Task OnStart(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + DurableMessagingActivationValidator.Validate(_grainContext); + EnsureMetricsActive(); + if (Count > 0) + { + LogPumpStartingOnActivation(_logger, Count); + QueueEnsureJobScheduled(replaceExisting: true); + } + + return Task.CompletedTask; + } + + /// + /// Called when the grain deactivates. Stops the background pump. + /// + public Task OnStop(CancellationToken cancellationToken = default) + { + _shutdown.Cancel(); + if (Interlocked.Exchange(ref _metricsActive, 0) != 0) + { + _instruments.OnOutboxDepthChanged(-Interlocked.Exchange(ref _reportedDepth, 0)); + } + + return Task.CompletedTask; + } + + private void EnsureMetricsActive() + { + if (Interlocked.Exchange(ref _metricsActive, 1) == 0) + { + Volatile.Write(ref _reportedDepth, Count); + _instruments.OnOutboxDepthChanged(Count); + } + } + + private void UpdateOutboxDepth(int delta) + { + if (Volatile.Read(ref _metricsActive) != 0) + { + Interlocked.Add(ref _reportedDepth, delta); + _instruments.OnOutboxDepthChanged(delta); + } + } + + private void ReconcileOutboxDepth() + { + if (Volatile.Read(ref _metricsActive) == 0) + { + return; + } + + var count = Count; + var delta = count - Interlocked.Exchange(ref _reportedDepth, count); + if (delta != 0) + { + _instruments.OnOutboxDepthChanged(delta); + } + } + + private void QueueEnsureJobScheduled(bool replaceExisting) + { + if (Interlocked.Exchange(ref _ensureJobScheduledQueued, 1) != 0) + { + return; + } + + var state = new EnsureJobTimerState(this, replaceExisting); + try + { + state.Handle.Attach(_timerRegistry.RegisterGrainTimer( + _grainContext, + static (state, cancellationToken) => state.RunAsync(cancellationToken), + state, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) + { + Interleave = false, + KeepAlive = true + })); + } + catch + { + Volatile.Write(ref _ensureJobScheduledQueued, 0); + throw; + } + } + + internal async Task EnsureJobScheduledAsync(bool replaceExisting, CancellationToken cancellationToken) + { + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdown.Token); + var token = linkedCancellation.Token; + while (!token.IsCancellationRequested) + { + try + { + await _gate.WaitAsync(token).ConfigureAwait(true); + try + { + if (Count - _pendingMessageIds.Count <= 0 + && _scheduledOwnershipId is null) + { + _replacementOwnershipId = null; + _replacementOwnershipDueTime = null; + return; + } + + if (!replaceExisting && !string.IsNullOrEmpty(_jobId.Value)) + { + return; + } + + string ownershipId; + DateTimeOffset dueTime; + bool persistOwnership; + if (_jobScheduleConfirmed && _scheduledOwnershipId is { } scheduledOwnershipId) + { + ownershipId = scheduledOwnershipId; + dueTime = _scheduledOwnershipDueTime ?? _jobTimeProvider.GetUtcNow(); + persistOwnership = true; + } + else + { + persistOwnership = replaceExisting || string.IsNullOrEmpty(_jobId.Value); + ownershipId = GetOrCreateReplacementOwnershipId(); + dueTime = _replacementOwnershipDueTime!.Value; + await _jobManager.ScheduleJobAsync( + new ScheduleJobRequest + { + JobId = DurableMessagingJobOwnership.CreateJobId( + JobName, + _grainContext.GrainId, + ownershipId), + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = dueTime, + Metadata = DurableMessagingJobOwnership.CreateMetadata(ownershipId) + }, + token).ConfigureAwait(true); + _jobScheduleConfirmed = true; + _scheduledOwnershipId = ownershipId; + _scheduledOwnershipDueTime = dueTime; + _replacementOwnershipId = null; + _replacementOwnershipDueTime = null; + } + + if (persistOwnership) + { + _jobId.Value = ownershipId; + await _stateManager.WriteStateAsync(token).ConfigureAwait(true); + } + + _scheduledOwnershipId = null; + _scheduledOwnershipDueTime = null; + return; + } + finally + { + _gate.Release(); + } + } + + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + LogPumpLoopError(_logger, exception); + await Task.Delay(_backpressureRetryDelay, _jobTimeProvider, token) + .ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } + } + } + + private string GetOrCreateReplacementOwnershipId() + { + if (_replacementOwnershipId is null) + { + _replacementOwnershipId = DurableMessagingJobOwnership.NextId(_ownershipEpoch, _jobSequence); + _replacementOwnershipDueTime = _jobTimeProvider.GetUtcNow(); + } + + return _replacementOwnershipId; + } + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + var hasStableOwnership = DurableMessagingJobOwnership.TryGetOwnershipId( + context.Job, + out var ownershipId); + if (hasStableOwnership && IsOwnershipTransitionPending(ownershipId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!string.Equals(_jobId.Value, ownershipId, StringComparison.Ordinal)) + { + if (!hasStableOwnership) + { + return DurableJobRunResult.Completed; + } + + var disposition = DurableMessagingJobOwnership.ResolveMismatch( + _recoveryCompleted, + !string.IsNullOrEmpty(_jobId.Value), + DurableMessagingJobOwnership.IsCompleted(_completedJobId.Value, ownershipId), + Count > 0); + if (disposition == OwnershipMismatchDisposition.ReclaimOrphan) + { + LogOrphanedJobReclaimed(_logger, ownershipId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return DurableJobRunResult.Completed; + } + + if (disposition == OwnershipMismatchDisposition.CompleteStale) + { + return DurableJobRunResult.Completed; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var key = new DurableMessagingPumpExecutionKey(JobName, context.Job.Id, context.RunId); + if (_pumpResults.TryTake(key, out var result, out var exception)) + { + if (exception is not null) + { + throw exception; + } + + return result!; + } + + if (_pumpResults.TryStart(key, cancellationToken, out var execution)) + { + var state = new PumpTimerState( + this, + execution, + ownershipId, + hasStableOwnership, + cancellationToken); + state.Handle.Attach(_timerRegistry.RegisterGrainTimer( + _grainContext, + static (state, timerCancellation) => state.RunAsync(timerCancellation), + state, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) + { + Interleave = false, + KeepAlive = true + })); + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + private async Task RunPumpTimerAsync( + DurableMessagingPumpExecution execution, + string ownershipId, + bool hasStableOwnership, + CancellationToken jobCancellation, + CancellationToken timerCancellation) + { + if (!_pumpResults.TryBegin(execution)) + { + return; + } + + DurableJobRunResult? result = null; + Exception? failure = null; + try + { + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + jobCancellation, + timerCancellation, + _shutdown.Token); + result = await ExecuteJobCoreAsync( + ownershipId, + hasStableOwnership, + linkedCancellation.Token); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + if (failure is null) + { + _pumpResults.Complete(execution, result!); + } + else + { + _pumpResults.Fail(execution, failure); + } + } + } + + internal async ValueTask ExecuteJobCoreAsync( + string jobId, + bool hasStableOwnership, + CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (hasStableOwnership && IsOwnershipTransitionPending(jobId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (string.IsNullOrEmpty(_jobId.Value)) + { + if (hasStableOwnership + && !DurableMessagingJobOwnership.IsCompleted(_completedJobId.Value, jobId)) + { + if (Count == 0) + { + LogOrphanedJobReclaimed(_logger, jobId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return DurableJobRunResult.Completed; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + else if (Count == 0) + { + return DurableJobRunResult.Completed; + } + else + { + _jobId.Value = jobId; + await _stateManager.WriteStateAsync(cancellationToken).ConfigureAwait(true); + } + + return DurableJobRunResult.RescheduleAt( + DurableMessagingTime.AddClamped( + _jobTimeProvider.GetUtcNow(), + TimeSpan.FromMilliseconds(10))); + } + else if (!string.Equals(_jobId.Value, jobId, StringComparison.Ordinal)) + { + return DurableJobRunResult.Completed; + } + } + finally + { + _gate.Release(); + } + + await DeliverPendingMessagesAsync(cancellationToken).ConfigureAwait(true); + + while (true) + { + var retryOwnershipClear = false; + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + if (hasStableOwnership && IsOwnershipTransitionPending(jobId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!string.Equals(_jobId.Value, jobId, StringComparison.Ordinal)) + { + return DurableJobRunResult.Completed; + } + + if (Count == 0) + { + _completedJobId.Value = jobId; + _jobId.Value = null; + try + { + await _stateManager.WriteStateAsync(cancellationToken).ConfigureAwait(true); + _jobScheduleConfirmed = false; + return DurableJobRunResult.Completed; + } + catch + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None).ConfigureAwait(true); + retryOwnershipClear = true; + } + } + + if (!retryOwnershipClear) + { + if (_pendingMessageIds.Count > 0) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var now = _jobTimeProvider.GetUtcNow(); + var attempts = _messages.Values + .Select(envelope => GetNextAttemptAt(envelope, now)) + .ToList(); + var nextAttempt = attempts.Any(value => value is null || value <= now) + ? now + : attempts.Min()!.Value; + return DurableJobRunResult.RescheduleAt(nextAttempt <= now ? now : nextAttempt); + } + } + + finally + { + _gate.Release(); + } + + await Task.Delay(_backpressureRetryDelay, _jobTimeProvider, cancellationToken).ConfigureAwait(true); + } + } + + private bool IsOwnershipTransitionPending(string ownershipId) + { + if (string.Equals(ownershipId, _replacementOwnershipId, StringComparison.Ordinal)) + { + return true; + } + + var currentOwnershipId = _jobId.Value; + return !string.Equals(_durableOwnershipId, currentOwnershipId, StringComparison.Ordinal) + && (string.Equals(ownershipId, _durableOwnershipId, StringComparison.Ordinal) + || string.Equals(ownershipId, currentOwnershipId, StringComparison.Ordinal)); + } + + private bool IsReadyForAttempt(DurableEnvelope envelope, DateTimeOffset now) + { + if (!_messageStates.TryGetValue(envelope.MessageId, out var state)) + { + return true; + } + + return state.EnqueuedAt is { } enqueuedAt && DurableMessagingTime.IsExpired(now, enqueuedAt, _maxRetryAge) + || state.NextAttemptAt is null + || state.NextAttemptAt <= now; + } + + private DateTimeOffset? GetNextAttemptAt(DurableEnvelope envelope, DateTimeOffset now) + { + if (!_messageStates.TryGetValue(envelope.MessageId, out var state)) + { + return null; + } + + var retryAt = state.NextAttemptAt ?? now; + var expiresAt = state.EnqueuedAt is { } enqueuedAt + ? DurableMessagingTime.AddClamped(enqueuedAt, _maxRetryAge) + : retryAt; + return retryAt <= expiresAt ? retryAt : expiresAt; + } + + // Structured logging using LoggerMessage source generator + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "No durable messages to deliver (all {Count} messages are still pending)")] + private static partial void LogNoDurableMessages(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Delivering {Count} durable messages from outbox")] + private static partial void LogDeliveringMessages(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Delivered message {MessageId} from {SenderId} to {ReceiverId} on route '{RouteKey}' (Status: {Status}, CorrelationKey: {CorrelationKey})")] + private static partial void LogMessageDelivered(ILogger logger, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, DeliveryStatus status, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Route not found for message {MessageId} from {SenderId} to {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey}): {Message}")] + private static partial void LogDeliveryRouteNotFound(ILogger logger, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, string? correlationKey, string? message); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Backpressured delivering message {MessageId} to {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey}), will retry later")] + private static partial void LogDeliveryBackpressured(ILogger logger, Guid messageId, GrainId receiverId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Unexpected delivery status {Status} for message {MessageId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, Guid messageId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error delivering message {MessageId} from {SenderId} to {ReceiverId} on route '{RouteKey}' (CorrelationKey: {CorrelationKey})")] + private static partial void LogDeliveryError(ILogger logger, Exception exception, Guid messageId, GrainId senderId, GrainId receiverId, string routeKey, string? correlationKey); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] + private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Grain activated with {Count} pending outbox messages, starting pump")] + private static partial void LogPumpStartingOnActivation(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error in outbox pump loop")] + private static partial void LogPumpLoopError(ILogger logger, Exception exception); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Reclaimed orphaned outbox job ownership {OwnershipId} for grain {GrainId}")] + private static partial void LogOrphanedJobReclaimed(ILogger logger, string ownershipId, GrainId grainId); + + private sealed class PumpTimerState( + DurableOutbox owner, + DurableMessagingPumpExecution execution, + string ownershipId, + bool hasStableOwnership, + CancellationToken jobCancellation) + { + public OneShotTimerHandle Handle { get; } = new(); + + public async Task RunAsync(CancellationToken timerCancellation) + { + try + { + await owner.RunPumpTimerAsync( + execution, + ownershipId, + hasStableOwnership, + jobCancellation, + timerCancellation); + } + finally + { + Handle.Complete(); + } + } + } + + private sealed class EnsureJobTimerState(DurableOutbox owner, bool replaceExisting) + { + public OneShotTimerHandle Handle { get; } = new(); + + public async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await owner.EnsureJobScheduledAsync(replaceExisting, cancellationToken); + } + catch (OperationCanceledException) + { + } + catch (Exception exception) + { + LogPumpLoopError(owner._logger, exception); + } + finally + { + Volatile.Write(ref owner._ensureJobScheduledQueued, 0); + Handle.Complete(); + } + } + } +} diff --git a/src/Orleans.DurableMessaging/HierarchicalKey.cs b/src/Orleans.DurableMessaging/HierarchicalKey.cs new file mode 100644 index 00000000000..350c2098f2d --- /dev/null +++ b/src/Orleans.DurableMessaging/HierarchicalKey.cs @@ -0,0 +1,685 @@ +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +namespace Orleans.DurableMessaging; + +/// +/// Represents a hierarchical correlation key with support for parent-child relationships and segment-based navigation. +/// +/// +/// provides a durable correlation identifier using slash-separated segments. +/// Segments can be escaped to allow literal slash characters. +/// +[GenerateSerializer, Immutable] +[Alias("Orleans.HierarchicalKey")] +public sealed class HierarchicalKey : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// + /// The character used to escape special characters in segments. + /// + public const char EscapeCharacter = '\\'; + + /// + /// The character used to separate segments in the hierarchical key. + /// + public const char SegmentSeparator = '/'; + private static ReadOnlySpan SegmentSeparatorSpan => "/"; + + [Id(0)] + private readonly HierarchicalKey? _parent; + + [Id(1)] + private readonly ReadOnlyMemory _value; + + private HierarchicalKey(ReadOnlyMemory value) + { + _value = value; + } + + private HierarchicalKey(HierarchicalKey? parent, ReadOnlyMemory value) : this(value) + { + _parent = parent; + } + + /// + /// Creates a new hierarchical key from the specified string value. + /// + /// The string value representing the key. + /// A new hierarchical key. + /// Thrown when the value contains empty segments. + public static HierarchicalKey Create(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (!IsSegmentationValid(value)) + { + throw new ArgumentException("Value must not contain empty segments.", nameof(value)); + } + + return new(value.AsMemory()); + } + + /// + /// Creates a new hierarchical key as a child of the specified parent. + /// + /// The parent key. + /// The value for the child key. + /// A new hierarchical key. + public static HierarchicalKey Create(HierarchicalKey? parent, string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (!IsSegmentationValid(value)) + { + throw new ArgumentException("Value must not contain empty segments.", nameof(value)); + } + + return new(parent, value.AsMemory()); + } + + /// + /// Gets the parent key of this hierarchical key. + /// + /// The parent key, or null if this is a root key. + public HierarchicalKey? GetParent() => WithoutLastSegment(_value) switch + { + { Length: > 0 } value => new(_parent, value), + _ => _parent, + }; + + /// + public static HierarchicalKey Parse(string s, IFormatProvider? provider) + { + ArgumentNullException.ThrowIfNull(s); + return TryParse(s, provider, out var result) + ? result + : throw new FormatException("The value is not a valid hierarchical key."); + } + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, [MaybeNullWhen(false)] out HierarchicalKey result) + { + if (s is { Length: > 0 } && IsSegmentationValid(s)) + { + // Avoid re-validating the key. + result = new HierarchicalKey(s.AsMemory()); + return true; + } + + result = null; + return false; + } + + /// + public static HierarchicalKey Parse(ReadOnlySpan s, IFormatProvider? provider) + { + return TryParse(s, provider, out var result) + ? result + : throw new FormatException("The value is not a valid hierarchical key."); + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, [MaybeNullWhen(false)] out HierarchicalKey result) + { + if (s is { Length: > 0 } && IsSegmentationValid(s)) + { + // Avoid re-validating the key. + result = new HierarchicalKey(new string(s).AsMemory()); + return true; + } + + result = null; + return false; + } + + /// + /// Creates a new hierarchical key with escaped segment separators. + /// + /// The parent key. + /// The value to escape. + /// A new hierarchical key with escaped segment separators. + public static HierarchicalKey CreateEscaped(HierarchicalKey? parent, ReadOnlyMemory value) + { + if (value.IsEmpty) + { + throw new ArgumentException("Value must not be empty.", nameof(value)); + } + + var unescapedChars = UnescapedCharCount(value.Span); + var escapedValue = unescapedChars == 0 + ? new string(value.Span).AsMemory() + : Escape(value.Span, unescapedChars).AsMemory(); + if (!IsSegmentationValid(escapedValue.Span)) + { + throw new ArgumentException("Value contains an incomplete or invalid escape sequence.", nameof(value)); + } + + return new HierarchicalKey(parent, escapedValue); + } + + private static string Escape(ReadOnlySpan value, int unescapedChars) + { + var resultArray = ArrayPool.Shared.Rent(value.Length + unescapedChars); + try + { + var isEscaped = false; + var insertions = 0; + for (var i = 0; i < value.Length; i++) + { + var c = value[i]; + if (!isEscaped && c == SegmentSeparator) + { + resultArray[i + insertions] = EscapeCharacter; + ++insertions; + } + + resultArray[i + insertions] = c; + isEscaped = c == EscapeCharacter && !isEscaped; + } + + return new string(resultArray.AsSpan(0, value.Length + unescapedChars)); + } + finally + { + ArrayPool.Shared.Return(resultArray); + } + } + + private static int UnescapedCharCount(ReadOnlySpan value) + { + var isEscaped = false; + var result = 0; + foreach (var c in value) + { + if (isEscaped) + { + isEscaped = false; + continue; + } + + if (!isEscaped && c == SegmentSeparator) + { + ++result; + } + + if (c == EscapeCharacter) + { + isEscaped = true; + } + } + + return result; + } + + private static ReadOnlyMemory WithoutLastSegment(ReadOnlyMemory value) + { + // Find the last segment in the value string by searching for the last unescaped segment separator + var isEscaped = false; + var lastSegmentStart = 0; + var valueSpan = value.Span; + for (var i = 0; i < valueSpan.Length; i++) + { + var c = valueSpan[i]; + if (c == SegmentSeparator) + { + if (!isEscaped) + { + lastSegmentStart = i + 1; + } + + isEscaped = false; + } + + if (c == EscapeCharacter) + { + isEscaped = !isEscaped; + } + } + + return lastSegmentStart == 0 ? ReadOnlyMemory.Empty : value[..(lastSegmentStart - 1)]; + } + + private static ReadOnlySpan GetLastSegment(ReadOnlySpan value) + { + // Find the last segment in the value string by searching for the last unescaped segment separator + var isEscaped = false; + var lastSegmentStart = 0; + for (var i = 0; i < value.Length; i++) + { + var c = value[i]; + if (!isEscaped && c == SegmentSeparator) + { + lastSegmentStart = i + 1; + } + + if (c == EscapeCharacter) + { + isEscaped = !isEscaped; + } + } + + return value[lastSegmentStart..]; + } + + private static bool IsSegmentationValid(ReadOnlySpan value) + { + var isEscaped = false; + var segmentLength = 0; + foreach (var c in value) + { + ++segmentLength; + + if (isEscaped && c != SegmentSeparator && c != EscapeCharacter) + { + // The only characters which can be escaped are the escape character itself and the segment separator. + return false; + } + + if (c == EscapeCharacter) + { + // The escape character is allowed and can be used to escape itself. + isEscaped = !isEscaped; + } + else if (c == SegmentSeparator) + { + // Check if this is the start of a new segment. + if (!isEscaped) + { + if (segmentLength <= 1) + { + // Empty segments are not allowed (the segment contains only an segment separator) + return false; + } + + segmentLength = 0; + } + + isEscaped = false; + } + } + + // The sequence must not end with an incomplete escape sequence. + if (isEscaped) + { + return false; + } + + // Empty segments are not valid + if (segmentLength == 0) + { + return false; + } + + return true; + } + + /// + /// Returns true if this key is direct descendant of the provided key, false otherwise. + /// + /// The key to check this key against. + /// true if this key is a direct descendant of , false otherwise. + public bool IsChildOf(HierarchicalKey? other) => other is not null && other.IsParentOf(this); + + /// + /// Returns true if this key is a direct ancestor of provided key, false otherwise. + /// + /// The key to check this key against. + /// true if this key is a direct ancestor of , false otherwise. + public bool IsParentOf(HierarchicalKey? other) + { + if (other is null) return false; + var left = GetEnumerator(); + var right = other.GetEnumerator(); + while (true) + { + var leftValid = left.MoveNext(); + var rightValid = right.MoveNext(); + if (!leftValid && !rightValid) + { + // Completed enumeration, both keys are equal and there is no parent/child relationship between them. + return false; + } + else if (leftValid && !rightValid) + { + // The left key is longer than the right key, so it is not a prefix of it. + return false; + } + else if (!leftValid && rightValid) + { + // The right key is longer than the left key, and all common components are equal, + // so the left is the parent of the right if the right has one more segment. + return !right.MoveNext(); + } + else if (!left.Current.SequenceEqual(right.Current)) + { + // Some segment is not equal and therefore neither is a prefix of the other. + return false; + } + } + } + + /// + /// Returns true if this key is an ancestor (parent or earlier) of the provided key, false otherwise. + /// + /// The key to check this key against. + /// true if this key is a prefix of , false otherwise. + public bool IsAncestorOf(HierarchicalKey? other) + { + if (other is null) return false; + var left = GetEnumerator(); + var right = other.GetEnumerator(); + while (true) + { + var leftValid = left.MoveNext(); + var rightValid = right.MoveNext(); + if (!leftValid && !rightValid) + { + // Completed enumeration, both keys are equal and therefore prefixes of each other. + return true; + } + else if (leftValid && !rightValid) + { + // The left key is longer than the right key, so it is not a prefix of it. + return false; + } + else if (!leftValid && rightValid) + { + // The right key is longer than the left key, and all common components are equal, + // so the left is a prefix of the right. + return true; + } + else if (!left.Current.SequenceEqual(right.Current)) + { + // Some segment is not equal and therefore neither is a prefix of the other. + return false; + } + } + } + + /// + /// Creates a new key, escaping any unescaped segment separators in , and returns it. + /// + /// The value. + public static HierarchicalKey CreateEscaped(string value) => CreateEscaped(null, value.AsMemory()); + + /// + /// Creates a key which is a child of this key, escaping any unescaped segment separators in , and returns it. + /// + /// The value for the child segments. + public HierarchicalKey CreateEscapedChildKey(string value) => CreateEscaped(this, value.AsMemory()); + + /// + /// Creates a key which is a child of this key and returns it. + /// + /// The value for the child segments. + /// + public HierarchicalKey CreateChildKey(string value) => Create(this, value); + + /// + public override string ToString() => $"{this}"; + + /// + /// Gets the number of characters which comprise the key. + /// + public int Length + { + get + { + var length = 0; + foreach (var segment in this) + { + // Account for segment separators. + if (length > 0) + { + ++length; + } + + length += segment.Length; + } + + return length; + } + } + + /// + public override bool Equals(object? obj) + { + if (obj is not HierarchicalKey other) return false; + return Equals(other); + } + + /// + public override int GetHashCode() + { + // Note that we want to ensure that GetHashCode returns equal values for semantically equivalent + // instances. To achieve this, we treat the instances as a sequence of bytes, independent of + // where in the chain of instances the various segments sit. + // This allows for one instance with a value "foo/bar" and a child with "baz" to have the same + // hash code as an instance with the value "foo/bar/baz". + var length = Length; + var array = length <= 256 ? null : ArrayPool.Shared.Rent(length); + Span buffer = array ?? stackalloc char[256]; + + // Write the value into the buffer. + var didFormat = TryFormat(buffer, out var len, ReadOnlySpan.Empty, null); + buffer = buffer[..len]; + Debug.Assert(didFormat); + + HashCode hashCode = new(); + hashCode.AddBytes(MemoryMarshal.AsBytes(buffer)); + + if (array is not null) + { + ArrayPool.Shared.Return(array); + } + + return hashCode.ToHashCode(); + } + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (_parent is not null) + { + if (_parent.TryFormat(destination, out charsWritten, format, provider)) + { + destination = destination[charsWritten..]; + if (destination.Length > 0) + { + destination[0] = SegmentSeparator; + destination = destination[1..]; + ++charsWritten; + } + } + else + { + return false; + } + } + else + { + charsWritten = 0; + } + + if (_value.Span.TryCopyTo(destination)) + { + charsWritten += _value.Length; + return true; + } + + return false; + } + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => ToString(); + + /// + /// Returns an enumerator that iterates through the segments of the hierarchical key. + /// + public SegmentEnumerator GetEnumerator() => new(this); + + /// + public bool Equals(HierarchicalKey? other) + { + if (other is null) return false; + + var left = GetEnumerator(); + var right = other.GetEnumerator(); + while (true) + { + var leftValid = left.MoveNext(); + var rightValid = right.MoveNext(); + if (!leftValid && !rightValid) + { + // Completed enumeration. + return true; + } + else if (leftValid ^ rightValid) + { + // One side is complete and the other is not. + return false; + } + else if (!left.Current.SequenceEqual(right.Current)) + { + // Some segment is not equal. + return false; + } + } + } + + /// + /// Enumerator for iterating through the segments of a hierarchical key. + /// + public ref struct SegmentEnumerator(HierarchicalKey id) + { + private StructureEnumerator _enumerator = new StructureEnumerator(id); + private ReadOnlySpan _buffer = ReadOnlySpan.Empty; + + /// + /// Gets the current segment. + /// + public ReadOnlySpan Current { get; private set; } + + /// + /// Advances the enumerator to the next segment. + /// + public bool MoveNext() + { + if (_buffer.Length == 0) + { + if (!_enumerator.MoveNext()) + { + return false; + } + + _buffer = _enumerator.Current; + } + + Current = GetNextSegment(); + _buffer = _buffer[Current.Length..]; + + if (_buffer.Length > 0 && _buffer[0] == SegmentSeparator) + { + _buffer = _buffer[1..]; + } + + while (Current.Length == 0) + { + // Advance + if (!MoveNext()) + { + return false; + } + } + + return true; + } + + private readonly ReadOnlySpan GetNextSegment() + { + var buffer = _buffer; + var isEscaped = false; + var length = 0; + foreach (var c in buffer) + { + ++length; + if (c == EscapeCharacter) + { + isEscaped = !isEscaped; + continue; + } + else if (c == SegmentSeparator && !isEscaped) + { + --length; + break; + } + + isEscaped = false; + } + + return buffer[..length]; + } + } + + private struct StructureEnumerator(HierarchicalKey value) + { + private readonly HierarchicalKey? _current = value; + private int _remaining = -2; + + public ReadOnlySpan Current => _remaining switch + { + -2 => throw new InvalidOperationException($"'{nameof(MoveNext)}' must be called before accessing '{nameof(Current)}'."), + -1 => throw new InvalidOperationException("No remaining elements."), + int depth => GetElement(_current, depth), + }; + + private static int GetElementCount(HierarchicalKey? current) + { + var elements = 0; + while (current is not null) + { + ++elements; + current = current._parent; + } + + // If there is more than one segment, insert a separator segment between each. + if (elements > 1) + { + elements += elements - 1; + } + + return elements; + } + + private static ReadOnlySpan GetElement(HierarchicalKey? current, int depth) + { + // Add a separator between each segment + if (depth % 2 == 1) return SegmentSeparatorSpan; + depth /= 2; + while (depth-- > 0) + { + current = current!._parent; + } + + return current!._value.Span; + } + + public bool MoveNext() + { + // Start: calculate the number of elements + if (_remaining == -2) + { + _remaining = GetElementCount(_current); + } + + // If there are no elements remaining + if (_remaining == 0) + { + return false; + } + + --_remaining; + return true; + } + } +} diff --git a/src/Orleans.DurableMessaging/Hosting/DurableMessagingExtensions.cs b/src/Orleans.DurableMessaging/Hosting/DurableMessagingExtensions.cs new file mode 100644 index 00000000000..721f44d3e67 --- /dev/null +++ b/src/Orleans.DurableMessaging/Hosting/DurableMessagingExtensions.cs @@ -0,0 +1,136 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.DurableMessaging; +using Orleans.DurableMessaging.Configuration; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Timers; + +namespace Orleans.Hosting; + +/// +/// Extensions for configuring durable messaging. +/// +public static class DurableMessagingExtensions +{ + private const string DurableMessagingJournalFormatKey = "orleans-binary"; + + /// + /// Adds durable inbox and outbox messaging support to the silo. + /// + public static ISiloBuilder AddDurableMessaging(this ISiloBuilder builder, Action? configureOptions = null) + { + builder.AddDurableJobs(); + return builder.ConfigureServices(services => services.AddDurableMessaging(configureOptions)); + } + + /// + /// Adds durable inbox and outbox messaging services. + /// + public static IServiceCollection AddDurableMessaging(this IServiceCollection services, Action? configureOptions = null) + { + services.AddDurableJobs(); + services.TryAddSingleton(TimeProvider.System); + services.PostConfigure( + options => options.JournalFormatKey = DurableMessagingJournalFormatKey); + + var optionsBuilder = services.AddOptions(); + if (configureOptions is not null) + { + optionsBuilder.Configure(configureOptions); + } + + optionsBuilder.Validate( + options => + { + try + { + options.Validate(); + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + }, + "DurableInboxOptions validation failed."); + + services.ConfigureNamedOptionForLogging(Options.DefaultName); + services.TryAddSingleton(); + + services.TryAddScoped(sp => + { + var stateManager = sp.GetRequiredService(); + var options = sp.GetRequiredService>().Value; + return new DurableInboxExtension( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + stateManager, + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.Inbox), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxProcessed), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxMessageState), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxDeadLetters), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxJobId), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxCompletedJobId), + sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxJobSequence), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + options); + }); + + services.TryAddKeyedScoped( + typeof(IDurableInboxExtension), + (sp, _) => sp.GetRequiredService()); + services.TryAddScoped(sp => + { + var options = sp.GetRequiredService>().Value; + _ = sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxMessageState); + _ = sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxDeadLetters); + _ = sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxJobId); + _ = sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxCompletedJobId); + _ = sp.GetRequiredKeyedService>(DurableMessagingStateNames.InboxJobSequence); + _ = sp.GetRequiredService(); + return new DurableInbox( + sp.GetRequiredKeyedService>(DurableMessagingStateNames.Inbox), + sp.GetServices(), + options.MaxCapacity); + }); + services.TryAddScoped(sp => sp.GetRequiredService()); + + services.TryAddKeyedScoped(DurableMessagingStateNames.Outbox); + services.TryAddScoped(sp => sp.GetRequiredKeyedService(DurableMessagingStateNames.Outbox)); + services.TryAddScoped(); + services.TryAddScoped(sp => + { + var options = sp.GetRequiredService>().Value; + var completedRetentionPeriod = TimeSpan.FromMinutes(10); + var abandonedRetentionPeriod = options.JobStatusPollInterval <= TimeSpan.MaxValue / 4 + ? options.JobStatusPollInterval * 4 + : TimeSpan.MaxValue; + return new DurableMessagingPumpResults( + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + completedRetentionPeriod, + TimeSpan.FromTicks(Math.Max(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks)), + maxRetainedEntries: 65_536); + }); + services.TryAddScoped(); + services.TryAddEnumerable( + ServiceDescriptor.Scoped()); + return services; + } +} diff --git a/src/Orleans.DurableMessaging/IDurableInbox.cs b/src/Orleans.DurableMessaging/IDurableInbox.cs new file mode 100644 index 00000000000..54e9e71ef7f --- /dev/null +++ b/src/Orleans.DurableMessaging/IDurableInbox.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +/// +/// Durable inbox for receiving and processing messages. +/// +public interface IDurableInbox +{ + /// + /// Number of unprocessed messages. + /// + int Count { get; } + + /// + /// Maximum capacity. When reached, DeliverAsync returns Backpressured. + /// + int Capacity { get; } + + /// + /// Gets all pending messages (no ordering guarantee). + /// + IEnumerable Messages { get; } + + /// + /// Tries to get a specific message by its key. + /// + /// The sender grain ID. + /// The message ID. + /// The envelope if found. + /// True if the message exists in the inbox; otherwise, false. + bool TryGetMessage(GrainId senderId, Guid messageId, [MaybeNullWhen(false)] out DurableEnvelope envelope); + + /// + /// Registers a handler that will be evaluated using its CanHandle method. + /// Handlers are evaluated in registration order (first-match-wins). + /// + /// The handler implementation. + /// + /// + /// This is the recommended registration method. Handlers are stored in a list and + /// evaluated in registration order. The first handler whose CanHandle method returns + /// true will process the message. + /// + /// + /// For best performance, register more specific handlers before more general ones. + /// For example, register RouteKeyHandler instances before RoutePrefixHandler instances. + /// + /// + void RegisterHandler(IInboxHandler handler); + + /// + /// Registers a handler for a specific route. + /// + /// The route key to handle. + /// The handler implementation. + /// + /// This overload adapts the handler to exact, ordinal route matching. Use + /// for metadata-based matching. An exact route + /// can only be registered once for an inbox. + /// + /// + /// An exact route handler is already registered for . + /// + void RegisterHandler(string routeKey, IInboxHandler handler); + + /// + /// Checks if an exact route has a registered handler. + /// + /// The route key to check. + /// True if a handler is registered for this route; otherwise, false. + /// Thrown if is null, empty, or whitespace. + bool HasHandler(string routeKey); + + /// + /// Tries to get a handler registered for an exact route. + /// + /// The route key to get the handler for. + /// The handler if found. + /// True if a handler is registered for this route; otherwise, false. + /// Thrown if is null, empty, or whitespace. + bool TryGetHandler(string routeKey, [MaybeNullWhen(false)] out IInboxHandler handler); +} diff --git a/src/Orleans.DurableMessaging/IDurableInboxExtension.cs b/src/Orleans.DurableMessaging/IDurableInboxExtension.cs new file mode 100644 index 00000000000..7340b05730f --- /dev/null +++ b/src/Orleans.DurableMessaging/IDurableInboxExtension.cs @@ -0,0 +1,24 @@ +using Orleans; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.DurableMessaging; + +/// +/// Non-generic grain extension for durable inbox message delivery. +/// +[Alias("IDurableInboxExtension")] +public interface IDurableInboxExtension : IGrainExtension +{ + /// + /// Delivers a message to this grain's durable inbox. + /// + /// The message envelope. + /// Cancellation token. + /// Result indicating delivery/processing status. + /// + /// identifies a receiver other than the grain handling the call. + /// + [Alias("DeliverAsync")] + ValueTask DeliverAsync(DurableEnvelope envelope, CancellationToken cancellationToken = default); +} diff --git a/src/Orleans.DurableMessaging/IDurableMessagingDiagnostics.cs b/src/Orleans.DurableMessaging/IDurableMessagingDiagnostics.cs new file mode 100644 index 00000000000..da949bb580d --- /dev/null +++ b/src/Orleans.DurableMessaging/IDurableMessagingDiagnostics.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +/// +/// Provides operational access to a grain's durable messaging state. +/// +public interface IDurableMessagingDiagnostics +{ + /// + /// Gets messages which failed during inbox processing. + /// + IReadOnlyList InboxDeadLetters { get; } + + /// + /// Gets messages which could not be delivered from the outbox. + /// + IReadOnlyList OutboxDeadLetters { get; } + + /// + /// Stages removal of an inbox dead letter. + /// + /// The original sender grain identifier. + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveInboxDeadLetter(GrainId senderId, Guid messageId); + + /// + /// Stages removal of an outbox dead letter. + /// + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveOutboxDeadLetter(Guid messageId); +} + +/// +/// Describes a dead-lettered durable message. +/// +public sealed class DurableDeadLetter +{ + /// + /// Gets the message. + /// + public required DurableEnvelope Message { get; init; } + + /// + /// Gets when the message was dead-lettered. + /// + public DateTimeOffset DeadLetteredAt { get; init; } + + /// + /// Gets the terminal failure reason. + /// + public required string Reason { get; init; } + + /// + /// Gets the number of attempts made. + /// + public int AttemptCount { get; init; } +} + +internal sealed class DurableMessagingDiagnostics( + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(DurableMessagingStateNames.InboxDeadLetters)] + IDurableDictionary<(Orleans.Runtime.GrainId, Guid), InboxDeadLetter> inbox, + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(DurableMessagingStateNames.OutboxDeadLetters)] + IDurableDictionary outbox) : IDurableMessagingDiagnostics +{ + public IReadOnlyList InboxDeadLetters => + inbox.Values.Select(static entry => new DurableDeadLetter + { + Message = entry.Envelope, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public IReadOnlyList OutboxDeadLetters => + outbox.Values.Select(static entry => new DurableDeadLetter + { + Message = entry.Envelope, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public bool RemoveInboxDeadLetter(GrainId senderId, Guid messageId) => + inbox.Remove((senderId, messageId)); + + public bool RemoveOutboxDeadLetter(Guid messageId) => outbox.Remove(messageId); +} diff --git a/src/Orleans.DurableMessaging/IDurableOutbox.cs b/src/Orleans.DurableMessaging/IDurableOutbox.cs new file mode 100644 index 00000000000..579f80e0bf8 --- /dev/null +++ b/src/Orleans.DurableMessaging/IDurableOutbox.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.DurableMessaging; + +/// +/// Durable outbox for sending messages. +/// Uses dictionary storage (no ordering guarantees). +/// Non-generic interface - use to create envelopes. +/// +/// +/// +/// The outbox stores pending outbound messages in a durable dictionary until they are successfully delivered +/// to the target grain's inbox. Messages persist atomically with grain state via IJournaledStateManager.WriteStateAsync(). +/// +/// +/// Delivery is driven by the outbox's background pump, which iterates +/// pending messages and calls IDurableInboxExtension.DeliverAsync() on target grains. On successful +/// delivery (DeliveryResult.Accepted or DeliveryResult.Duplicate), messages are removed from +/// the outbox. +/// +/// +/// The outbox does NOT guarantee ordering of messages. If ordering is required, it must be implemented at +/// a higher level (e.g., using sequence numbers or correlation keys). +/// +/// +/// +/// +/// // Create and send a message via outbox +/// var envelope = context.CreateEnvelope() +/// .To(targetGrain, "payment/process") +/// .WithBody(new PaymentRequest { Amount = 100.00m }) +/// .WithCorrelationKey("order-12345") +/// .WithReplyTo(context.GrainId) +/// .Build(); +/// +/// context.Outbox.Send(envelope); +/// +/// // The message is now persisted and will be delivered by the outbox pump +/// +/// +public interface IDurableOutbox +{ + /// + /// Number of pending outbound messages. + /// + /// + /// Used for monitoring and backpressure signaling. A high count may indicate delivery issues + /// or backpressure from target grains. + /// + int Count { get; } + + /// + /// Gets all pending outbound messages (no ordering guarantee). + /// + /// + /// Used by the delivery pump to iterate and deliver pending messages. The order of enumeration + /// is undefined and may change between calls. + /// + IEnumerable Messages { get; } + + /// + /// Enqueues a fully-built envelope for delivery (non-generic). + /// Use to create the envelope. + /// + /// The envelope to send. + /// + /// + /// The message is persisted atomically with grain state when IJournaledStateManager.WriteStateAsync() + /// is called. The message remains in the outbox until the infrastructure confirms durable inbox acceptance. + /// + /// + /// Repeatedly sending an equivalent envelope with the same message ID is an idempotent no-op. + /// Sending a different envelope with an ID which is already present throws . + /// Envelope equivalence includes routing and correlation fields, creation time, body bytes, and request-context bytes. + /// + /// + /// To create an envelope, use context.CreateEnvelope() in a handler, or create a + /// directly with the appropriate SerializerSessionPool + /// and SenderId. + /// + /// + /// + /// + /// var envelope = context.CreateEnvelope() + /// .To(targetGrain, "order/confirm") + /// .WithBody(new OrderConfirmation { OrderId = "order-123" }) + /// .Build(); + /// + /// outbox.Send(envelope); + /// + /// + /// + /// An envelope with the same message ID but different content is already present. + /// + void Send(DurableEnvelope envelope); + + /// + /// Tries to get a specific outbox message. + /// + /// The unique identifier of the message. + /// When this method returns, contains the envelope if found; otherwise, the default value. + /// true if the message was found; otherwise, false. + /// + /// Used for diagnostics, monitoring, or manual retry operations. In normal operation, the delivery pump + /// iterates messages via the property. + /// + bool TryGetMessage(Guid messageId, [MaybeNullWhen(false)] out DurableEnvelope envelope); + +} diff --git a/src/Orleans.DurableMessaging/IInboxHandler.cs b/src/Orleans.DurableMessaging/IInboxHandler.cs new file mode 100644 index 00000000000..e2f03d516d9 --- /dev/null +++ b/src/Orleans.DurableMessaging/IInboxHandler.cs @@ -0,0 +1,230 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.DurableMessaging; + +/// +/// Handler for messages delivered to a specific route. +/// +/// +/// +/// Handlers are registered with an inbox using IDurableInbox.RegisterHandler(string routeKey, IInboxHandler handler). +/// When a message arrives with a matching RouteKey, the inbox invokes the handler with the full envelope and a context +/// for sending outbound messages. +/// +/// +/// For strongly-typed message handling, implement instead, which provides +/// automatic deserialization and type checking. +/// +/// +/// +/// +/// public class PaymentHandler : IInboxHandler<PaymentRequest> +/// { +/// public async ValueTask HandleAsync(PaymentRequest request, IInboxHandlerContext context, CancellationToken ct) +/// { +/// var result = await ProcessPayment(request); +/// +/// // Send reply if requested +/// if (context.Envelope.ReplyTo is { } replyTo) +/// { +/// var response = context.CreateEnvelope() +/// .To(replyTo, "payment/response") +/// .WithBody(result) +/// .WithCorrelationKey(context.Envelope.CorrelationKey) +/// .Build(); +/// +/// context.Send(response); +/// } +/// } +/// } +/// +/// // Registration +/// inbox.RegisterHandler("payment/process", new PaymentHandler()); +/// +/// +public interface IInboxHandler +{ + /// + /// Determines whether this handler can handle a message based on its metadata. + /// + /// The handler context containing the envelope and grain information. + /// true if this handler can process the message; otherwise, false. + /// + /// + /// This method enables capability-based dispatch, allowing handlers to be selected based on + /// message metadata (route key, correlation key, context values, etc.) without requiring + /// pre-registration with explicit route keys. + /// + /// + /// Performance Note: This method should perform fast, metadata-only checks. Avoid + /// deserialization, I/O operations, or expensive computations. The inbox processing pump + /// may call this method multiple times per message when searching for a matching handler. + /// + /// + /// Selection is read-only. , + /// , and + /// throw when called from this method. Stage journaled effects and outgoing messages from + /// after selection completes. + /// + /// + /// Handler Precedence: When multiple handlers return true, the first registered + /// handler wins. Register more specific handlers before generic ones to ensure correct dispatch. + /// + /// + /// + /// + /// public class OrderHandler : IInboxHandler<OrderRequest> + /// { + /// public bool CanHandle(IInboxHandlerContext context) + /// { + /// // Match specific route key + /// return context.Envelope.RouteKey == "order/process"; + /// } + /// + /// public async ValueTask HandleAsync(OrderRequest message, IInboxHandlerContext context, CancellationToken ct) + /// { + /// // Handle the order + /// } + /// } + /// + /// public class PrefixHandler : IInboxHandler + /// { + /// public bool CanHandle(IInboxHandlerContext context) + /// { + /// // Match route prefix + /// return context.Envelope.RouteKey?.StartsWith("orders/") == true; + /// } + /// + /// public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken ct) + /// { + /// // Handle any order message using context.Envelope + /// } + /// } + /// + /// + bool CanHandle(IInboxHandlerContext context); + + /// + /// Handles a message from the inbox. + /// + /// Handler context containing the envelope, grain information, and methods for sending messages. + /// Cancellation token. + /// A representing the asynchronous operation. + /// + /// + /// The envelope is available via , eliminating the need + /// for a redundant parameter. This simplifies the method signature and follows Orleans' established + /// patterns for context-based APIs. + /// + /// + /// The handler should not throw exceptions for business logic errors; instead, it should handle them + /// gracefully (e.g., log, send error response, etc.). Unhandled exceptions will be logged and may + /// prevent the message from being marked as processed, depending on the inbox configuration. + /// + /// + ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); +} + +/// +/// Typed handler adapter for strongly-typed message handling. +/// +/// The type of message this handler processes. +/// +/// +/// Implementing this interface provides automatic deserialization and type checking of the message body. +/// If deserialization fails (type mismatch, missing type, etc.), the handler throws an . +/// +/// +/// For handlers that need to handle deserialization failures gracefully, implement +/// directly and use envelope.Data.TryGetBody<T>() to attempt deserialization. +/// +/// +/// +/// +/// [GenerateSerializer] +/// public record PaymentRequest +/// { +/// [Id(0)] public required decimal Amount { get; init; } +/// [Id(1)] public required string AccountId { get; init; } +/// } +/// +/// public class PaymentHandler : IInboxHandler<PaymentRequest> +/// { +/// private readonly IPaymentService _paymentService; +/// +/// public PaymentHandler(IPaymentService paymentService) +/// { +/// _paymentService = paymentService; +/// } +/// +/// public async ValueTask HandleAsync(PaymentRequest message, IInboxHandlerContext context, CancellationToken ct) +/// { +/// // Message is already deserialized and type-checked +/// var result = await _paymentService.ProcessPayment(message.AccountId, message.Amount, ct); +/// +/// // Send reply with result +/// if (context.Envelope.ReplyTo is { } replyTo) +/// { +/// var response = context.CreateEnvelope() +/// .To(replyTo, "payment/response") +/// .WithBody(new PaymentResult { Success = result, TransactionId = Guid.NewGuid() }) +/// .WithCorrelationKey(context.Envelope.CorrelationKey) +/// .Build(); +/// +/// context.Send(response); +/// } +/// } +/// } +/// +/// +public interface IInboxHandler : IInboxHandler +{ + /// + /// Handles a typed message. + /// + /// + /// The deserialized message body. This can be when the sender + /// serialized a null reference or nullable value. + /// + /// Handler context for creating and sending envelopes. + /// Cancellation token. + /// A representing the asynchronous operation. + ValueTask HandleAsync([AllowNull] TMessage message, IInboxHandlerContext context, CancellationToken cancellationToken); + + /// + /// Default implementation that returns true (capability check deferred to derived class). + /// + /// + /// + /// The default implementation returns true, meaning typed handlers accept all messages + /// by default. Derived classes can override CanHandle to add route-based, correlation-based, + /// or other metadata filters before message processing. + /// + /// + /// Type checking happens later during the envelope handling when the message body is deserialized. + /// This design allows handlers to inspect metadata without deserialization overhead. + /// + /// + bool IInboxHandler.CanHandle(IInboxHandlerContext context) => true; + + /// + /// Default implementation with type check and deferred deserialization. + /// + /// + /// This method attempts to deserialize the envelope body as . + /// If deserialization fails, it throws an . + /// + ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (context.Envelope.Data.TryGetBody(out var typed)) + { + return HandleAsync(typed, context, cancellationToken); + } + + throw new InvalidOperationException( + $"Failed to deserialize message body for route '{context.Envelope.RouteKey}' as '{typeof(TMessage).FullName}'."); + } +} diff --git a/src/Orleans.DurableMessaging/IInboxHandlerContext.cs b/src/Orleans.DurableMessaging/IInboxHandlerContext.cs new file mode 100644 index 00000000000..409b92326a0 --- /dev/null +++ b/src/Orleans.DurableMessaging/IInboxHandlerContext.cs @@ -0,0 +1,219 @@ +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +/// +/// Context available during inbox message handling. +/// Non-generic interface using builder pattern for envelope creation. +/// +/// +/// +/// The handler context provides access to the current envelope being processed, the grain's identity, +/// and methods for creating and sending outbound messages. It follows Orleans' established patterns +/// for non-generic extension interfaces with builder-based message creation. +/// +/// +/// The method returns a pre-configured +/// with the current grain's SenderId and serialization session pool. This ensures that outbound +/// messages are properly attributed and serialized without requiring handlers to manage infrastructure concerns. +/// +/// +/// +/// +/// public class OrderHandler : IInboxHandler<OrderRequest> +/// { +/// public async ValueTask HandleAsync(OrderRequest message, IInboxHandlerContext context, CancellationToken ct) +/// { +/// // Process the order +/// var result = await ProcessOrder(message); +/// +/// // Send confirmation to requester +/// if (context.Envelope.ReplyTo is { } replyTo) +/// { +/// var response = context.CreateEnvelope() +/// .To(replyTo, "order/confirmation") +/// .WithBody(new OrderConfirmation +/// { +/// OrderId = message.OrderId, +/// Status = result.Status +/// }) +/// .WithCorrelationKey(context.Envelope.CorrelationKey) +/// .Build(); +/// +/// context.Send(response); +/// } +/// +/// // Send notification to fulfillment service +/// var fulfillmentMessage = context.CreateEnvelope() +/// .To(fulfillmentGrain, "fulfillment/create") +/// .WithBody(new FulfillmentRequest { OrderId = message.OrderId }) +/// .WithContextValue("priority", message.Priority) +/// .Build(); +/// +/// context.Send(fulfillmentMessage); +/// } +/// } +/// +/// +public interface IInboxHandlerContext +{ + /// + /// The envelope being processed. + /// + /// + /// Provides access to envelope metadata such as SenderId, CorrelationKey, ReplyTo, + /// and CreatedAt. The envelope's Data property can be used to access context values or + /// raw body bytes without deserialization. + /// + /// + /// + /// // Access correlation key + /// if (context.Envelope.CorrelationKey is { } key) + /// { + /// _logger.LogInformation("Processing message with correlation key: {Key}", key); + /// } + /// + /// // Access request context values + /// if (context.Envelope.Data.TryGetContextValue<string>("trace-id", out var traceId)) + /// { + /// Activity.Current?.SetTag("trace-id", traceId); + /// } + /// + /// // Check reply-to for request/response pattern + /// if (context.Envelope.ReplyTo is { } replyTo) + /// { + /// // This is a request that expects a response + /// } + /// + /// + DurableEnvelope Envelope { get; } + + /// + /// Gets the current grain's grain ID. + /// + /// + /// Used when setting ReplyTo on outbound messages for application-defined follow-up routing. + /// The GrainId is automatically set as the SenderId on envelopes created via . + /// + /// + /// + /// // Request with reply-to set to current grain + /// var request = context.CreateEnvelope() + /// .To(workerGrain, "work/process") + /// .WithBody(workItem) + /// .WithReplyTo(context.GrainId) // Responses come back to this grain + /// .Build(); + /// + /// context.Send(request); + /// + /// + GrainId GrainId { get; } + + /// + /// Creates a new envelope builder for sending messages. + /// The builder's method handles serialization. + /// + /// A new envelope builder pre-configured with the current grain's SenderId and session pool. + /// + /// + /// The returned builder has its SenderId and SerializerSessionPool properties already set + /// to the appropriate values for the current grain. This ensures that outbound messages are properly + /// attributed and serialized without requiring handlers to manage these infrastructure concerns. + /// + /// + /// The builder follows a fluent API pattern: + /// + /// + /// Call .To(target, routeKey) to set destination and handler route + /// Call .WithBody(value) to serialize the message body + /// Optionally call .WithCorrelationKey(), .WithReplyTo(), .WithContextValue() + /// Call .Build() to create the envelope + /// Pass the envelope to to enqueue for delivery + /// + /// + /// + /// + /// // Simple one-way message + /// var envelope = context.CreateEnvelope() + /// .To(notificationGrain, "notification/send") + /// .WithBody(new NotificationMessage { Text = "Order complete" }) + /// .Build(); + /// context.Send(envelope); + /// + /// // Request with correlation and reply-to + /// var requestBuilder = context.CreateEnvelope() + /// .To(paymentGrain, "payment/authorize") + /// .WithBody(new PaymentRequest { Amount = 100.00m }) + /// .WithReplyTo(context.GrainId) + /// .WithContextValue("idempotency-key", Guid.NewGuid().ToString()); + /// if (context.Envelope.CorrelationKey is { } correlationKey) + /// { + /// requestBuilder.WithCorrelationKey(correlationKey.CreateChildKey("payment")); + /// } + /// + /// var request = requestBuilder.Build(); + /// context.Send(request); + /// + /// + DurableEnvelopeBuilder CreateEnvelope(); + + /// + /// Sends a message via the outbox (non-generic). + /// The envelope must be fully built via . + /// + /// The envelope to send. + /// + /// + /// The message is added to the grain's outbox and will be persisted atomically with grain state + /// when IJournaledStateManager.WriteStateAsync() is called. The message will remain in the + /// outbox until it is successfully delivered to the target grain's inbox. + /// + /// + /// Delivery is handled by the outbox's background pump, which + /// iterates pending outbox messages and calls IDurableInboxExtension.DeliverAsync() on + /// target grains. + /// + /// + /// + /// + /// // Send a message + /// var envelope = context.CreateEnvelope() + /// .To(targetGrain, "order/process") + /// .WithBody(orderData) + /// .Build(); + /// + /// context.Send(envelope); + /// // Message is now in the outbox and will be delivered asynchronously + /// + /// + void Send(DurableEnvelope envelope); + + /// + /// Gets the current grain's outbox for advanced scenarios. + /// + /// + /// Most handlers should use instead of accessing the outbox directly. + /// Direct access is provided for inspecting pending messages and integrating message creation + /// with application logic. Delivery remains owned by the durable messaging infrastructure. + /// + /// + /// + /// // Check if there are pending messages + /// if (context.Outbox.Count > 100) + /// { + /// _logger.LogWarning("High outbox backlog: {Count} messages", context.Outbox.Count); + /// } + /// + /// // Inspect pending messages (advanced) + /// foreach (var pending in context.Outbox.Messages) + /// { + /// var oneHourAgo = DateTimeOffset.UtcNow.AddHours(-1); + /// if (pending.CreatedAt < oneHourAgo) + /// { + /// _logger.LogWarning("Message {Id} has been pending for over 1 hour", pending.MessageId); + /// } + /// } + /// + /// + IDurableOutbox Outbox { get; } +} diff --git a/src/Orleans.DurableMessaging/InboxHandlerContext.cs b/src/Orleans.DurableMessaging/InboxHandlerContext.cs new file mode 100644 index 00000000000..ea3a88cd168 --- /dev/null +++ b/src/Orleans.DurableMessaging/InboxHandlerContext.cs @@ -0,0 +1,161 @@ +using Orleans.Runtime; +using Orleans.Serialization.Session; + +namespace Orleans.DurableMessaging; + +/// +/// Implementation of that provides handler access to envelope metadata +/// and methods for creating and sending outbound messages. +/// +/// +/// +/// This class is instantiated by the inbox processing pump when invoking handlers. It wraps the current +/// envelope and outbox, and provides a factory method for creating pre-configured envelope builders. +/// +/// +/// The implementation is immutable and thread-safe. Envelope builders created via +/// are independent instances and can be used concurrently (though individual builders are not thread-safe). +/// +/// +internal sealed class InboxHandlerContext : IInboxHandlerContext +{ + private readonly SerializerSessionPool _sessionPool; + + /// + /// Initializes a new instance of the class. + /// + /// The envelope being processed. + /// The current grain's identity. + /// The outbox for sending messages. + /// The serializer session pool for creating envelope builders. + /// + /// This constructor is typically called by the inbox processing pump. The parameters are captured + /// and exposed via the interface properties. + /// + public InboxHandlerContext( + DurableEnvelope envelope, + GrainId grainId, + IDurableOutbox outbox, + SerializerSessionPool sessionPool) + { + Envelope = envelope; + GrainId = grainId; + Outbox = outbox; + _sessionPool = sessionPool; + } + + /// + /// + /// The envelope contains all message metadata including SenderId, CorrelationKey, + /// ReplyTo, and CreatedAt. The Data property provides access to the body + /// and request context values via deferred deserialization. + /// + public DurableEnvelope Envelope { get; } + + /// + /// + /// This GrainId is automatically set as the SenderId on all envelopes created via + /// . + /// + public GrainId GrainId { get; } + + /// + /// + /// Direct access to the outbox is provided for advanced scenarios. Most handlers should use + /// instead of calling Outbox.Send() directly. + /// + public IDurableOutbox Outbox { get; } + + /// + /// + /// + /// The returned builder has its internal properties pre-configured: + /// + /// + /// SessionPool - Set to the grain's serializer session pool + /// SenderId - Set to the current grain's GrainId + /// + /// + /// Each call creates a new builder instance. Builders are lightweight and intended to be used + /// for a single envelope creation, then discarded. For high-throughput scenarios, the builder + /// supports pooling via its internal Reset() method, though this is typically managed + /// by infrastructure code rather than user handlers. + /// + /// + /// + /// + /// // Create and send a message + /// var envelope = context.CreateEnvelope() + /// .To(targetGrain, "order/confirm") + /// .WithBody(new OrderConfirmation { OrderId = orderId }) + /// .Build(); + /// + /// context.Send(envelope); + /// + /// // Create multiple messages with the same context + /// var notification = context.CreateEnvelope() + /// .To(notificationGrain, "notification/send") + /// .WithBody(new Notification { Message = "Order confirmed" }) + /// .Build(); + /// + /// var audit = context.CreateEnvelope() + /// .To(auditGrain, "audit/log") + /// .WithBody(new AuditEvent { Action = "OrderConfirmed", OrderId = orderId }) + /// .Build(); + /// + /// context.Send(notification); + /// context.Send(audit); + /// + /// + public DurableEnvelopeBuilder CreateEnvelope() + { + return new DurableEnvelopeBuilder + { + SessionPool = _sessionPool, + SenderId = GrainId + }; + } + + /// + /// + /// + /// The envelope is added to the outbox immediately, but persistence is deferred until + /// IJournaledStateManager.WriteStateAsync() is called (typically after the handler completes + /// successfully). This ensures that outbound messages are persisted atomically with any grain + /// state changes made during handler execution. + /// + /// + /// If the handler throws an exception before state is persisted, the message will not be sent. + /// This provides transactional semantics: either the handler completes and all outbound messages + /// are sent, or the handler fails and no messages are sent. + /// + /// + /// + /// + /// public async ValueTask HandleAsync(OrderRequest request, IInboxHandlerContext context, CancellationToken ct) + /// { + /// // Process order (may throw exceptions) + /// var result = await ProcessOrder(request); + /// + /// // These messages are only persisted if ProcessOrder succeeds + /// var confirmation = context.CreateEnvelope() + /// .To(request.CustomerId, "order/confirmed") + /// .WithBody(result) + /// .Build(); + /// context.Send(confirmation); + /// + /// var fulfillment = context.CreateEnvelope() + /// .To(fulfillmentGrain, "fulfillment/create") + /// .WithBody(result) + /// .Build(); + /// context.Send(fulfillment); + /// + /// // If we reach here, both messages will be persisted atomically + /// } + /// + /// + public void Send(DurableEnvelope envelope) + { + Outbox.Send(envelope); + } +} diff --git a/src/Orleans.DurableMessaging/InboxHandlerSelectionContext.cs b/src/Orleans.DurableMessaging/InboxHandlerSelectionContext.cs new file mode 100644 index 00000000000..b63cbbd02c5 --- /dev/null +++ b/src/Orleans.DurableMessaging/InboxHandlerSelectionContext.cs @@ -0,0 +1,21 @@ +using Orleans.Runtime; + +namespace Orleans.DurableMessaging; + +internal sealed class InboxHandlerSelectionContext( + DurableEnvelope envelope, + GrainId grainId) : IInboxHandlerContext +{ + public DurableEnvelope Envelope { get; } = envelope; + + public GrainId GrainId { get; } = grainId; + + public IDurableOutbox Outbox => + throw new InvalidOperationException("Handler selection is read-only and cannot access the durable outbox."); + + public DurableEnvelopeBuilder CreateEnvelope() => + throw new InvalidOperationException("Handler selection is read-only and cannot create outbound envelopes."); + + public void Send(DurableEnvelope envelope) => + throw new InvalidOperationException("Handler selection is read-only and cannot send outbound messages."); +} diff --git a/src/Orleans.DurableMessaging/Orleans.DurableMessaging.csproj b/src/Orleans.DurableMessaging/Orleans.DurableMessaging.csproj new file mode 100644 index 00000000000..acedbcbe3ca --- /dev/null +++ b/src/Orleans.DurableMessaging/Orleans.DurableMessaging.csproj @@ -0,0 +1,23 @@ + + + Microsoft.Orleans.DurableMessaging + Microsoft Orleans Durable Messaging + Durable inbox and outbox messaging for Microsoft Orleans. + $(PackageTags) Messaging Inbox Outbox Durable + true + $(DefaultTargetFrameworks) + enable + $(NoWarn);ORLEANSEXP005 + $(VersionSuffix).alpha.1 + alpha.1 + + + + + + + + + + + diff --git a/src/Orleans.DurableMessaging/README.md b/src/Orleans.DurableMessaging/README.md new file mode 100644 index 00000000000..d758442bddb --- /dev/null +++ b/src/Orleans.DurableMessaging/README.md @@ -0,0 +1,34 @@ +# Microsoft Orleans Durable Messaging + +`Microsoft.Orleans.DurableMessaging` adds grain-scoped durable inboxes and outboxes to +Orleans Journaling. Configure the silo after selecting Durable Jobs and Journaling +storage: + +```csharp +siloBuilder + .UseInMemoryDurableJobs() + .AddDurableMessaging(); +``` + +Inject `IDurableInbox` to register handlers and `IDurableOutbox` to enqueue envelopes. +Inject `IDurableMessagingDiagnostics` to inspect dead letters and remove records after +they have been handled operationally. Removal is staged and becomes durable with the +grain's next journal write. +An outbox enqueue allocates stable job ownership and durably schedules that job before +the grain journal captures the envelope and ownership in one commit. The job safely +polls while the envelope is provisional, and dispatch starts only after the commit. If +the journal write fails, the scheduled job observes no committed envelope and completes +without sending it after activation recovery; before recovery completes, it polls the +same attempt. If recovered work exists without matching ownership, recovery establishes +a new generation before the stale job terminates. +The receiver uses the same schedule-before-commit ordering and returns `Accepted` only +after the inbox envelope and its durable drain-job ownership are stable. + +Transport is at-least-once and unordered. The receiver deduplicates by +`(SenderId, MessageId)`, providing effectively-once handler effects while the configured +deduplication record is retained. Applications which require ordering must include and +enforce their own sequence numbers. + +Durable Messaging requires a Journaling state manager with rollback support and Durable +Jobs storage appropriate for the deployment. In-memory storage is for development and +tests only. diff --git a/src/Orleans.DurableMessaging/RouteKeyHandler.cs b/src/Orleans.DurableMessaging/RouteKeyHandler.cs new file mode 100644 index 00000000000..e4a1165e373 --- /dev/null +++ b/src/Orleans.DurableMessaging/RouteKeyHandler.cs @@ -0,0 +1,133 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.DurableMessaging; + +/// +/// Base class for handlers that match messages based on an exact route key. +/// +/// +/// +/// simplifies implementing handlers that only respond to messages +/// with a specific . Derived classes override +/// to implement +/// the message processing logic. +/// +/// +/// For prefix-based routing (e.g., "orders/" matches "orders/create" and "orders/update"), derive +/// from . +/// +/// +/// Handler Precedence: When registering multiple handlers, more specific handlers +/// (like RouteKeyHandler) should be registered before generic handlers (like prefix or +/// correlation handlers) to ensure correct dispatch order. +/// +/// +/// +/// +/// public class OrderProcessingHandler : RouteKeyHandler +/// { +/// private readonly IOrderService _orderService; +/// +/// public OrderProcessingHandler(IOrderService orderService) +/// : base("order/process") +/// { +/// _orderService = orderService; +/// } +/// +/// protected override async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken ct) +/// { +/// // Deserialize the message +/// if (!context.Envelope.Data.TryGetBody<OrderRequest>(out var request)) +/// { +/// throw new InvalidOperationException("Failed to deserialize OrderRequest"); +/// } +/// +/// // Process the order +/// var result = await _orderService.ProcessOrder(request, ct); +/// +/// // Send reply if requested +/// if (context.Envelope.ReplyTo is { } replyTo) +/// { +/// var response = context.CreateEnvelope() +/// .To(replyTo, "order/response") +/// .WithBody(result) +/// .WithCorrelationKey(context.Envelope.CorrelationKey) +/// .Build(); +/// +/// context.Send(response); +/// } +/// } +/// } +/// +/// // Registration +/// inbox.RegisterHandler("order/process", new OrderProcessingHandler(orderService)); +/// +/// +public abstract class RouteKeyHandler : IInboxHandler +{ + private readonly string _routeKey; + + /// + /// Initializes a new instance of the class. + /// + /// The exact route key to match. + /// Thrown when is null. + /// Thrown when is empty or whitespace. + protected RouteKeyHandler(string routeKey) + { + ArgumentException.ThrowIfNullOrWhiteSpace(routeKey); + + _routeKey = routeKey; + } + + /// + /// Gets the route key that this handler matches. + /// + protected string RouteKey => _routeKey; + + /// + /// Determines whether this handler can handle a message based on exact route key matching. + /// + /// The handler context containing the envelope. + /// + /// true if the envelope's route key exactly matches this handler's route key; + /// otherwise, false. + /// + /// + /// This implementation performs an exact string comparison (case-sensitive) between + /// and the route key provided in the constructor. + /// + public bool CanHandle(IInboxHandlerContext context) + { + return context.Envelope.RouteKey == _routeKey; + } + + /// + /// Handles a message that matches the configured route key. + /// + /// Handler context containing the envelope and methods for sending messages. + /// Cancellation token. + /// A representing the asynchronous operation. + /// + /// + /// This method is only called when returns true, meaning the + /// envelope's route key matches the configured route key. + /// + /// + /// Derived classes should handle business logic errors gracefully (e.g., log and send error + /// response) rather than throwing exceptions. Unhandled exceptions will be logged and may + /// prevent the message from being marked as processed. + /// + /// + protected abstract ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); + + /// + /// Explicit interface implementation that delegates to the protected method. + /// + ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + return HandleAsync(context, cancellationToken); + } +} diff --git a/src/Orleans.DurableMessaging/RoutePrefixHandler.cs b/src/Orleans.DurableMessaging/RoutePrefixHandler.cs new file mode 100644 index 00000000000..571fa7e3602 --- /dev/null +++ b/src/Orleans.DurableMessaging/RoutePrefixHandler.cs @@ -0,0 +1,214 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.DurableMessaging; + +/// +/// Base class for handlers that match messages based on a route key prefix. +/// +/// +/// +/// simplifies implementing handlers that respond to messages +/// with a that starts with a specific prefix. +/// For example, a prefix of "orders/" matches "orders/create", "orders/update", and "orders/archive". +/// Derived classes override +/// to implement the message processing logic. +/// +/// +/// The prefix is automatically normalized to end with a forward slash ('/') to ensure +/// proper boundary matching. For example, "orders" becomes "orders/". This prevents false matches +/// where "order" would incorrectly match "order-archive/request". +/// +/// +/// For exact route matching, use instead. +/// +/// +/// Handler Precedence: When registering multiple handlers, more specific handlers +/// (like ) should be registered before generic prefix handlers +/// to ensure correct dispatch order. First-match-wins semantics apply. +/// +/// +/// +/// +/// public class OrderPrefixHandler : RoutePrefixHandler +/// { +/// public OrderPrefixHandler() : base("orders/") +/// { +/// } +/// +/// protected override async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken ct) +/// { +/// // Get the route suffix to determine the specific operation +/// var suffix = GetRouteSuffix(context.Envelope.RouteKey); +/// +/// switch (suffix) +/// { +/// case "create": +/// await HandleCreate(context, ct); +/// break; +/// case "archive": +/// await HandleArchive(context, ct); +/// break; +/// default: +/// throw new InvalidOperationException($"Unknown order operation: {suffix}"); +/// } +/// } +/// +/// private async ValueTask HandleCreate(IInboxHandlerContext context, CancellationToken ct) +/// { +/// if (!context.Envelope.Data.TryGetBody<CreateOrder>(out var request)) +/// { +/// throw new InvalidOperationException("Failed to deserialize CreateOrder"); +/// } +/// +/// // Process and send reply +/// var result = await ProcessRequest(request, ct); +/// +/// if (context.Envelope.ReplyTo is { } replyTo) +/// { +/// var response = context.CreateEnvelope() +/// .To(replyTo, "orders/created") +/// .WithBody(result) +/// .WithCorrelationKey(context.Envelope.CorrelationKey) +/// .Build(); +/// +/// context.Send(response); +/// } +/// } +/// +/// private async ValueTask HandleArchive(IInboxHandlerContext context, CancellationToken ct) +/// { +/// // ... +/// } +/// } +/// +/// // Registration +/// inbox.RegisterHandler(new OrderPrefixHandler()); +/// +/// +public abstract class RoutePrefixHandler : IInboxHandler +{ + private readonly string _prefix; + + /// + /// Initializes a new instance of the class. + /// + /// The route key prefix to match. Automatically normalized to end with '/'. + /// Thrown when is null. + /// Thrown when is empty or whitespace. + protected RoutePrefixHandler(string prefix) + { + ArgumentException.ThrowIfNullOrWhiteSpace(prefix); + + // Normalize prefix to always end with '/' for proper boundary matching + _prefix = prefix.EndsWith('/') ? prefix : prefix + '/'; + } + + /// + /// Gets the normalized route key prefix that this handler matches (always ends with '/'). + /// + protected string Prefix => _prefix; + + /// + /// Determines whether this handler can handle a message based on route key prefix matching. + /// + /// The handler context containing the envelope. + /// + /// true if the envelope's route key starts with this handler's prefix; + /// otherwise, false. + /// + /// + /// This implementation performs a case-sensitive prefix comparison using + /// with + /// . Returns false if the route key is null. + /// + public bool CanHandle(IInboxHandlerContext context) + { + return context.Envelope.RouteKey?.StartsWith(_prefix, StringComparison.Ordinal) == true; + } + + /// + /// Gets the suffix of a route key after removing this handler's prefix. + /// + /// The full route key from the envelope. + /// + /// The route key suffix after removing the prefix, or null if the route key + /// does not start with the prefix or is null. + /// + /// + /// + /// For example, if the prefix is "orders/" and the route key is "orders/create", + /// this method returns "create". + /// + /// + /// This helper method is useful when implementing to + /// determine the specific operation within the prefix namespace. + /// + /// + /// + /// + /// protected override async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken ct) + /// { + /// var operation = GetRouteSuffix(context.Envelope.RouteKey); + /// + /// switch (operation) + /// { + /// case "create": + /// await HandleCreate(context, ct); + /// break; + /// case "archive": + /// await HandleArchive(context, ct); + /// break; + /// default: + /// throw new InvalidOperationException($"Unknown operation: {operation}"); + /// } + /// } + /// + /// + protected string? GetRouteSuffix(string? routeKey) + { + if (string.IsNullOrEmpty(routeKey)) + { + return null; + } + + if (routeKey.StartsWith(_prefix, StringComparison.Ordinal)) + { + return routeKey.Substring(_prefix.Length); + } + + return null; + } + + /// + /// Handles a message that matches the configured route key prefix. + /// + /// Handler context containing the envelope and methods for sending messages. + /// Cancellation token. + /// A representing the asynchronous operation. + /// + /// + /// This method is only called when returns true, meaning the + /// envelope's route key starts with the configured prefix. + /// + /// + /// Derived classes can use to extract the portion of the + /// route key after the prefix to determine the specific operation to perform. + /// + /// + /// Derived classes should handle business logic errors gracefully (e.g., log and send error + /// response) rather than throwing exceptions. Unhandled exceptions will be logged and may + /// prevent the message from being marked as processed. + /// + /// + protected abstract ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); + + /// + /// Explicit interface implementation that delegates to the protected method. + /// + ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + return HandleAsync(context, cancellationToken); + } +} diff --git a/src/Orleans.Journaling/DurableGrain.cs b/src/Orleans.Journaling/DurableGrain.cs index 7ffec5d7c2a..5e530085add 100644 --- a/src/Orleans.Journaling/DurableGrain.cs +++ b/src/Orleans.Journaling/DurableGrain.cs @@ -11,6 +11,11 @@ protected DurableGrain() { participant.Participate(((IGrainBase)this).GrainContext.ObservableLifecycle); } + + foreach (var feature in ServiceProvider.GetServices().ToArray()) + { + feature.Initialize(); + } } protected IJournaledStateManager StateManager { get; } @@ -31,5 +36,11 @@ protected TState GetOrCreateState(string name, Func return result; } - protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => StateManager.WriteStateAsync(cancellationToken); + /// + /// Writes pending journaled state. + /// + /// The cancellation token. + /// A task which completes when the state is written. + protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => + StateManager.WriteStateAsync(cancellationToken); } diff --git a/src/Orleans.Journaling/IJournaledGrainParticipant.cs b/src/Orleans.Journaling/IJournaledGrainParticipant.cs new file mode 100644 index 00000000000..cbb6a5dc370 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledGrainParticipant.cs @@ -0,0 +1,12 @@ +namespace Orleans.Journaling; + +/// +/// Initializes a feature which contributes journaled state to a grain activation. +/// +public interface IJournaledGrainParticipant +{ + /// + /// Materializes the feature's grain-scoped services before journal recovery begins. + /// + void Initialize(); +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 2861d6e3deb..7dc1a12f792 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -24,6 +24,16 @@ public interface IJournaledStateManager : IAsyncDisposable /// The state instance to register. void RegisterState(string name, IJournaledState state); + /// + /// Registers an observer for durable write and recovery notifications. + /// + /// The observer. + /// + /// Each operation uses a stable snapshot of registered observers. + /// + void RegisterObserver(IJournaledStateObserver observer) => + throw new NotSupportedException("This journaled state manager does not support observers."); + /// /// Attempts to get a state registered with the manager. /// diff --git a/src/Orleans.Journaling/IJournaledStateObserver.cs b/src/Orleans.Journaling/IJournaledStateObserver.cs new file mode 100644 index 00000000000..35e32efad79 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateObserver.cs @@ -0,0 +1,76 @@ +namespace Orleans.Journaling; + +/// +/// Observes durable state manager commit and recovery boundaries. +/// +/// +/// Each operation uses a stable snapshot of registered observers. Preparation runs before +/// state capture, completion runs after a successful write boundary, and recovery completion +/// runs after every registered state has been restored. +/// +public interface IJournaledStateObserver +{ + /// + /// Validates a write request before it is queued. + /// + void OnWriteRequested() { } + + /// + /// Validates a delete request before it is queued. + /// + void OnDeleteRequested() { } + + /// + /// Called when recovery is requested, before it is queued. + /// + void OnRecoveryRequested() { } + + /// + /// Called before registered state is reset and replay begins. + /// + void OnRecoveryStarted() { } + + /// + /// Prepares external prerequisites before registered states are captured. + /// + /// The cancellation token. + /// A task representing the preparation operation. + ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) => default; + + /// + /// Finalizes prerequisites after every observer has prepared and before state capture begins. + /// + /// The cancellation token. + /// A task representing the finalization operation. + /// + /// Implementations can validate the fully prepared state and mutate only state they exclusively own. + /// + ValueTask OnWriteFinalizingAsync(CancellationToken cancellationToken) => default; + + /// + /// Validates prerequisites before all journaled state is deleted. + /// + /// The cancellation token. + /// A task representing the validation operation. + ValueTask OnDeletePreparingAsync(CancellationToken cancellationToken) => default; + + /// + /// Called after persisted journal state has been deleted successfully. + /// + void OnDeleteCompleted() { } + + /// + /// Called immediately before registered states are captured. + /// + void OnWriteStarted(); + + /// + /// Called after the write operation completes successfully. + /// + void OnWriteCompleted(); + + /// + /// Called after all registered states have been restored. + /// + void OnRecoveryCompleted(); +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 658e3594695..1f5219e47ea 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -19,6 +19,7 @@ internal sealed partial class JournaledStateManager : IJournaledStateManager, IJ #endif private readonly Dictionary _states = new(StringComparer.Ordinal); private readonly Dictionary _statesMap = []; + private readonly HashSet _observers = []; private readonly JournaledStateManagerShared _shared; private readonly IJournalStorage _storage; private readonly JournalBufferWriter _journalWriter; @@ -130,6 +131,25 @@ public void RegisterState(string name, IJournaledState state) _workSignal.Signal(); } + public void RegisterObserver(IJournaledStateObserver observer) + { + ArgumentNullException.ThrowIfNull(observer); + lock (_lock) + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_workLoop is not null) + { + throw new NotSupportedException( + "Registering a journaled state observer after initialization has started is not supported."); + } + + if (!_observers.Add(observer)) + { + throw new InvalidOperationException("The journaled state observer is already registered."); + } + } + } + public async ValueTask InitializeAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -138,6 +158,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken) bool didEnqueue; lock (_lock) { + ThrowIfInitializationFenced(); if (_workLoop is null) { _workLoop = Start(); @@ -207,7 +228,7 @@ private async Task WorkLoop() { if (fenceOnFailure) { - _state = ManagerState.Ready; + _state = ManagerState.Fenced; } } @@ -281,6 +302,34 @@ private async Task WorkLoop() var hasBufferToConsume = false; var bufferToConsumeIsCommittedBuffer = false; + IJournaledStateObserver[] observers; + lock (_lock) + { + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + await observer.OnWritePreparingAsync(_shutdownCancellation.Token).ConfigureAwait(true); + } + + foreach (var observer in observers) + { + await observer.OnWriteFinalizingAsync(_shutdownCancellation.Token).ConfigureAwait(true); + } + + foreach (var observer in observers) + { + try + { + observer.OnWriteStarted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteStarted)); + } + } + lock (_lock) { if (isSnapshot) @@ -444,11 +493,34 @@ private async Task WorkLoop() } } + foreach (var observer in observers) + { + try + { + observer.OnWriteCompleted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteCompleted)); + } + } + break; } case DeleteStateWorkItem: { + IJournaledStateObserver[] observers; + lock (_lock) + { + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + await observer.OnDeletePreparingAsync(_shutdownCancellation.Token).ConfigureAwait(true); + } + // Clear storage. await DeleteStorageAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -465,6 +537,19 @@ private async Task WorkLoop() _journalStreamDirectory.Set(name, id); } } + + foreach (var observer in observers) + { + try + { + observer.OnDeleteCompleted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnDeleteCompleted)); + } + } + break; } @@ -483,6 +568,7 @@ private async Task WorkLoop() { lock (_lock) { + ThrowIfInitializationFenced(); _state = ManagerState.Ready; } break; @@ -573,7 +659,10 @@ private async Task WorkLoop() } catch (Exception exception) { - needsRecovery = true; + lock (_lock) + { + needsRecovery = _state is not ManagerState.Fenced; + } if (_shutdownCancellation.Token.IsCancellationRequested) { CompleteRecoveryTrigger(); @@ -695,6 +784,18 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + IJournaledStateObserver[] observers; + lock (_lock) + { + ThrowIfStateOperationsUnavailable(); + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + observer.OnDeleteRequested(); + } + Task task; bool didEnqueue; lock (_lock) @@ -724,6 +825,24 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) private async Task RecoverAsync(CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); + IJournaledStateObserver[] observers; + lock (_lock) + { + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + try + { + observer.OnRecoveryStarted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnRecoveryStarted)); + } + } + lock (_lock) { ResetForRecovery(); @@ -742,6 +861,14 @@ private async Task RecoverAsync(CancellationToken cancellationToken) lock (_lock) { + foreach (var (name, state) in _states) + { + if (state is not RetiredState && !_journalStreamDirectory.ContainsKey(name)) + { + _journalStreamDirectory.Set(name, _journalStreamDirectory.GetNextJournalStreamId()); + } + } + foreach ((var name, var state) in _states) { state.OnRecoveryCompleted(); @@ -754,6 +881,19 @@ private async Task RecoverAsync(CancellationToken cancellationToken) LogRetiredStateDetected(_shared.Logger, name); } } + + } + } + + foreach (var observer in observers) + { + try + { + observer.OnRecoveryCompleted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnRecoveryCompleted)); } } } @@ -868,6 +1008,17 @@ private InvalidOperationException CreateRecoveryFormatException(Exception except public async ValueTask WriteStateAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + IJournaledStateObserver[] observers; + lock (_lock) + { + ThrowIfStateOperationsUnavailable(); + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + observer.OnWriteRequested(); + } Task pendingWrite; bool didEnqueue; @@ -903,6 +1054,23 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + IJournaledStateObserver[] observers; + lock (_lock) + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_state is ManagerState.Unknown) + { + throw new InvalidOperationException("The journaled state manager has not been initialized."); + } + + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + observer.OnRecoveryRequested(); + } + Task pendingRecovery; bool didEnqueue; lock (_lock) @@ -943,6 +1111,15 @@ private void ThrowIfStateOperationsUnavailable() } } + private void ThrowIfInitializationFenced() + { + if (_state is ManagerState.Fenced) + { + throw new InvalidOperationException( + "The journaled state manager is fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); + } + } + private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -1348,6 +1525,11 @@ void IJournaledState.AppendEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journaled state observer callback {Callback} failed.")] + private static partial void LogObserverError(ILogger logger, Exception exception, string callback); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/api/Orleans.DurableMessaging/Orleans.DurableMessaging.cs b/src/api/Orleans.DurableMessaging/Orleans.DurableMessaging.cs new file mode 100644 index 00000000000..39574b84f74 --- /dev/null +++ b/src/api/Orleans.DurableMessaging/Orleans.DurableMessaging.cs @@ -0,0 +1,495 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Orleans.DurableMessaging +{ + public abstract partial class CorrelationHandler : IInboxHandler + { + protected CorrelationHandler(HierarchicalKey correlationKey) { } + + protected HierarchicalKey CorrelationKey { get { throw null; } } + + public bool CanHandle(IInboxHandlerContext context) { throw null; } + + protected abstract System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken) { throw null; } + } + + [GenerateSerializer] + [Alias("Orleans.DurableMessaging.DeliveryResult")] + public readonly partial struct DeliveryResult + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(2)] + public string? Message { get { throw null; } init { } } + + [Id(0)] + public DeliveryStatus Status { get { throw null; } init { } } + + public static DeliveryResult Accepted() { throw null; } + + public static DeliveryResult Backpressured() { throw null; } + + public static DeliveryResult DeadLettered(string reason) { throw null; } + + public static DeliveryResult Duplicate() { throw null; } + + public static DeliveryResult RouteNotFound(string routeKey) { throw null; } + } + + public enum DeliveryStatus + { + Accepted = 0, + Duplicate = 1, + Backpressured = 2, + RouteNotFound = 3, + DeadLettered = 6 + } + + public sealed partial class DurableDeadLetter + { + public int AttemptCount { get { throw null; } init { } } + + public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + + public required DurableEnvelope Message { get { throw null; } init { } } + + public required string Reason { get { throw null; } init { } } + } + + [GenerateSerializer] + [Alias("Orleans.DurableMessaging.DurableEnvelope")] + public readonly partial struct DurableEnvelope + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(4)] + public HierarchicalKey? CorrelationKey { get { throw null; } init { } } + + [Id(7)] + public System.DateTimeOffset CreatedAt { get { throw null; } init { } } + + [Id(6)] + public required DurableEnvelopeData Data { get { throw null; } init { } } + + [Id(0)] + public required System.Guid MessageId { get { throw null; } init { } } + + [Id(2)] + public required Runtime.GrainId ReceiverId { get { throw null; } init { } } + + [Id(5)] + public Runtime.GrainId? ReplyTo { get { throw null; } init { } } + + [Id(3)] + public required string RouteKey { get { throw null; } init { } } + + [Id(1)] + public required Runtime.GrainId SenderId { get { throw null; } init { } } + } + + public sealed partial class DurableEnvelopeBuilder : System.Buffers.IBufferWriter + { + public DurableEnvelopeBuilder(Serialization.Session.SerializerSessionPool sessionPool, Runtime.GrainId senderId) { } + + public DurableEnvelope Build() { throw null; } + + void System.Buffers.IBufferWriter.Advance(int count) { } + + System.Memory System.Buffers.IBufferWriter.GetMemory(int sizeHint) { throw null; } + + System.Span System.Buffers.IBufferWriter.GetSpan(int sizeHint) { throw null; } + + public DurableEnvelopeBuilder To(Runtime.GrainId target, string routeKey) { throw null; } + + public DurableEnvelopeBuilder WithBody(T body) { throw null; } + + public DurableEnvelopeBuilder WithContextValue(string key, T value) { throw null; } + + public DurableEnvelopeBuilder WithCorrelationKey(HierarchicalKey correlationKey) { throw null; } + + public DurableEnvelopeBuilder WithCorrelationKey(string correlationKey) { throw null; } + + public DurableEnvelopeBuilder WithReplyTo(Runtime.GrainId replyTo) { throw null; } + } + + [GenerateSerializer] + [Alias("Orleans.DurableMessaging.DurableEnvelopeData")] + public sealed partial class DurableEnvelopeData + { + internal DurableEnvelopeData() { } + + public System.Collections.Generic.IEnumerable ContextKeys { get { throw null; } } + + public System.Buffers.ReadOnlySequence GetBodyBytes() { throw null; } + + public bool HasContextKey(string key) { throw null; } + + public bool TryGetBody(out T value) { throw null; } + + public bool TryGetContextBytes(string key, out System.Buffers.ReadOnlySequence value) { throw null; } + + public bool TryGetContextValue(string key, out T value) { throw null; } + } + + [GenerateSerializer] + [Immutable] + [Alias("Orleans.HierarchicalKey")] + public sealed partial class HierarchicalKey : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + internal HierarchicalKey() { } + + public const char EscapeCharacter = '\\'; + public const char SegmentSeparator = '/'; + public int Length { get { throw null; } } + + public static HierarchicalKey Create(HierarchicalKey? parent, string value) { throw null; } + + public static HierarchicalKey Create(string value) { throw null; } + + public HierarchicalKey CreateChildKey(string value) { throw null; } + + public static HierarchicalKey CreateEscaped(HierarchicalKey? parent, System.ReadOnlyMemory value) { throw null; } + + public static HierarchicalKey CreateEscaped(string value) { throw null; } + + public HierarchicalKey CreateEscapedChildKey(string value) { throw null; } + + public bool Equals(HierarchicalKey? other) { throw null; } + + public override bool Equals(object? obj) { throw null; } + + public SegmentEnumerator GetEnumerator() { throw null; } + + public override int GetHashCode() { throw null; } + + public HierarchicalKey? GetParent() { throw null; } + + public bool IsAncestorOf(HierarchicalKey? other) { throw null; } + + public bool IsChildOf(HierarchicalKey? other) { throw null; } + + public bool IsParentOf(HierarchicalKey? other) { throw null; } + + static HierarchicalKey System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static HierarchicalKey System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override string ToString() { throw null; } + + public string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + public ref partial struct SegmentEnumerator + { + private object _dummy; + private int _dummyPrimitive; + public SegmentEnumerator(HierarchicalKey id) { } + + public System.ReadOnlySpan Current { get { throw null; } } + + public bool MoveNext() { throw null; } + } + } + + public partial interface IDurableInbox + { + int Capacity { get; } + + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + bool HasHandler(string routeKey); + void RegisterHandler(IInboxHandler handler); + void RegisterHandler(string routeKey, IInboxHandler handler); + bool TryGetHandler(string routeKey, out IInboxHandler handler); + bool TryGetMessage(Runtime.GrainId senderId, System.Guid messageId, out DurableEnvelope envelope); + } + + [Alias("IDurableInboxExtension")] + public partial interface IDurableInboxExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(DurableEnvelope envelope, System.Threading.CancellationToken cancellationToken = default); + } + + public partial interface IDurableMessagingDiagnostics + { + System.Collections.Generic.IReadOnlyList InboxDeadLetters { get; } + + System.Collections.Generic.IReadOnlyList OutboxDeadLetters { get; } + + bool RemoveInboxDeadLetter(Runtime.GrainId senderId, System.Guid messageId); + bool RemoveOutboxDeadLetter(System.Guid messageId); + } + + public partial interface IDurableOutbox + { + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + void Send(DurableEnvelope envelope); + bool TryGetMessage(System.Guid messageId, out DurableEnvelope envelope); + } + + public partial interface IInboxHandler + { + bool CanHandle(IInboxHandlerContext context); + System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public partial interface IInboxHandlerContext + { + DurableEnvelope Envelope { get; } + + Runtime.GrainId GrainId { get; } + + IDurableOutbox Outbox { get; } + + DurableEnvelopeBuilder CreateEnvelope(); + void Send(DurableEnvelope envelope); + } + + public partial interface IInboxHandler : IInboxHandler + { + System.Threading.Tasks.ValueTask HandleAsync(TMessage message, IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + bool IInboxHandler.CanHandle(IInboxHandlerContext context); + System.Threading.Tasks.ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public abstract partial class RouteKeyHandler : IInboxHandler + { + protected RouteKeyHandler(string routeKey) { } + + protected string RouteKey { get { throw null; } } + + public bool CanHandle(IInboxHandlerContext context) { throw null; } + + protected abstract System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken) { throw null; } + } + + public abstract partial class RoutePrefixHandler : IInboxHandler + { + protected RoutePrefixHandler(string prefix) { } + + protected string Prefix { get { throw null; } } + + public bool CanHandle(IInboxHandlerContext context) { throw null; } + + protected string? GetRouteSuffix(string? routeKey) { throw null; } + + protected abstract System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask IInboxHandler.HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken) { throw null; } + } +} + +namespace Orleans.DurableMessaging.Configuration +{ + public partial class DurableInboxOptions + { + public System.TimeSpan BackpressureRetryDelay { get { throw null; } set { } } + + public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + + public int InboxBatchSize { get { throw null; } set { } } + + public int MaxCapacity { get { throw null; } set { } } + + public int MaxDeliveryAttempts { get { throw null; } set { } } + + public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } + + public int MaxProcessingAttempts { get { throw null; } set { } } + + public int OutboxBatchSize { get { throw null; } set { } } + + public void Validate() { } + } +} + +namespace Orleans.Hosting +{ + public static partial class DurableMessagingExtensions + { + public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddDurableMessaging(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action? configureOptions = null) { throw null; } + + public static ISiloBuilder AddDurableMessaging(this ISiloBuilder builder, System.Action? configureOptions = null) { throw null; } + } +} + +namespace OrleansCodeGen.Orleans.DurableMessaging +{ + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeliveryResult : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DeliveryResult(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.DurableMessaging.DeliveryResult instance) { } + + public global::Orleans.DurableMessaging.DeliveryResult ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.DurableMessaging.DeliveryResult instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.DurableMessaging.DeliveryResult value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DurableEnvelope : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DurableEnvelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.DurableMessaging.DurableEnvelope instance) { } + + public global::Orleans.DurableMessaging.DurableEnvelope ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.DurableMessaging.DurableEnvelope instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.DurableMessaging.DurableEnvelope value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DurableEnvelopeData : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_DurableEnvelopeData(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.DurableMessaging.DurableEnvelopeData instance) { } + + public global::Orleans.DurableMessaging.DurableEnvelopeData ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.DurableMessaging.DurableEnvelopeData instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.DurableMessaging.DurableEnvelopeData value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_HierarchicalKey : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_HierarchicalKey(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.DurableMessaging.HierarchicalKey instance) { } + + public global::Orleans.DurableMessaging.HierarchicalKey ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.DurableMessaging.HierarchicalKey instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.DurableMessaging.HierarchicalKey value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B instance) { } + + public Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_DurableEnvelope : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_DurableEnvelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.DurableMessaging.DurableEnvelope DeepCopy(global::Orleans.DurableMessaging.DurableEnvelope original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_DurableEnvelopeData : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_DurableEnvelopeData(global::Orleans.Serialization.Activators.IActivator _activator) { } + + public global::Orleans.DurableMessaging.DurableEnvelopeData DeepCopy(global::Orleans.DurableMessaging.DurableEnvelopeData original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B DeepCopy(Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.DurableMessaging.IDurableInboxExtension), typeof(global::Orleans.DurableMessaging.IDurableInboxExtension), "DeliverAsync" })] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.DurableMessaging.IDurableInboxExtension), typeof(global::Orleans.DurableMessaging.IDurableInboxExtension), "03DB806B" })] + public sealed partial class Invokable_IDurableInboxExtension_GrainReference_Ext_03DB806B : global::Orleans.Runtime.Request + { + public global::Orleans.DurableMessaging.DurableEnvelope arg0; + public System.Threading.CancellationToken arg1; + public override bool IsCancellable { get { throw null; } } + + public override void Dispose() { } + + public override string GetActivityName() { throw null; } + + public override object GetArgument(int index) { throw null; } + + public override int GetArgumentCount() { throw null; } + + public override System.Threading.CancellationToken GetCancellationToken() { throw null; } + + public override string GetInterfaceName() { throw null; } + + public override System.Type GetInterfaceType() { throw null; } + + public override System.Reflection.MethodInfo GetMethod() { throw null; } + + public override string GetMethodName() { throw null; } + + public override object GetTarget() { throw null; } + + protected override System.Threading.Tasks.ValueTask InvokeInner() { throw null; } + + public override void SetArgument(int index, object value) { } + + public override void SetTarget(global::Orleans.Serialization.Invocation.ITargetHolder holder) { } + + public override bool TryCancel() { throw null; } + } +} \ No newline at end of file diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index e7fcd7aa32d..2787afff58c 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,12 +219,18 @@ public partial interface IJournaledState void Reset(JournalStreamWriter writer); } + public partial interface IJournaledGrainParticipant + { + void Initialize(); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken); + void RegisterObserver(IJournaledStateObserver observer) { throw null; } void RegisterState(string name, IJournaledState state); System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); @@ -237,6 +243,21 @@ public partial interface IJournaledStateManagerFactory IJournaledStateManager Create(JournalId journalId); } + public partial interface IJournaledStateObserver + { + void OnDeleteCompleted() { } + System.Threading.Tasks.ValueTask OnDeletePreparingAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + void OnDeleteRequested() { } + void OnRecoveryCompleted(); + void OnRecoveryRequested() { } + void OnRecoveryStarted() { } + void OnWriteCompleted(); + System.Threading.Tasks.ValueTask OnWriteFinalizingAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + System.Threading.Tasks.ValueTask OnWritePreparingAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + void OnWriteRequested() { } + void OnWriteStarted(); + } + public partial interface IJournalFormat { string FormatKey { get; } diff --git a/test/Orleans.Core.Tests/DurableJobs/DurableJobReceiverExtensionTests.cs b/test/Orleans.Core.Tests/DurableJobs/DurableJobReceiverExtensionTests.cs index b90243b459d..4c3efde8ff0 100644 --- a/test/Orleans.Core.Tests/DurableJobs/DurableJobReceiverExtensionTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/DurableJobReceiverExtensionTests.cs @@ -236,6 +236,31 @@ public async Task HandleDurableJobAsync_WhenExecutionGenerationChanges_StartsNew await handler.Received(2).ExecuteJobAsync(Arg.Any(), Arg.Any()); } + [Fact] + public async Task HandleDurableJobAsync_SameJobIdInDifferentShardsStartsIndependentExecutions() + { + var firstExecution = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondExecution = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var invocationCount = 0; + var handler = Substitute.For(); + handler.ExecuteJobAsync(Arg.Any(), Arg.Any()) + .Returns(_ => Interlocked.Increment(ref invocationCount) == 1 + ? firstExecution.Task + : secondExecution.Task); + var extension = CreateExtension(handler, TimeSpan.FromMinutes(1)); + var firstContext = CreateJobContext("run-1", jobId: "stable-job", shardId: "shard-1"); + var secondContext = CreateJobContext("run-2", jobId: "stable-job", shardId: "shard-2"); + + var first = extension.HandleDurableJobAsync(firstContext, CancellationToken.None); + var second = extension.HandleDurableJobAsync(secondContext, CancellationToken.None); + + await handler.Received(2).ExecuteJobAsync(Arg.Any(), Arg.Any()); + firstExecution.SetResult(true); + secondExecution.SetResult(true); + Assert.Equal(DurableJobRunStatus.Completed, (await first).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await second).Status); + } + [Fact] public async Task HandleDurableJobAsync_FeatureNameMatchTakesPrecedenceOverGrainHandler() { @@ -569,7 +594,8 @@ private static IJobRunContext CreateJobContext( string jobId = "job-1", int dequeueCount = 1, long executionGeneration = 0, - string? jobName = null) + string? jobName = null, + string shardId = "shard-1") { var context = Substitute.For(); context.RunId.Returns(runId); @@ -580,7 +606,7 @@ private static IJobRunContext CreateJobContext( Name = jobName ?? jobId, DueTime = DateTimeOffset.UtcNow, TargetGrainId = GrainId.Create("test", "grain-1"), - ShardId = "shard-1", + ShardId = shardId, ExecutionGeneration = executionGeneration }); diff --git a/test/Orleans.Core.Tests/DurableJobs/DurableJobsExtensionsTests.cs b/test/Orleans.Core.Tests/DurableJobs/DurableJobsExtensionsTests.cs index 86da3d5804e..19ddb6dae1d 100644 --- a/test/Orleans.Core.Tests/DurableJobs/DurableJobsExtensionsTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/DurableJobsExtensionsTests.cs @@ -8,7 +8,6 @@ namespace NonSilo.Tests.ScheduledJobs; -[TestCategory("BVT"), TestCategory("DurableJobs")] [TestSuite("BVT")] [TestProvider("None")] [TestArea("DurableJobs")] diff --git a/test/Orleans.Core.Tests/DurableJobs/JobShardTests.cs b/test/Orleans.Core.Tests/DurableJobs/JobShardTests.cs index 2ee818bd5d2..0048e5c9f41 100644 --- a/test/Orleans.Core.Tests/DurableJobs/JobShardTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/JobShardTests.cs @@ -66,6 +66,67 @@ public async Task TryScheduleJobAsync_ForwardsCompleteJobForPersistence() Assert.Equal(traceState, persistedJob.TraceState); } + [Fact] + public async Task TryScheduleJobAsync_StableJobIdIsIdempotent() + { + var dueTime = DateTimeOffset.UtcNow.AddMinutes(1); + var shard = new TestJobShard(dueTime.AddMinutes(-1), dueTime.AddMinutes(1)); + var request = new ScheduleJobRequest + { + JobId = "stable-job", + Target = GrainId.Create("test", "job"), + JobName = "job", + DueTime = dueTime + }; + + var first = await shard.TryScheduleJobAsync(request, CancellationToken.None); + var second = await shard.TryScheduleJobAsync( + new ScheduleJobRequest + { + JobId = request.JobId, + Target = request.Target, + JobName = request.JobName, + DueTime = dueTime.AddSeconds(1), + Metadata = request.Metadata, + TraceParent = "different-attempt-trace" + }, + CancellationToken.None); + + Assert.Same(first, second); + Assert.Equal("stable-job", first!.Id); + Assert.Equal(1, shard.PersistAddCount); + Assert.Equal(1, await shard.GetJobCountAsync()); + } + + [Fact] + public async Task TryScheduleJobAsync_ConflictingStableJobIdIsRejected() + { + var dueTime = DateTimeOffset.UtcNow.AddMinutes(1); + var shard = new TestJobShard(dueTime.AddMinutes(-1), dueTime.AddMinutes(1)); + await shard.TryScheduleJobAsync( + new ScheduleJobRequest + { + JobId = "stable-job", + Target = GrainId.Create("test", "job"), + JobName = "job", + DueTime = dueTime + }, + CancellationToken.None); + + var exception = await Assert.ThrowsAsync( + () => shard.TryScheduleJobAsync( + new ScheduleJobRequest + { + JobId = "stable-job", + Target = GrainId.Create("test", "other"), + JobName = "job", + DueTime = dueTime + }, + CancellationToken.None)); + + Assert.Contains("different properties", exception.Message, StringComparison.Ordinal); + } + [Fact] public async Task RetryJobLaterAsync_ForwardsCompleteRunContextForPersistence() { @@ -209,6 +270,7 @@ private sealed class TestJobShard(DateTimeOffset startTime, DateTimeOffset endTi : JobShard("shard", startTime, endTime) { public DurableJob? PersistedJob { get; private set; } + public int PersistAddCount { get; private set; } public IJobRunContext? PersistedRetryContext { get; private set; } @@ -216,6 +278,7 @@ private sealed class TestJobShard(DateTimeOffset startTime, DateTimeOffset endTi protected override Task PersistAddJobAsync(DurableJob job, CancellationToken cancellationToken) { + PersistAddCount++; PersistedJob = job; return Task.CompletedTask; } diff --git a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs index 76b5d932452..a8d149bdfa3 100644 --- a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs @@ -541,6 +541,58 @@ public async Task ScheduleJobAsync_WhenShardStripingEnabled_DistributesJobsAcros secondRoundJobs.Select(static job => job.ShardId).OrderBy(static id => id).ToArray()); } + [Fact] + public void GetWritableShardStripe_StableJobIdIsDeterministicAcrossManagers() + { + var options = CreateOptions(); + options.ShardStripeCount = 8; + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var first = new LocalDurableJobManager.TestAccessor( + CreateManager(new TestJobShardManager(), timeProvider, options)); + var second = new LocalDurableJobManager.TestAccessor( + CreateManager(new TestJobShardManager(), timeProvider, options)); + var request = new ScheduleJobRequest + { + JobId = "stable-job", + Target = GrainId.Create("test", "target"), + JobName = "job", + DueTime = timeProvider.GetUtcNow() + }; + + Assert.Equal(first.GetWritableShardStripe(request), second.GetWritableShardStripe(request)); + } + + [Fact] + public async Task ScheduleJobAsync_TraceEnrichmentPreservesStableJobId() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var options = CreateOptions(); + var shardManager = new TestJobShardManager(); + var manager = CreateManager(shardManager, timeProvider, options); + var dueTime = timeProvider.GetUtcNow().AddMinutes(1); + var shardKey = new DateTimeOffset( + (dueTime.UtcTicks / options.ShardDuration.Ticks) * options.ShardDuration.Ticks, + TimeSpan.Zero); + var shard = new SchedulingShard("trace-shard", shardKey, shardKey.Add(options.ShardDuration)); + new LocalDurableJobManager.TestAccessor(manager).AddWritableShard(shardKey, shard); + using var activity = new System.Diagnostics.Activity("schedule").SetIdFormat( + System.Diagnostics.ActivityIdFormat.W3C); + activity.Start(); + + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + JobId = "stable-job", + Target = GrainId.Create("test", "target"), + JobName = "job", + DueTime = dueTime + }, + CancellationToken.None); + + Assert.Equal("stable-job", shard.LastRequest.JobId); + Assert.NotNull(shard.LastRequest.TraceParent); + } + [Fact] public async Task ExpiredJournaledShard_DrainsUnregistersAndDeletesStorage() { @@ -1246,6 +1298,8 @@ private sealed class SchedulingShard(string id, DateTimeOffset start, DateTimeOf public bool IsAddingCompleted => false; + public ScheduleJobRequest LastRequest { get; private set; } + public IAsyncEnumerable ConsumeDurableJobsAsync() => ConsumeAsync(); public ValueTask GetJobCountAsync() => ValueTask.FromResult(0); @@ -1263,9 +1317,10 @@ public Task TryStartAttemptAsync(IJobRunContext jobCon public Task TryScheduleJobAsync(ScheduleJobRequest request, CancellationToken cancellationToken) { + LastRequest = request; return Task.FromResult(new() { - Id = Guid.NewGuid().ToString(), + Id = request.JobId ?? Guid.NewGuid().ToString(), Name = request.JobName, DueTime = request.DueTime, TargetGrainId = request.Target, diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/DurableJobFeatureHandlerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/DurableJobFeatureHandlerTests.cs index 710c6406b30..d7c8d76884a 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/DurableJobFeatureHandlerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/DurableJobFeatureHandlerTests.cs @@ -4,7 +4,6 @@ namespace Tester.DurableJobs; -[TestCategory("BVT"), TestCategory("DurableJobs")] [TestSuite("BVT")] [TestProvider("None")] [TestArea("DurableJobs")] diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs new file mode 100644 index 00000000000..060ed5a342d --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -0,0 +1,182 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using NSubstitute; +using Orleans.DurableMessaging.Configuration; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class DeliveryAndOptionsContractTests +{ + [Fact] + public void RetentionTimeArithmetic_HandlesMaximumDurationsWithoutOverflow() + { + var timeType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableMessagingTime", + throwOnError: true)!; + var isExpired = timeType.GetMethod( + "IsExpired", + BindingFlags.Static | BindingFlags.Public)!; + var addClamped = timeType.GetMethod( + "AddClamped", + BindingFlags.Static | BindingFlags.Public)!; + var timestamp = DateTimeOffset.MaxValue - TimeSpan.FromTicks(1); + + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, timestamp, TimeSpan.MaxValue])!); + Assert.Equal( + DateTimeOffset.MaxValue, + (DateTimeOffset)addClamped.Invoke(null, [timestamp, TimeSpan.MaxValue])!); + } + + [Fact] + public void DeliveryResult_EachFactory_PreservesStatusAndPayload() + { + var routeMissing = DeliveryResult.RouteNotFound("orders/missing"); + var deadLettered = DeliveryResult.DeadLettered("poison body"); + + Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); + Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); + Assert.Equal(DeliveryStatus.Backpressured, DeliveryResult.Backpressured().Status); + Assert.Equal(DeliveryStatus.RouteNotFound, routeMissing.Status); + Assert.Equal("No handler for route 'orders/missing'", routeMissing.Message); + Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); + Assert.Equal("poison body", deadLettered.Message); + } + + [Fact] + public void DeliveryStatus_AllValues_HaveStableDistinctValues() + { + Assert.Equal( + [ + DeliveryStatus.Accepted, + DeliveryStatus.Duplicate, + DeliveryStatus.Backpressured, + DeliveryStatus.RouteNotFound, + DeliveryStatus.DeadLettered + ], + Enum.GetValues()); + Assert.Equal([0, 1, 2, 3, 6], Enum.GetValues().Select(static value => (int)value)); + } + + [Fact] + public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() + { + var options = new DurableInboxOptions(); + + options.Validate(); + + Assert.Equal(1000, options.MaxCapacity); + Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); + Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); + Assert.Equal(5, options.MaxProcessingAttempts); + Assert.Equal(100, options.MaxDeliveryAttempts); + Assert.Equal(32, options.InboxBatchSize); + Assert.Equal(32, options.OutboxBatchSize); + } + + [Fact] + public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() + { + var invalidCases = new (string Parameter, Action Mutate)[] + { + (nameof(DurableInboxOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(DurableInboxOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(DurableInboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), + (nameof(DurableInboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(DurableInboxOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(DurableInboxOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(DurableInboxOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(DurableInboxOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(DurableInboxOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + }; + + foreach (var (parameter, mutate) in invalidCases) + { + var options = new DurableInboxOptions(); + mutate(options); + var exception = Assert.Throws(options.Validate); + Assert.Equal(parameter, exception.ParamName); + } + } + + [Fact] + public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() + { + foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) + { + var options = new DurableInboxOptions + { + DeduplicationWindow = TimeSpan.FromHours(2), + MaxOutboxRetryAge = retryAge, + }; + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(DurableInboxOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); + } + } + + [Fact] + public async Task InboxLifecycleStart_ObservesPreCanceledLifecycleToken() + { + var extensionType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableInboxExtension", + throwOnError: true)!; + var extension = (ILifecycleObserver)RuntimeHelpers.GetUninitializedObject(extensionType); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => extension.OnStart(cancellation.Token)); + } + + [Fact] + public async Task InboxLifecycleStart_CancellationInterruptsBlockedResume() + { + var extensionType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableInboxExtension", + throwOnError: true)!; + var extension = (ILifecycleObserver)RuntimeHelpers.GetUninitializedObject(extensionType); + var grainContext = Substitute.For(); + grainContext.GrainInstance.Returns(new object()); + extensionType.GetField("_grainContext", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(extension, grainContext); + extensionType.GetField("_gate", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(extension, new SemaphoreSlim(0, 1)); + extensionType.GetField("_metricsActive", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(extension, 1); + using var cancellation = new CancellationTokenSource(); + + var start = extension.OnStart(cancellation.Token); + Assert.False(start.IsCompleted); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => start); + } + + [Fact] + public void PhysicalJobId_EncodesGrainTypeAndKeyBoundaries() + { + var ownershipType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableMessagingJobOwnership", + throwOnError: true)!; + var createJobId = ownershipType.GetMethod( + "CreateJobId", + BindingFlags.Static | BindingFlags.Public)!; + + var first = (string)createJobId.Invoke( + null, + ["job", GrainId.Create("a/b", "c"), "epoch:1"])!; + var second = (string)createJobId.Invoke( + null, + ["job", GrainId.Create("a", "b/c"), "epoch:1"])!; + + Assert.NotEqual(first, second); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/DurableEnvelopeContractTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/DurableEnvelopeContractTests.cs new file mode 100644 index 00000000000..cc5fe81514d --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/DurableEnvelopeContractTests.cs @@ -0,0 +1,183 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class DurableEnvelopeContractTests : IDisposable +{ + private readonly ServiceProvider _services; + private readonly SerializerSessionPool _sessions; + + public DurableEnvelopeContractTests() + { + var services = new ServiceCollection(); + services.AddSerializer(); + _services = services.BuildServiceProvider(); + _sessions = _services.GetRequiredService(); + } + + [Fact] + public void EnvelopeBuilder_Complete_RoundTripsAllEnvelopeFieldsIncludingGeneralReplyTo() + { + var sender = GrainId.Create("sender", "17"); + var receiver = GrainId.Create("receiver", "23"); + var replyTo = GrainId.Create("audit", "general-reply"); + var correlation = HierarchicalKey.Create("orders/2026/42"); + var before = DateTimeOffset.UtcNow; + + var envelope = new DurableEnvelopeBuilder(_sessions, sender) + .WithContextValue("tenant", "northwind") + .To(receiver, "orders/submit") + .WithReplyTo(replyTo) + .WithCorrelationKey(correlation) + .WithBody(new TestMessage(42, "ship")) + .WithContextValue("attempt", 3) + .Build(); + + Assert.NotEqual(Guid.Empty, envelope.MessageId); + Assert.Equal(sender, envelope.SenderId); + Assert.Equal(receiver, envelope.ReceiverId); + Assert.Equal("orders/submit", envelope.RouteKey); + Assert.Equal(correlation, envelope.CorrelationKey); + Assert.Equal(replyTo, envelope.ReplyTo); + Assert.InRange(envelope.CreatedAt, before, DateTimeOffset.UtcNow); + Assert.True(envelope.Data.TryGetBody(out var body)); + Assert.Equal(new TestMessage(42, "ship"), body); + Assert.True(envelope.Data.TryGetContextValue("tenant", out var tenant)); + Assert.Equal("northwind", tenant); + Assert.True(envelope.Data.TryGetContextValue("attempt", out var attempt)); + Assert.Equal(3, attempt); + Assert.Equal(["attempt", "tenant"], envelope.Data.ContextKeys.Order()); + + } + + [Fact] + public void EnvelopeBuilder_MissingRequiredField_ThrowsWithoutProducingEnvelope() + { + var sender = GrainId.Create("sender", "missing"); + var receiver = GrainId.Create("receiver", "missing"); + + var missingBody = new DurableEnvelopeBuilder(_sessions, sender).To(receiver, "route"); + var missingTarget = new DurableEnvelopeBuilder(_sessions, sender).WithBody("payload"); + + var bodyError = Assert.Throws(() => missingBody.Build()); + var targetError = Assert.Throws(() => missingTarget.Build()); + Assert.Contains("body", bodyError.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("route", targetError.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void EnvelopeBuilder_DuplicateBodyAndContextKey_RejectsAmbiguousMetadata() + { + var builder = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "duplicate")) + .To(GrainId.Create("receiver", "duplicate"), "route") + .WithBody("first") + .WithContextValue("trace", "one"); + + var bodyError = Assert.Throws(() => builder.WithBody("second")); + var contextError = Assert.Throws(() => builder.WithContextValue("trace", "two")); + Assert.Contains("already", bodyError.Message, StringComparison.Ordinal); + Assert.Contains("trace", contextError.Message, StringComparison.Ordinal); + } + + [Fact] + public void EnvelopeBuilder_AfterBuild_RejectsEveryMutation() + { + var builder = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "built")) + .To(GrainId.Create("receiver", "built"), "route") + .WithBody("payload"); + _ = builder.Build(); + + Assert.Throws(() => builder.To(GrainId.Create("receiver", "other"), "other")); + Assert.Throws(() => builder.WithBody("other")); + Assert.Throws(() => builder.WithContextValue("key", "value")); + Assert.Throws(() => builder.WithCorrelationKey("correlation")); + Assert.Throws(() => builder.WithReplyTo(GrainId.Create("reply", "other"))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void EnvelopeBuilder_InvalidRoute_Throws(string? route) + { + var builder = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "route")); + Assert.ThrowsAny(() => builder.To(GrainId.Create("receiver", "route"), route!)); + } + + [Fact] + public void EnvelopeBuilder_DefaultDestination_ThrowsAtConfiguration() + { + var targetBuilder = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "target")); + var replyBuilder = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "reply")); + + var targetException = Assert.Throws(() => targetBuilder.To(default, "route")); + var replyException = Assert.Throws(() => replyBuilder.WithReplyTo(default)); + + Assert.Equal("target", targetException.ParamName); + Assert.Equal("replyTo", replyException.ParamName); + } + + [Fact] + public void EnvelopeData_WrongBodyOrContextType_FailsWithoutCorruptingOtherValues() + { + var envelope = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "types")) + .To(GrainId.Create("receiver", "types"), "types") + .WithContextValue("count", 7) + .WithContextValue("label", "valid") + .WithBody(new TestMessage(9, "body")) + .Build(); + + Assert.False(envelope.Data.TryGetBody(out var wrongBody)); + Assert.Null(wrongBody); + Assert.False(envelope.Data.TryGetContextValue("count", out var wrongContext)); + Assert.Equal(Guid.Empty, wrongContext); + Assert.True(envelope.Data.TryGetBody(out var body)); + Assert.NotNull(body); + Assert.Equal(9, body.Id); + Assert.True(envelope.Data.TryGetContextValue("label", out var label)); + Assert.Equal("valid", label); + Assert.True(envelope.Data.GetBodyBytes().Length > 0); + Assert.True(envelope.Data.TryGetContextBytes("count", out var rawCount)); + Assert.True(rawCount.Length > 0); + Assert.False(envelope.Data.TryGetContextBytes("absent", out var absent)); + Assert.True(absent.IsEmpty); + + } + + [Fact] + public void EnvelopeSerializer_RoundTripsCorrelationReplyBodyAndContext() + { + var serializer = _services.GetRequiredService>(); + var correlation = HierarchicalKey.Create("orders/42/dispatch"); + var replyTo = GrainId.Create("audit", "42"); + var envelope = new DurableEnvelopeBuilder(_sessions, GrainId.Create("sender", "42")) + .To(GrainId.Create("receiver", "42"), "orders/dispatch") + .WithCorrelationKey(correlation) + .WithReplyTo(replyTo) + .WithContextValue("tenant", "northwind") + .WithBody(new TestMessage(42, "dispatch")) + .Build(); + + var copy = serializer.Deserialize(serializer.SerializeToArray(envelope)); + + Assert.Equal(envelope.MessageId, copy.MessageId); + Assert.Equal(correlation, copy.CorrelationKey); + Assert.Equal(replyTo, copy.ReplyTo); + Assert.True(copy.Data.TryGetBody(out var body)); + Assert.Equal(new TestMessage(42, "dispatch"), body); + Assert.True(copy.Data.TryGetContextValue("tenant", out var tenant)); + Assert.Equal("northwind", tenant); + } + + public void Dispose() => _services.Dispose(); + + [GenerateSerializer, Immutable] + public sealed record TestMessage([property: Id(0)] int Id, [property: Id(1)] string Action); +} diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/DurableMessagingPumpResultsTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/DurableMessagingPumpResultsTests.cs new file mode 100644 index 00000000000..c6e5a86006b --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/DurableMessagingPumpResultsTests.cs @@ -0,0 +1,181 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.Time.Testing; +using Orleans.DurableJobs; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class DurableMessagingPumpResultsTests +{ + [Fact] + public void ConcurrentStarts_SuppressDuplicateExecution() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + var starts = new bool[64]; + + Parallel.For(0, starts.Length, index => starts[index] = results.TryStart(key, out _)); + + Assert.Equal(1, starts.Count(static started => started)); + } + + [Fact] + public void CanceledWaitingExecution_BecomesTakeableAndDoesNotRun() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + cancellation.Cancel(); + + Assert.False(results.TryBegin(execution)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Null(result); + Assert.IsType(exception); + } + + [Fact] + public void CompletedResultWithoutSecondPoll_Expires() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 16); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + results.Complete(execution); + + clock.Advance(TimeSpan.FromMinutes(2)); + _ = results.TryStart(results.CreateKey("job", "other", "run"), out _); + + Assert.False(results.TryTake(key, out _, out _)); + } + + [Fact] + public void RetainedEntries_AreBounded() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromHours(1), TimeSpan.FromHours(1), 4); + + for (var index = 0; index < 100; index++) + { + Assert.True(results.TryStart(results.CreateKey("job", index.ToString(), "run"), out _)); + } + + Assert.InRange(results.Count, 0, 4); + } + + [Fact] + public void RunningExecution_IsNotExpiredOrDuplicated() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 4); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + + clock.Advance(TimeSpan.FromHours(1)); + + Assert.False(results.TryStart(key, out _)); + results.Complete(execution); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void CapacityExhaustedByRunningExecution_RejectsNewStartWithoutLosingRunningExecution() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 1); + var runningKey = results.CreateKey("job", "running", "run"); + var rejectedKey = results.CreateKey("job", "rejected", "run"); + Assert.True(results.TryStart(runningKey, out var execution)); + Assert.True(results.TryBegin(execution)); + + Assert.False(results.TryStart(rejectedKey, out _)); + results.Complete(execution); + + Assert.True(results.TryTake(runningKey, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void DifferentRunId_DoesNotObserveOlderResult() + { + var results = new PumpResults(); + var firstKey = results.CreateKey("job", "id", "run-1"); + var secondKey = results.CreateKey("job", "id", "run-2"); + Assert.True(results.TryStart(firstKey, out var firstExecution)); + Assert.True(results.TryBegin(firstExecution)); + results.Complete(firstExecution); + + Assert.True(results.TryStart(secondKey, out var secondExecution)); + Assert.False(results.TryTake(secondKey, out _, out _)); + Assert.True(results.TryTake(firstKey, out var firstResult, out _)); + Assert.Same(DurableJobRunResult.Completed, firstResult); + Assert.True(results.TryBegin(secondExecution)); + } + + private sealed class PumpResults + { + private static readonly Assembly Assembly = typeof(IDurableOutbox).Assembly; + private static readonly Type ResultsType = Assembly.GetType("Orleans.DurableMessaging.DurableMessagingPumpResults", throwOnError: true)!; + private static readonly Type KeyType = Assembly.GetType("Orleans.DurableMessaging.DurableMessagingPumpExecutionKey", throwOnError: true)!; + private readonly object _instance; + + public PumpResults() + { + _instance = Activator.CreateInstance(ResultsType, nonPublic: true)!; + } + + public PumpResults(TimeProvider timeProvider, TimeSpan completedRetention, TimeSpan abandonedRetention, int maxEntries) + { + _instance = Activator.CreateInstance( + ResultsType, + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [timeProvider, completedRetention, abandonedRetention, maxEntries], + culture: null)!; + } + + public int Count => ((IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!).Count; + + public object CreateKey(string jobName, string jobId, string runId) => + Activator.CreateInstance(KeyType, [jobName, jobId, runId])!; + + public bool TryStart(object key, out object execution) => + TryStartWithCancellation(key, out execution, TestContext.Current.CancellationToken); + + public bool TryStartWithCancellation( + object key, + out object execution, + CancellationToken cancellationToken) + { + object?[] arguments = [key, cancellationToken, null]; + var result = (bool)ResultsType.GetMethod("TryStart")!.Invoke(_instance, arguments)!; + execution = arguments[2]!; + return result; + } + + public bool TryBegin(object execution) => + (bool)ResultsType.GetMethod("TryBegin")!.Invoke(_instance, [execution])!; + + public void Complete(object execution) => + ResultsType.GetMethod("Complete")!.Invoke(_instance, [execution, DurableJobRunResult.Completed]); + + public bool TryTake(object key, out DurableJobRunResult? result, out Exception? exception) + { + object?[] arguments = [key, null, null]; + var taken = (bool)ResultsType.GetMethod("TryTake")!.Invoke(_instance, arguments)!; + result = arguments[1] as DurableJobRunResult; + exception = arguments[2] as Exception; + return taken; + } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/DurableOutboxDeliveryBatchTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/DurableOutboxDeliveryBatchTests.cs new file mode 100644 index 00000000000..26352fbb60c --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/DurableOutboxDeliveryBatchTests.cs @@ -0,0 +1,1167 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.DurableMessaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Timers; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class DurableOutboxDeliveryBatchTests +{ + [Fact] + public async Task RecoveryAndLifecycleStart_CoalesceReplacementScheduling() + { + var timerRegistry = Substitute.For(); + var fixture = new OutboxFixture(hasDurableMessage: true, timerRegistry: timerRegistry); + + fixture.Manager.NotifyRecoveryCompleted(); + await fixture.StartAsync(); + + Assert.Single( + timerRegistry.ReceivedCalls(), + static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + } + + [Fact] + public void DeleteCompletion_RotatesOwnershipEpoch() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + var before = fixture.GetOwnershipEpoch(); + + fixture.Manager.NotifyDeleteCompleted(); + + Assert.NotEqual(before, fixture.GetOwnershipEpoch()); + } + + [Fact] + public void RecoveryCompletion_RotatesOwnershipEpoch() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + var before = fixture.GetOwnershipEpoch(); + + fixture.Manager.NotifyRecoveryCompleted(); + + Assert.NotEqual(before, fixture.GetOwnershipEpoch()); + } + + [Fact] + public async Task EnsureJobTimerCompletion_ReleasesCoalescingSlot() + { + var timerRegistry = Substitute.For(); + var fixture = new OutboxFixture( + hasDurableMessage: true, + timerRegistry: timerRegistry, + jobManager: new RecordingJobManager()); + + fixture.Manager.NotifyRecoveryCompleted(); + await fixture.RunRegisteredTimerAsync(); + fixture.Manager.NotifyRecoveryCompleted(); + + Assert.Equal( + 2, + timerRegistry.ReceivedCalls().Count(static call => call.GetMethodInfo().Name == "RegisterGrainTimer")); + } + + [Fact] + public async Task OwnershipPersistenceRetry_DoesNotScheduleDuplicateJob() + { + var jobManager = new RecordingJobManager(); + var fixture = new OutboxFixture( + hasDurableMessage: true, + jobManager: jobManager, + backpressureRetryDelay: TimeSpan.FromMilliseconds(1)); + fixture.Manager.FailNextWrite(new IOException("Injected ownership write failure.")); + + await fixture.EnsureJobScheduledAsync(replaceExisting: true, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(1, jobManager.AttemptCount); + Assert.Equal(2, fixture.Manager.WriteCount); + } + + [Fact] + public async Task ReplacementOwnershipPersistenceRetry_KeepsBothJobsPollingUntilCommit() + { + const string recoveredOwnershipId = "recovered:1"; + var clock = new FakeTimeProvider(); + var jobManager = new RecordingJobManager(); + var fixture = new OutboxFixture( + hasDurableMessage: true, + jobManager: jobManager, + jobTimeProvider: clock, + backpressureRetryDelay: TimeSpan.FromMinutes(1), + durableJobId: recoveredOwnershipId); + fixture.Manager.FailNextWrite(new IOException("Injected replacement ownership write failure.")); + + var scheduling = fixture.EnsureJobScheduledAsync( + replaceExisting: true, + TestContext.Current.CancellationToken); + await jobManager.WaitForAttemptCountAsync(1); + await fixture.Manager.WaitForWriteCountAsync(1); + var replacementOwnershipId = Assert.IsType(fixture.JobId.Value); + + Assert.NotEqual(recoveredOwnershipId, replacementOwnershipId); + Assert.True((await fixture.ExecuteJobAsync(recoveredOwnershipId)).IsInProgress); + Assert.True((await fixture.ExecuteJobAsync(replacementOwnershipId)).IsInProgress); + + clock.Advance(TimeSpan.FromMinutes(1)); + await scheduling.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(1, jobManager.AttemptCount); + Assert.Equal(2, fixture.Manager.WriteCount); + Assert.Equal( + DurableJobRunStatus.Completed, + (await fixture.ExecuteJobAsync(recoveredOwnershipId)).Status); + } + + [Fact] + public async Task ConcurrentWriteBeforeReplacementSchedule_KeepsDurableOwnership() + { + const string recoveredOwnershipId = "recovered:1"; + var jobManager = new BlockingJobManager(); + var fixture = new OutboxFixture( + hasDurableMessage: true, + jobManager: jobManager, + durableJobId: recoveredOwnershipId); + + var scheduling = fixture.EnsureJobScheduledAsync( + replaceExisting: true, + TestContext.Current.CancellationToken); + await jobManager.WaitUntilScheduledAsync(); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var replacementOwnershipId = Assert.IsType(jobManager.OwnershipId); + + Assert.Equal(recoveredOwnershipId, fixture.JobId.Value); + Assert.Equal(recoveredOwnershipId, fixture.GetDurableOwnershipId()); + Assert.True((await fixture.ExecuteJobAsync(recoveredOwnershipId)).IsInProgress); + Assert.True((await fixture.ExecuteJobAsync(replacementOwnershipId)).IsInProgress); + + jobManager.Release(); + await scheduling.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.NotEqual(recoveredOwnershipId, fixture.JobId.Value); + Assert.Equal(fixture.JobId.Value, fixture.GetDurableOwnershipId()); + } + + [Fact] + public async Task DeliveryWriteWithInterleavedReplacement_KeepsDurableOwnerPolling() + { + const string recoveredOwnershipId = "recovered:1"; + const string replacementOwnershipId = "replacement:2"; + var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.Accepted()), + durableJobId: recoveredOwnershipId); + fixture.Manager.InterleaveNextWrite(() => fixture.JobId.Value = replacementOwnershipId); + + var result = await fixture.ExecuteJobCoreAsync(recoveredOwnershipId, hasStableOwnership: true); + + Assert.True(result.IsInProgress); + Assert.Equal(recoveredOwnershipId, fixture.GetDurableOwnershipId()); + Assert.Equal(replacementOwnershipId, fixture.JobId.Value); + } + + [Fact] + public async Task SchedulingRetry_UsesConfiguredTimeProvider() + { + var clock = new FakeTimeProvider(); + var jobManager = new RecordingJobManager(alwaysFail: true); + var fixture = new OutboxFixture( + hasDurableMessage: true, + jobManager: jobManager, + jobTimeProvider: clock, + backpressureRetryDelay: TimeSpan.FromMinutes(1)); + using var cancellation = new CancellationTokenSource(); + + var scheduling = fixture.EnsureJobScheduledAsync(replaceExisting: true, cancellation.Token); + await jobManager.WaitForAttemptCountAsync(1); + Assert.False(scheduling.IsCompleted); + + clock.Advance(TimeSpan.FromMinutes(1)); + await jobManager.WaitForAttemptCountAsync(2); + cancellation.Cancel(); + await scheduling; + + Assert.Equal(2, jobManager.AttemptCount); + } + + [Fact] + public async Task RecoveryDuringDelivery_DiscardsStaleFailureResult() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fixture = new OutboxFixture( + async _ => + { + entered.TrySetResult(); + await release.Task; + throw new IOException("Stale delivery failure."); + }, + maxDeliveryAttempts: 1); + + var delivery = fixture.DeliverAsync(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + fixture.SimulateRecoveryWithoutMessage(); + release.TrySetResult(); + await delivery; + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + } + + [Fact] + public async Task DuplicateAfterCommitRemainsDeliverableAndUnfenced() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + fixture.Send(fixture.Envelope); + await fixture.CommitAsync(); + var duplicate = fixture.CreateEquivalentEnvelope(); + + fixture.Send(duplicate); + + Assert.NotSame(fixture.Envelope.Data, duplicate.Data); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + } + + [Fact] + public async Task DurableDuplicateFollowedByNoOpWriteDoesNotFenceDelivery() + { + var fixture = new OutboxFixture(); + + fixture.Send(fixture.CreateEquivalentEnvelope()); + await fixture.CommitAsync(); + + Assert.Equal(0, fixture.Manager.WriteCompletedCount); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Fact] + public async Task DuplicateBeforeFirstCommitRemainsPendingUntilOwningCommit() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + + fixture.Send(fixture.Envelope); + fixture.Send(fixture.CreateEquivalentEnvelope()); + + Assert.Single(fixture.Messages); + Assert.Equal(1, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(0, fixture.DeliveryCount); + + await fixture.CommitAsync(); + + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConflictingDuplicateFailsWithoutMutatingDurableOrProvisionalMessage(bool commitFirst) + { + var fixture = new OutboxFixture(hasDurableMessage: false); + fixture.Send(fixture.Envelope); + if (commitFirst) + { + await fixture.CommitAsync(); + } + + var exception = Assert.Throws( + () => fixture.Send(fixture.CreateConflictingEnvelope())); + + Assert.Contains(fixture.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.True(fixture.Messages.TryGetValue(fixture.MessageId, out var stored)); + Assert.Equal(fixture.Envelope.RouteKey, stored.RouteKey); + Assert.Equal(commitFirst ? 0 : 1, fixture.PendingMessageCount); + } + + [Fact] + public async Task RollbackClearsFenceAndAllowsMessageToBeSentAgain() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + fixture.Send(fixture.Envelope); + + await fixture.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken); + + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.PendingMessageCount); + fixture.Send(fixture.CreateEquivalentEnvelope()); + Assert.Equal(1, fixture.PendingMessageCount); + + await fixture.CommitAsync(); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Fact] + public async Task MessageAddedAfterWriteCaptureRemainsFencedForNextCommit() + { + var fixture = new OutboxFixture(hasDurableMessage: false); + fixture.Send(fixture.Envelope); + var reentrantEnvelope = fixture.CreateEnvelope(Guid.NewGuid()); + + fixture.Manager.CommitWithInterleavedMutation(() => fixture.Send(reentrantEnvelope)); + + Assert.False(fixture.IsPending(fixture.MessageId)); + Assert.True(fixture.IsPending(reentrantEnvelope.MessageId)); + await fixture.CommitAsync(); + Assert.Equal(0, fixture.PendingMessageCount); + } + + [Fact] + public async Task CancellationAfterSuccessfulDeliveryRevertsRemoval() + { + CancellationTokenSource? cancellation = null; + var fixture = new OutboxFixture( + _ => + { + cancellation!.Cancel(); + return ValueTask.FromResult(DeliveryResult.Accepted()); + }); + fixture.ActivateMetrics(); + Assert.Equal(1, fixture.GetOutboxDepth()); + + for (var attempt = 0; attempt < 2; attempt++) + { + using var currentCancellation = new CancellationTokenSource(); + cancellation = currentCancellation; + var exception = await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(currentCancellation.Token)); + + Assert.Equal(currentCancellation.Token, exception.CancellationToken); + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.True(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.GetOutboxDepth()); + } + + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(2, fixture.Manager.RevertCount); + } + + [Fact] + public async Task CancellationAfterDeadLetterMutationRevertsFailureState() + { + using var cancellation = new CancellationTokenSource(); + var fixture = new OutboxFixture( + _ => + { + cancellation.Cancel(); + return ValueTask.FromResult(DeliveryResult.RouteNotFound("missing")); + }, + maxDeliveryAttempts: 1); + + await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(cancellation.Token)); + + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(1, fixture.Manager.RevertCount); + } + + [Fact] + public async Task LaterStateWriteDoesNotCommitRevertedDeliveryMutation() + { + using var cancellation = new CancellationTokenSource(); + var fixture = new OutboxFixture( + _ => + { + cancellation.Cancel(); + return ValueTask.FromResult(DeliveryResult.Accepted()); + }); + + await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(cancellation.Token)); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken); + + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.True(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + } + + [Fact] + public async Task RevertFailureIsSurfaced() + { + var writeFailure = new IOException("Injected delivery batch write failure."); + var revertFailure = new InvalidOperationException("Injected fenced recovery failure."); + var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.Accepted()), + writeException: writeFailure, + revertException: revertFailure); + + var exception = await Assert.ThrowsAsync( + () => fixture.DeliverAsync()); + + Assert.Same(revertFailure, exception); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(1, fixture.Manager.RevertCount); + } + + [Fact] + public async Task WriteFailureRevertsProvisionalMutationsAndPreservesError() + { + var writeFailure = new IOException("Injected delivery batch write failure."); + var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.Accepted()), + writeException: writeFailure); + fixture.ActivateMetrics(); + Assert.Equal(1, fixture.GetOutboxDepth()); + + var exception = await Assert.ThrowsAsync( + () => fixture.DeliverAsync()); + + Assert.Same(writeFailure, exception); + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.True(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.Manager.RevertCount); + Assert.Equal(1, fixture.GetOutboxDepth()); + } + + [Fact] + public void ConstructionWithoutObserverSupport_FailsWithSpecificDiagnostic() + { + var exception = Assert.Throws( + () => new OutboxFixture(supportsObservers: false)); + + var diagnostic = Assert.IsType(exception.InnerException); + Assert.Contains("Durable messaging", diagnostic.Message, StringComparison.Ordinal); + Assert.Contains("IJournaledStateManager.RegisterObserver", diagnostic.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task NormalDeliveryCommitsBatchOnce() + { + var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.Accepted())); + + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.RevertCount); + } + + [Fact] + public async Task ReceiverDeadLetterRemovesMessageWithoutCreatingSenderDeadLetter() + { + var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.DeadLettered("Receiver rejected the payload."))); + + await fixture.DeliverAsync(); + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.RevertCount); + } + + [Fact] + public async Task CancellationBeforeMutationDoesNotRevert() + { + using var cancellation = new CancellationTokenSource(); + var fixture = new OutboxFixture( + token => + { + cancellation.Cancel(); + return ValueTask.FromException(new OperationCanceledException(token)); + }); + + await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(cancellation.Token)); + + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.RevertCount); + } + + [Fact] + public async Task UnrelatedDeliveryCancellation_IsPersistedAsTerminalFailure() + { + var fixture = new OutboxFixture( + _ => ValueTask.FromException( + new OperationCanceledException("Receiver canceled its operation.")), + maxDeliveryAttempts: 1); + + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(1, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.RevertCount); + } + + private sealed class OutboxFixture + { + private static readonly Assembly DurableMessagingAssembly = typeof(IDurableOutbox).Assembly; + private readonly IDurableOutbox _outbox; + private readonly MethodInfo _deliverMethod; + private readonly Instrument _outboxDepthInstrument; + private readonly FieldInfo _pendingMessageIdsField; + + public OutboxFixture( + Func>? deliver = null, + int maxDeliveryAttempts = 3, + Exception? writeException = null, + Exception? revertException = null, + bool hasDurableMessage = true, + bool supportsObservers = true, + ILocalDurableJobManager? jobManager = null, + ITimerRegistry? timerRegistry = null, + TimeProvider? jobTimeProvider = null, + TimeSpan? backpressureRetryDelay = null, + string? durableJobId = null) + { + MessageId = Guid.NewGuid(); + SenderId = GrainId.Create("sender", "1"); + ReceiverId = GrainId.Create("receiver", "1"); + Envelope = CreateEnvelope(MessageId); + + MessageStates = CreateInternalDictionary("Orleans.DurableMessaging.OutboxMessageState"); + DeadLetters = CreateInternalDictionary("Orleans.DurableMessaging.OutboxDeadLetter"); + JobId = new TestDurableValue { Value = durableJobId }; + CompletedJobId = new TestDurableValue(); + JobSequence = new TestDurableValue(); + if (hasDurableMessage) + { + Messages.Add(MessageId, Envelope); + var messageState = CreateInternal("Orleans.DurableMessaging.OutboxMessageState"); + messageState.GetType().GetProperty("EnqueuedAt")!.SetValue(messageState, TimeProvider.System.GetUtcNow()); + MessageStates.Add(MessageId, messageState); + } + + Manager = new TestStateManager( + [Messages, MessageStates, DeadLetters, JobId, CompletedJobId, JobSequence], + writeException, + revertException, + supportsObservers); + + var delivery = deliver ?? (_ => ValueTask.FromResult(DeliveryResult.Accepted())); + var inbox = Substitute.For(); + inbox.DeliverAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + DeliveryCount++; + return delivery(call.ArgAt(1)); + }); + var grainFactory = Substitute.For(); + grainFactory.GetGrain(Arg.Any()).Returns(inbox); + var grainContext = Substitute.For(); + grainContext.GrainId.Returns(SenderId); + grainContext.GrainInstance.Returns(new object()); + grainContext.ObservableLifecycle.Returns(Substitute.For()); + + TimerRegistry = timerRegistry ?? Substitute.For(); + JobManager = jobManager ?? Substitute.For(); + var outboxType = GetInternalType("Orleans.DurableMessaging.DurableOutbox"); + var instrumentsType = GetInternalType("Orleans.DurableMessaging.DurableMessagingInstruments"); + var instruments = instrumentsType + .GetMethod("CreateForDirectConstruction", BindingFlags.Static | BindingFlags.NonPublic)! + .Invoke(null, null)!; + var depthTracker = instrumentsType + .GetField("_outboxDepth", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(instruments)!; + _outboxDepthInstrument = (Instrument)depthTracker.GetType() + .GetField("_gauge", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(depthTracker)!; + var pumpResults = Activator.CreateInstance( + GetInternalType("Orleans.DurableMessaging.DurableMessagingPumpResults"), + nonPublic: true)!; + var logger = Activator.CreateInstance(typeof(NullLogger<>).MakeGenericType(outboxType))!; + + _outbox = (IDurableOutbox)Activator.CreateInstance( + outboxType, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + [ + Manager, + Messages, + grainFactory, + grainContext, + TimerRegistry, + logger, + instruments, + MessageStates.Instance, + DeadLetters.Instance, + JobId, + CompletedJobId, + JobSequence, + JobManager, + Substitute.For(), + pumpResults, + jobTimeProvider ?? TimeProvider.System, + Options.Create( + new DurableInboxOptions + { + BackpressureRetryDelay = backpressureRetryDelay ?? TimeSpan.FromMilliseconds(1), + MaxOutboxRetryAge = TimeSpan.FromMinutes(5), + MaxDeliveryAttempts = maxDeliveryAttempts, + OutboxBatchSize = 8 + }) + ], + culture: null)!; + if (durableJobId is not null) + { + outboxType.GetField("_durableOwnershipId", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(_outbox, durableJobId); + outboxType.GetField("_recoveryCompleted", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(_outbox, true); + } + + _deliverMethod = outboxType.GetMethod("DeliverPendingMessagesAsync")!; + _pendingMessageIdsField = outboxType.GetField("_pendingMessageIds", BindingFlags.Instance | BindingFlags.NonPublic)!; + } + + public Guid MessageId { get; } + public GrainId SenderId { get; } + public GrainId ReceiverId { get; } + public DurableEnvelope Envelope { get; } + public int DeliveryCount { get; private set; } + public TestDurableDictionary Messages { get; } = new(); + public UntypedDurableDictionary MessageStates { get; } + public UntypedDurableDictionary DeadLetters { get; } + public TestStateManager Manager { get; } + public ILocalDurableJobManager JobManager { get; } + public ITimerRegistry TimerRegistry { get; } + public TestDurableValue JobId { get; } + public TestDurableValue CompletedJobId { get; } + public TestDurableValue JobSequence { get; } + public int PendingMessageCount => GetPendingMessageIds().Count; + + public Task DeliverAsync() => + DeliverWithCancellationAsync(TestContext.Current.CancellationToken); + + public Task DeliverWithCancellationAsync(CancellationToken cancellationToken) => + (Task)_deliverMethod.Invoke(_outbox, [cancellationToken])!; + + public Task EnsureJobScheduledAsync(bool replaceExisting, CancellationToken cancellationToken) => + (Task)_outbox.GetType() + .GetMethod("EnsureJobScheduledAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(_outbox, [replaceExisting, cancellationToken])!; + + public ValueTask ExecuteJobAsync(string ownershipId) + { + var context = Substitute.For(); + context.Job.Returns(new DurableJob + { + Id = $"job-{ownershipId}", + Name = "orleans.messaging.outbox-flush", + DueTime = DateTimeOffset.UnixEpoch, + TargetGrainId = SenderId, + ShardId = "test", + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["orleans.messaging.ownership-id"] = ownershipId + } + }); + context.RunId.Returns($"run-{ownershipId}"); + context.DequeueCount.Returns(1); + return (ValueTask)_outbox.GetType() + .GetMethod("ExecuteJobAsync", BindingFlags.Instance | BindingFlags.Public)! + .Invoke(_outbox, [context, TestContext.Current.CancellationToken])!; + } + + public ValueTask ExecuteJobCoreAsync( + string ownershipId, + bool hasStableOwnership) => + (ValueTask)_outbox.GetType() + .GetMethod("ExecuteJobCoreAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(_outbox, [ownershipId, hasStableOwnership, TestContext.Current.CancellationToken])!; + + public string? GetDurableOwnershipId() => + (string?)_outbox.GetType() + .GetField("_durableOwnershipId", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_outbox); + + public Task StartAsync() => + StartWithCancellationAsync(TestContext.Current.CancellationToken); + + public Task StartWithCancellationAsync(CancellationToken cancellationToken) => + ((ILifecycleObserver)_outbox).OnStart(cancellationToken); + + public void SimulateRecoveryWithoutMessage() + { + Messages.Remove(MessageId); + MessageStates.Remove(MessageId); + Manager.NotifyRecoveryStarted(); + Manager.NotifyRecoveryCompleted(); + } + + public string GetOwnershipEpoch() => + (string)_outbox.GetType() + .GetField("_ownershipEpoch", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_outbox)!; + + public async Task RunRegisteredTimerAsync() + { + var call = Assert.Single( + TimerRegistry.ReceivedCalls(), + static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + var arguments = call.GetArguments(); + var callback = (Delegate)arguments[1]!; + await (Task)callback.DynamicInvoke(arguments[2], TestContext.Current.CancellationToken)!; + } + + public void ActivateMetrics() => + _outbox.GetType() + .GetMethod("EnsureMetricsActive", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(_outbox, null); + + public long GetOutboxDepth() + { + long? result = null; + using var listener = new MeterListener + { + InstrumentPublished = (instrument, meterListener) => + { + if (ReferenceEquals(instrument, _outboxDepthInstrument)) + { + meterListener.EnableMeasurementEvents(instrument); + } + } + }; + listener.SetMeasurementEventCallback( + (instrument, measurement, tags, state) => result = measurement); + listener.Start(); + listener.RecordObservableInstruments(); + return result ?? throw new InvalidOperationException("The outbox depth gauge did not report a value."); + } + + public void Send(DurableEnvelope envelope) => _outbox.Send(envelope); + + public ValueTask CommitAsync() => Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + public bool IsPending(Guid messageId) => GetPendingMessageIds().Contains(messageId); + + public DurableEnvelope CreateEquivalentEnvelope() => CreateEnvelope(MessageId); + + public DurableEnvelope CreateConflictingEnvelope() => CreateEnvelope(MessageId, routeKey: "conflict"); + + public DurableEnvelope CreateEnvelope(Guid messageId, string routeKey = "test") => new() + { + MessageId = messageId, + SenderId = SenderId, + ReceiverId = ReceiverId, + RouteKey = routeKey, + CorrelationKey = HierarchicalKey.Create("operation/1"), + ReplyTo = SenderId, + Data = CreateEnvelopeData(), + CreatedAt = DateTimeOffset.UnixEpoch + }; + + private HashSet GetPendingMessageIds() => + (HashSet)_pendingMessageIdsField.GetValue(_outbox)!; + + private static DurableEnvelopeData CreateEnvelopeData() + { + var result = (DurableEnvelopeData)RuntimeHelpers.GetUninitializedObject(typeof(DurableEnvelopeData)); + typeof(DurableEnvelopeData) + .GetMethod("Initialize", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke( + result, + [ + new byte[] { 1, 2 }, + (Offset: 0, Length: 1), + new Dictionary + { + ["trace"] = (1, 1) + } + ]); + return result; + } + + private static UntypedDurableDictionary CreateInternalDictionary(string valueTypeName) + { + var dictionary = Activator.CreateInstance( + typeof(TestDurableDictionary<,>).MakeGenericType(typeof(Guid), GetInternalType(valueTypeName)))!; + return new UntypedDurableDictionary(dictionary); + } + + private static Type GetInternalType(string typeName) => + DurableMessagingAssembly.GetType(typeName, throwOnError: true)!; + + private static object CreateInternal(string typeName) => + Activator.CreateInstance(GetInternalType(typeName), nonPublic: true)!; + } + + private interface ITestDurableState + { + long Version { get; } + object Capture(); + void Restore(object snapshot); + } + + private sealed class UntypedDurableDictionary(object instance) : ITestDurableState + { + private readonly Type _type = instance.GetType(); + + public object Instance { get; } = instance; + public int Count => (int)_type.GetProperty("Count")!.GetValue(Instance)!; + public long Version => ((ITestDurableState)Instance).Version; + + public void Add(Guid key, object value) => + _type.GetMethod("Add", [typeof(Guid), value.GetType()])!.Invoke(Instance, [key, value]); + + public bool ContainsKey(Guid key) => + (bool)_type.GetMethod("ContainsKey")!.Invoke(Instance, [key])!; + + public bool Remove(Guid key) => + (bool)_type.GetMethod("Remove", [typeof(Guid)])!.Invoke(Instance, [key])!; + + public T GetProperty(Guid key, string propertyName) + { + var value = _type.GetProperty("Item")!.GetValue(Instance, [key])!; + return (T)value.GetType().GetProperty(propertyName)!.GetValue(value)!; + } + + public object Capture() => ((ITestDurableState)Instance).Capture(); + public void Restore(object snapshot) => ((ITestDurableState)Instance).Restore(snapshot); + } + + private sealed class TestStateManager( + IEnumerable states, + Exception? writeException, + Exception? revertException, + bool supportsObservers) : IJournaledStateManager + { + private readonly ITestDurableState[] _states = states.ToArray(); + private object[] _durableSnapshots = states.Select(static state => state.Capture()).ToArray(); + private long[] _durableVersions = states.Select(static state => state.Version).ToArray(); + private IJournaledStateObserver? _observer; + + public int WriteCount { get; private set; } + public int WriteCompletedCount { get; private set; } + public int RevertCount { get; private set; } + private Exception? _nextWriteException = writeException; + private Action? _nextWriteMutation; + private readonly SemaphoreSlim _writes = new(0); + + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + public void RegisterState(string name, IJournaledState state) { } + public void RegisterObserver(IJournaledStateObserver observer) + { + if (!supportsObservers) + { + throw new NotSupportedException(); + } + + _observer = observer; + } + + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + WriteCount++; + _writes.Release(); + if (_nextWriteException is { } exception) + { + _nextWriteException = null; + return ValueTask.FromException(exception); + } + + _observer?.OnWriteStarted(); + var currentVersions = _states.Select(static state => state.Version).ToArray(); + if (currentVersions.SequenceEqual(_durableVersions)) + { + return default; + } + + var durableSnapshots = _states.Select(static state => state.Capture()).ToArray(); + var mutation = _nextWriteMutation; + _nextWriteMutation = null; + mutation?.Invoke(); + _durableSnapshots = durableSnapshots; + _durableVersions = currentVersions; + _observer?.OnWriteCompleted(); + WriteCompletedCount++; + return default; + } + + public void FailNextWrite(Exception exception) => _nextWriteException = exception; + + public void InterleaveNextWrite(Action mutation) => _nextWriteMutation = mutation; + + public async Task WaitForWriteCountAsync(int expected) + { + while (WriteCount < expected) + { + await _writes.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + } + + public void NotifyRecoveryCompleted() => _observer?.OnRecoveryCompleted(); + + public void NotifyRecoveryStarted() => _observer?.OnRecoveryStarted(); + + public void NotifyDeleteCompleted() => _observer?.OnDeleteCompleted(); + + public void CommitWithInterleavedMutation(Action mutation) + { + WriteCount++; + _observer?.OnWriteStarted(); + var committedSnapshots = _states.Select(static state => state.Capture()).ToArray(); + var committedVersions = _states.Select(static state => state.Version).ToArray(); + mutation(); + _durableSnapshots = committedSnapshots; + _durableVersions = committedVersions; + _observer?.OnWriteCompleted(); + WriteCompletedCount++; + } + + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + RevertCount++; + if (revertException is not null) + { + return ValueTask.FromException(revertException); + } + + for (var i = 0; i < _states.Length; i++) + { + _states[i].Restore(_durableSnapshots[i]); + } + + _durableVersions = _states.Select(static state => state.Version).ToArray(); + _observer?.OnRecoveryCompleted(); + return default; + } + + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } + + private sealed class RecordingJobManager(bool alwaysFail = false) : ILocalDurableJobManager + { + private readonly SemaphoreSlim _attempted = new(0); + private int _attemptCount; + + public int AttemptCount => Volatile.Read(ref _attemptCount); + + public Task ScheduleJobAsync(ScheduleJobRequest request, CancellationToken cancellationToken) + { + var count = Interlocked.Increment(ref _attemptCount); + _attempted.Release(); + if (alwaysFail) + { + return Task.FromException(new IOException($"Injected scheduling failure {count}.")); + } + + return Task.FromResult(new DurableJob + { + Id = $"job-{count}", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + ShardId = "test", + Metadata = request.Metadata + }); + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + Task.FromResult(true); + + public async Task WaitForAttemptCountAsync(int expected) + { + while (AttemptCount < expected) + { + await _attempted.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + } + + private sealed class BlockingJobManager : ILocalDurableJobManager + { + private readonly TaskCompletionSource _attempted = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public string? OwnershipId { get; private set; } + + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + OwnershipId = request.Metadata!["orleans.messaging.ownership-id"]; + _attempted.TrySetResult(); + await _release.Task.WaitAsync(cancellationToken); + return new DurableJob + { + Id = "replacement", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + ShardId = "test", + Metadata = request.Metadata + }; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + Task.FromResult(true); + + public Task WaitUntilScheduledAsync() => + _attempted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + public void Release() => _release.TrySetResult(); + } + + private sealed class TestDurableValue : IDurableValue, ITestDurableState + { + private T? _value; + + public T? Value + { + get => _value; + set + { + _value = value; + Version++; + } + } + + public long Version { get; private set; } + + public object Capture() => new Snapshot(_value); + + public void Restore(object snapshot) + { + _value = ((Snapshot)snapshot).Value; + Version++; + } + + private sealed record Snapshot(T? Value); + } + + private sealed class TestDurableDictionary + : IDurableDictionary, ITestDurableState + where TKey : notnull + { + private readonly Dictionary _items = []; + private long _version; + + public TValue this[TKey key] + { + get => _items[key]; + set + { + _items[key] = value; + _version++; + } + } + + public ICollection Keys => _items.Keys; + public ICollection Values => _items.Values; + public int Count => _items.Count; + public bool IsReadOnly => false; + public long Version => _version; + + public void Add(TKey key, TValue value) + { + _items.Add(key, value); + _version++; + } + + public void Add(KeyValuePair item) + { + ((ICollection>)_items).Add(item); + _version++; + } + + public void Clear() + { + if (_items.Count > 0) + { + _items.Clear(); + _version++; + } + } + + public bool Contains(KeyValuePair item) => ((ICollection>)_items).Contains(item); + public bool ContainsKey(TKey key) => _items.ContainsKey(key); + public void CopyTo(KeyValuePair[] array, int arrayIndex) => + ((ICollection>)_items).CopyTo(array, arrayIndex); + public IEnumerator> GetEnumerator() => _items.GetEnumerator(); + public bool Remove(TKey key) + { + if (!_items.Remove(key)) + { + return false; + } + + _version++; + return true; + } + + public bool Remove(KeyValuePair item) + { + if (!((ICollection>)_items).Remove(item)) + { + return false; + } + + _version++; + return true; + } + + public bool TryGetValue(TKey key, out TValue value) => _items.TryGetValue(key, out value!); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + object ITestDurableState.Capture() => + _items.ToDictionary(static pair => pair.Key, pair => CloneValue(pair.Value)); + + void ITestDurableState.Restore(object snapshot) + { + _items.Clear(); + foreach (var (key, value) in (Dictionary)snapshot) + { + _items.Add(key, CloneValue(value)); + } + + _version++; + } + + private static TValue CloneValue(TValue value) + { + if (value is null || typeof(TValue).IsValueType || value is string) + { + return value; + } + + return (TValue)typeof(object) + .GetMethod("MemberwiseClone", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(value, null)!; + } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/HandlerRoutingContractTests.cs new file mode 100644 index 00000000000..561c1a61a9d --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -0,0 +1,179 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class HandlerRoutingContractTests : IDisposable +{ + private readonly ServiceProvider _services; + private readonly SerializerSessionPool _sessions; + + public HandlerRoutingContractTests() + { + var services = new ServiceCollection(); + services.AddSerializer(); + _services = services.BuildServiceProvider(); + _sessions = _services.GetRequiredService(); + } + + [Theory] + [InlineData("orders/submit", true)] + [InlineData("orders/Submit", false)] + [InlineData("orders/submit/child", false)] + [InlineData("orders", false)] + public void RouteKeyHandler_MatchesOnlyExactOrdinalRoute(string route, bool expected) + { + var handler = new ExactHandler("orders/submit"); + using var context = CreateContext(route); + + Assert.Equal(expected, handler.CanHandle(context)); + Assert.Equal("orders/submit", handler.ExposedRoute); + } + + [Theory] + [InlineData("orders/new", true, "new")] + [InlineData("orders/new/priority", true, "new/priority")] + [InlineData("orders", false, null)] + [InlineData("orders-archive/new", false, null)] + [InlineData("Orders/new", false, null)] + public void RoutePrefixHandler_NormalizesBoundaryAndExtractsSuffix(string route, bool expected, string? suffix) + { + var handler = new PrefixHandler("orders"); + using var context = CreateContext(route); + + Assert.Equal(expected, handler.CanHandle(context)); + Assert.Equal("orders/", handler.ExposedPrefix); + Assert.Equal(suffix, handler.Suffix(route)); + } + + [Fact] + public void RoutePrefixHandler_NullPrefix_ThrowsArgumentNullException() + { + var exception = Assert.Throws(() => new PrefixHandler(null!)); + + Assert.Equal("prefix", exception.ParamName); + } + + [Theory] + [InlineData("workflow/order-42", true)] + [InlineData("workflow/order-42/payment", true)] + [InlineData("workflow/order-420", false)] + [InlineData("workflow/other", false)] + public void CorrelationHandler_MatchesOnlyConfiguredHierarchy(string correlation, bool expected) + { + var root = HierarchicalKey.Create("workflow/order-42"); + var handler = new HierarchyHandler(root); + using var context = CreateContext("events", HierarchicalKey.Create(correlation)); + + Assert.Equal(expected, handler.CanHandle(context)); + Assert.Equal(root, handler.ExposedCorrelation); + } + + [Fact] + public async Task TypedHandler_DeserializesExpectedTypeAndInvokesTypedMethod() + { + var handler = new TypedHandler(); + using var context = CreateContext("typed", body: new RoutedMessage(81, "typed-body")); + + Assert.True(((IInboxHandler)handler).CanHandle(context)); + await ((IInboxHandler)handler).HandleAsync(context, CancellationToken.None); + + Assert.Equal(1, handler.CallCount); + Assert.Equal(new RoutedMessage(81, "typed-body"), handler.Message); + Assert.Same(context, handler.Context); + } + + [Fact] + public async Task TypedHandler_WrongType_ThrowsBeforeInvokingTypedMethod() + { + var handler = new TypedHandler(); + using var context = CreateContext("typed", body: "not-a-routed-message"); + + var exception = await Assert.ThrowsAsync( + async () => await ((IInboxHandler)handler).HandleAsync(context, CancellationToken.None)); + + Assert.Contains(typeof(RoutedMessage).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains("typed", exception.Message, StringComparison.Ordinal); + Assert.Equal(0, handler.CallCount); + Assert.Null(handler.Message); + } + + public void Dispose() => _services.Dispose(); + + private TestContext CreateContext(string route, HierarchicalKey? correlation = null, object? body = null) + { + var sender = GrainId.Create("sender", Guid.NewGuid().ToString("N")); + var receiver = GrainId.Create("receiver", Guid.NewGuid().ToString("N")); + var builder = new DurableEnvelopeBuilder(_sessions, sender).To(receiver, route); + var envelope = body switch + { + RoutedMessage message => builder.WithBody(message).WithCorrelationKeyIfPresent(correlation).Build(), + string text => builder.WithBody(text).WithCorrelationKeyIfPresent(correlation).Build(), + _ => builder.WithBody(0).WithCorrelationKeyIfPresent(correlation).Build(), + }; + return new TestContext(envelope, receiver); + } + + private sealed class ExactHandler(string route) : RouteKeyHandler(route) + { + public string ExposedRoute => RouteKey; + protected override ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => default; + } + + private sealed class PrefixHandler(string prefix) : RoutePrefixHandler(prefix) + { + public string ExposedPrefix => Prefix; + public string? Suffix(string? route) => GetRouteSuffix(route); + protected override ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => default; + } + + private sealed class HierarchyHandler(HierarchicalKey correlation) : CorrelationHandler(correlation) + { + public HierarchicalKey ExposedCorrelation => CorrelationKey; + protected override ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => default; + } + + private sealed class TypedHandler : IInboxHandler + { + public int CallCount { get; private set; } + public RoutedMessage? Message { get; private set; } + public IInboxHandlerContext? Context { get; private set; } + + public ValueTask HandleAsync(RoutedMessage? message, IInboxHandlerContext context, CancellationToken cancellationToken) + { + CallCount++; + Message = message ?? throw new InvalidOperationException("A routed message is required."); + Context = context; + return default; + } + } + + private sealed class TestContext(DurableEnvelope envelope, GrainId grainId) : IInboxHandlerContext, IDisposable + { + public DurableEnvelope Envelope { get; } = envelope; + public GrainId GrainId { get; } = grainId; + public IDurableOutbox Outbox => throw new NotSupportedException(); + public DurableEnvelopeBuilder CreateEnvelope() => throw new NotSupportedException(); + public void Send(DurableEnvelope envelope) => throw new NotSupportedException(); + public void Dispose() + { + } + } + + [GenerateSerializer, Immutable] + public sealed record RoutedMessage([property: Id(0)] int Id, [property: Id(1)] string Value); +} + +internal static class DurableEnvelopeBuilderTestExtensions +{ + public static DurableEnvelopeBuilder WithCorrelationKeyIfPresent( + this DurableEnvelopeBuilder builder, + HierarchicalKey? correlation) => + correlation is null ? builder : builder.WithCorrelationKey(correlation); +} diff --git a/test/Orleans.DurableMessaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.DurableMessaging.Tests/Contracts/HierarchicalKeyTests.cs new file mode 100644 index 00000000000..2539363f320 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -0,0 +1,423 @@ +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Contracts; + +/// +/// Tests for hierarchical message correlation keys. +/// +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public class HierarchicalKeyTests +{ + [Fact] + public void SerializationContract_PreservesDraftTypeAlias() + { + var alias = Assert.Single(typeof(HierarchicalKey).GetCustomAttributes(inherit: false).OfType()); + + Assert.Equal("Orleans.HierarchicalKey", alias.Alias); + } + + [Fact] + public void Create_WithValidString_CreatesKey() + { + var key = HierarchicalKey.Create("foo"); + Assert.NotNull(key); + Assert.Equal("foo", key.ToString()); + } + + [Fact] + public void Create_WithMultipleSegments_CreatesKey() + { + var key = HierarchicalKey.Create("foo/bar/baz"); + Assert.NotNull(key); + Assert.Equal("foo/bar/baz", key.ToString()); + } + + [Fact] + public void Create_WithEmptyString_ThrowsArgumentException() + { + Assert.Throws(() => HierarchicalKey.Create("")); + } + + [Fact] + public void Create_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Create(null!)); + } + + [Fact] + public void Create_WithEmptySegment_ThrowsArgumentException() + { + Assert.Throws(() => HierarchicalKey.Create("foo//bar")); + } + + [Fact] + public void Create_WithTrailingSeparator_ThrowsArgumentException() + { + Assert.Throws(() => HierarchicalKey.Create("foo/")); + } + + [Fact] + public void Create_WithLeadingSeparator_ThrowsArgumentException() + { + Assert.Throws(() => HierarchicalKey.Create("/foo")); + } + + [Fact] + public void Parse_WithValidString_CreatesKey() + { + var key = HierarchicalKey.Parse("foo/bar", null); + Assert.NotNull(key); + Assert.Equal("foo/bar", key.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("foo//bar")] + [InlineData("foo\\")] + public void Parse_WithInvalidString_ThrowsFormatException(string value) + { + Assert.Throws(() => HierarchicalKey.Parse(value, null)); + Assert.Throws(() => HierarchicalKey.Parse(value.AsSpan(), null)); + } + + [Fact] + public void Parse_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Parse(null!, null)); + } + + [Fact] + public void TryParse_WithValidString_ReturnsTrue() + { + var result = HierarchicalKey.TryParse("foo/bar", null, out var key); + Assert.True(result); + Assert.NotNull(key); + Assert.Equal("foo/bar", key.ToString()); + } + + [Fact] + public void TryParse_WithInvalidString_ReturnsFalse() + { + var result = HierarchicalKey.TryParse("", null, out var key); + Assert.False(result); + Assert.Null(key); + } + + [Fact] + public void TryParse_WithEmptySegment_ReturnsFalse() + { + var result = HierarchicalKey.TryParse("foo//bar", null, out var key); + Assert.False(result); + Assert.Null(key); + } + + [Fact] + public void CreateChildKey_CreatesChildKey() + { + var parent = HierarchicalKey.Create("foo"); + var child = parent.CreateChildKey("bar"); + Assert.Equal("foo/bar", child.ToString()); + } + + [Fact] + public void CreateChildKey_WithMultipleSegments_CreatesChildKey() + { + var parent = HierarchicalKey.Create("foo"); + var child = parent.CreateChildKey("bar/baz"); + Assert.Equal("foo/bar/baz", child.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("bar//baz")] + [InlineData("bar\\")] + public void CreateChildKey_WithInvalidSegments_Throws(string value) + { + var parent = HierarchicalKey.Create("foo"); + + Assert.Throws(() => parent.CreateChildKey(value)); + Assert.Throws(() => HierarchicalKey.Create(parent, value)); + } + + [Fact] + public void CreateEscapedChildKey_EscapesSegmentSeparators() + { + var parent = HierarchicalKey.Create("foo"); + var child = parent.CreateEscapedChildKey("bar/baz"); + Assert.Equal("foo/bar\\/baz", child.ToString()); + } + + [Fact] + public void CreateEscaped_WithSegmentSeparator_EscapesIt() + { + var key = HierarchicalKey.CreateEscaped("foo/bar"); + Assert.Equal("foo\\/bar", key.ToString()); + } + + [Fact] + public void CreateEscaped_WithExistingEscape_StillEscapesLaterSeparators() + { + var key = HierarchicalKey.CreateEscaped(@"foo\/bar/baz"); + + Assert.Equal(@"foo\/bar\/baz", key.ToString()); + } + + [Fact] + public void CreateEscaped_CopiesCallerOwnedMemory() + { + var characters = "foo".ToCharArray(); + var key = HierarchicalKey.CreateEscaped(parent: null, characters); + + characters[0] = 'b'; + + Assert.Equal("foo", key.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("foo\\")] + public void CreateEscaped_WithInvalidInput_Throws(string value) + { + Assert.Throws(() => HierarchicalKey.CreateEscaped(value)); + } + + [Fact] + public void GetParent_ReturnsParentKey() + { + var key = HierarchicalKey.Create("foo/bar/baz"); + var parent = key.GetParent(); + Assert.NotNull(parent); + Assert.Equal("foo/bar", parent.ToString()); + } + + [Fact] + public void GetParent_ForSingleSegment_ReturnsNull() + { + var key = HierarchicalKey.Create("foo"); + var parent = key.GetParent(); + Assert.Null(parent); + } + + [Fact] + public void IsParentOf_WithDirectChild_ReturnsTrue() + { + var parent = HierarchicalKey.Create("foo"); + var child = HierarchicalKey.Create("foo/bar"); + Assert.True(parent.IsParentOf(child)); + } + + [Fact] + public void IsParentOf_WithGrandchild_ReturnsFalse() + { + var parent = HierarchicalKey.Create("foo"); + var grandchild = HierarchicalKey.Create("foo/bar/baz"); + Assert.False(parent.IsParentOf(grandchild)); + } + + [Fact] + public void IsParentOf_WithSameKey_ReturnsFalse() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("foo/bar"); + Assert.False(key1.IsParentOf(key2)); + } + + [Fact] + public void IsParentOf_WithUnrelatedKey_ReturnsFalse() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("baz/qux"); + Assert.False(key1.IsParentOf(key2)); + } + + [Fact] + public void IsChildOf_WithDirectParent_ReturnsTrue() + { + var parent = HierarchicalKey.Create("foo"); + var child = HierarchicalKey.Create("foo/bar"); + Assert.True(child.IsChildOf(parent)); + } + + [Fact] + public void IsChildOf_WithGrandparent_ReturnsFalse() + { + var grandparent = HierarchicalKey.Create("foo"); + var grandchild = HierarchicalKey.Create("foo/bar/baz"); + Assert.False(grandchild.IsChildOf(grandparent)); + } + + [Fact] + public void IsAncestorOf_WithDirectChild_ReturnsTrue() + { + var parent = HierarchicalKey.Create("foo"); + var child = HierarchicalKey.Create("foo/bar"); + Assert.True(parent.IsAncestorOf(child)); + } + + [Fact] + public void IsAncestorOf_WithGrandchild_ReturnsTrue() + { + var grandparent = HierarchicalKey.Create("foo"); + var grandchild = HierarchicalKey.Create("foo/bar/baz"); + Assert.True(grandparent.IsAncestorOf(grandchild)); + } + + [Fact] + public void IsAncestorOf_WithSameKey_ReturnsTrue() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("foo/bar"); + Assert.True(key1.IsAncestorOf(key2)); + } + + [Fact] + public void IsAncestorOf_WithUnrelatedKey_ReturnsFalse() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("baz/qux"); + Assert.False(key1.IsAncestorOf(key2)); + } + + [Fact] + public void Equals_WithSameValue_ReturnsTrue() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("foo/bar"); + Assert.True(key1.Equals(key2)); + } + + [Fact] + public void Equals_WithDifferentValue_ReturnsFalse() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("foo/baz"); + Assert.False(key1.Equals(key2)); + } + + [Fact] + public void GetHashCode_WithSameValue_ReturnsSameHashCode() + { + var key1 = HierarchicalKey.Create("foo/bar"); + var key2 = HierarchicalKey.Create("foo/bar"); + Assert.Equal(key1.GetHashCode(), key2.GetHashCode()); + } + + [Fact] + public void GetHashCode_WithComposedKey_ReturnsSameHashCodeAsDirectKey() + { + var parent = HierarchicalKey.Create("foo"); + var child = parent.CreateChildKey("bar"); + var direct = HierarchicalKey.Create("foo/bar"); + Assert.Equal(child.GetHashCode(), direct.GetHashCode()); + } + + [Fact] + public void Length_ReturnsCorrectLength() + { + var key = HierarchicalKey.Create("foo/bar"); + Assert.Equal(7, key.Length); // "foo/bar" = 7 characters + } + + [Fact] + public void Length_WithEscapedCharacters_ReturnsCorrectLength() + { + var key = HierarchicalKey.CreateEscaped("foo/bar"); + Assert.Equal(8, key.Length); // "foo\/bar" = 8 characters + } + + [Fact] + public void EscapeCharacter_IsBackslash() + { + Assert.Equal('\\', HierarchicalKey.EscapeCharacter); + } + + [Fact] + public void SegmentSeparator_IsForwardSlash() + { + Assert.Equal('/', HierarchicalKey.SegmentSeparator); + } + + [Fact] + public void Create_WithEscapedSeparator_ParsesCorrectly() + { + var key = HierarchicalKey.Create("foo\\/bar"); + Assert.Equal("foo\\/bar", key.ToString()); + } + + [Fact] + public void Create_WithEscapedEscapeCharacter_ParsesCorrectly() + { + var key = HierarchicalKey.Create("foo\\\\bar"); + Assert.Equal("foo\\\\bar", key.ToString()); + } + + [Fact] + public void Create_WithInvalidEscapeSequence_ThrowsArgumentException() + { + // Escape character must be followed by either '/' or '\' + Assert.Throws(() => HierarchicalKey.Create("foo\\bar")); + } + + [Fact] + public void Create_WithIncompleteEscapeSequence_ThrowsArgumentException() + { + // Escape character at end of string is invalid + Assert.Throws(() => HierarchicalKey.Create("foo\\")); + } + + [Fact] + public void GetEnumerator_EnumeratesAllSegments() + { + var key = HierarchicalKey.Create("foo/bar/baz"); + var segments = new List(); + var enumerator = key.GetEnumerator(); + while (enumerator.MoveNext()) + { + segments.Add(enumerator.Current.ToString()); + } + Assert.Equal(new[] { "foo", "bar", "baz" }, segments); + } + + [Fact] + public void GetEnumerator_WithSingleSegment_EnumeratesOneSegment() + { + var key = HierarchicalKey.Create("foo"); + var segments = new List(); + var enumerator = key.GetEnumerator(); + while (enumerator.MoveNext()) + { + segments.Add(enumerator.Current.ToString()); + } + Assert.Equal(new[] { "foo" }, segments); + } + + [Fact] + public void GetEnumerator_WithEscapedSegment_EnumeratesEscapedSegment() + { + var key = HierarchicalKey.Create("foo/bar\\/baz/qux"); + var segments = new List(); + var enumerator = key.GetEnumerator(); + while (enumerator.MoveNext()) + { + segments.Add(enumerator.Current.ToString()); + } + Assert.Equal(new[] { "foo", "bar\\/baz", "qux" }, segments); + } + + [Fact] + public void CreateWithParent_CreatesKeyWithParent() + { + var parent = HierarchicalKey.Create("foo"); + var key = HierarchicalKey.Create(parent, "bar"); + Assert.Equal("foo/bar", key.ToString()); + } + + [Fact] + public void CreateWithNullParent_CreatesKeyWithoutParent() + { + var key = HierarchicalKey.Create(null, "bar"); + Assert.Equal("bar", key.ToString()); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.DurableMessaging.Tests/Functional/DedupeExpiryBehaviorTests.cs new file mode 100644 index 00000000000..853228178d5 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -0,0 +1,123 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableMessaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DedupeExpiryClusterCollection : ICollectionFixture +{ + public const string Name = "Durable messaging dedupe expiry cluster"; +} + +[Collection(DedupeExpiryClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class DedupeExpiryBehaviorTests(DedupeExpiryClusterFixture fixture) +{ + [Fact] + public async Task IdleReplay_AtDeduplicationBoundary_IsAcceptedWithoutCompactionTrigger() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var original = new DurableTestMessage(Guid.NewGuid(), 15, "expires"); + using var first = CreateEnvelope(receiver, original); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first.Value)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first.Value)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first.Value)).Status); + var state = await fixture.WaitForEffectCountAsync(receiver, 2); + + Assert.Equal(2, state.Effects.Single(effect => effect.LogicalId == original.LogicalId).Count); + } + + [Fact] + public async Task ExpiryReplacement_WhenJournalWriteFails_RetainsDedupeRecord() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + using var envelope = CreateEnvelope( + receiver, + new DurableTestMessage(Guid.NewGuid(), 16, "failed-expiry-replacement")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => DeliverAsync(receiver, envelope.Value)); + + var failed = await receiver.GetSnapshotAsync(); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, Assert.Single(failed.Effects).Count); + Assert.Equal(1, failed.ProcessedMessageCount); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var retried = await fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, Assert.Single(retried.Effects).Count); + } + + private Task WaitForIdleInboxAsync(IDurableMessagingTestGrain receiver) => + fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && string.IsNullOrEmpty(snapshot.InboxJobId)); + + private static async Task DeliverAsync( + IDurableMessagingTestGrain receiver, + DurableEnvelope envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private EnvelopeLease CreateEnvelope( + IDurableMessagingTestGrain receiver, + DurableTestMessage message, + Guid? messageId = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("expiry-test-sender", "stable"); + var built = new DurableEnvelopeBuilder(sessions, sender) + .To(receiver.GetGrainId(), "messages/expiry") + .WithBody(message) + .Build(); + if (messageId is { } id) + { + built = new DurableEnvelope + { + MessageId = id, + SenderId = built.SenderId, + ReceiverId = built.ReceiverId, + RouteKey = built.RouteKey, + CorrelationKey = built.CorrelationKey, + ReplyTo = built.ReplyTo, + Data = built.Data, + CreatedAt = built.CreatedAt, + }; + } + + return new EnvelopeLease(built); + } + + private sealed class EnvelopeLease(DurableEnvelope value) : IDisposable + { + public DurableEnvelope Value { get; } = value; + public void Dispose() + { + } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.DurableMessaging.Tests/Functional/InboxCapacityBehaviorTests.cs new file mode 100644 index 00000000000..a2a2cebb72e --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableMessaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class InboxCapacityCollection : ICollectionFixture +{ + public const string Name = "Durable messaging inbox capacity"; +} + +[Collection(InboxCapacityCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class InboxCapacityBehaviorTests(InboxCapacityClusterFixture fixture) +{ + [Fact] + public async Task InboxAtCapacity_BackpressuresWithoutPersistenceAndRecoversWhenCapacityFrees() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("capacity-test-sender", Guid.NewGuid().ToString("N")); + var poison = new DurableEnvelopeBuilder(sessions, sender) + .To(receiver.GetGrainId(), "messages/capacity") + .WithBody(new DurableTestMessage(Guid.NewGuid(), 31, "poison", ThrowAfterStaging: true)) + .Build(); + var rejected = new DurableEnvelopeBuilder(sessions, sender) + .To(receiver.GetGrainId(), "messages/capacity") + .WithBody(new DurableTestMessage(Guid.NewGuid(), 32, "accepted-after-capacity")) + .Build(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, poison)).Status); + var full = await fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(full.Effects); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + fixture.Clock.Advance(TimeSpan.FromHours(2)); + await receiver.RequestDeactivationAsync(); + _ = await receiver.GetSnapshotAsync(); + await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("accepted-after-capacity", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + private static async Task DeliverAsync( + IDurableMessagingTestGrain receiver, + DurableEnvelope envelope) => + await receiver.AsReference().DeliverAsync(envelope); +} diff --git a/test/Orleans.DurableMessaging.Tests/Functional/MultiSiloDurableMessagingFailoverTests.cs b/test/Orleans.DurableMessaging.Tests/Functional/MultiSiloDurableMessagingFailoverTests.cs new file mode 100644 index 00000000000..0d4e61ef036 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Functional/MultiSiloDurableMessagingFailoverTests.cs @@ -0,0 +1,61 @@ +using Orleans.DurableMessaging.Tests.Support; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MultiSiloDurableMessagingCollection : ICollectionFixture +{ + public const string Name = "Durable messaging multi-silo cluster"; +} + +[Collection(MultiSiloDurableMessagingCollection.Name)] +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class MultiSiloDurableMessagingFailoverTests(MultiSiloDurableMessagingClusterFixture fixture) +{ + [Fact] + public async Task ReceiverOwnerStops_DuringBlockedHandler_NewOwnerRecoversStableJobAndProcessesOnce() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failover"); + var sender = fixture.Client.GetGrain(Guid.NewGuid()); + var logicalId = Guid.NewGuid(); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/failover", + new DurableTestMessage(logicalId, 81, "failover")); + await barrier.WaitUntilEnteredAsync(); + var before = fixture.GetSnapshot(receiver); + var owner = fixture.Cluster.Silos.Single( + silo => silo.SiloAddress.ToParsableString() == before.SiloAddress); + + await fixture.Cluster.KillSiloAsync(owner, TestContext.Current.CancellationToken); + await fixture.Cluster.WaitForLivenessToStabilizeAsync(); + var reactivated = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, reactivated.ActivationId); + barrier.Release(); + DurableEndpointSnapshot recovered; + try + { + recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovery did not complete. Activation={snapshot.ActivationId}, silo={snapshot.SiloAddress}, inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + + Assert.Equal(reactivated.ActivationId, recovered.ActivationId); + Assert.NotEqual(before.SiloAddress, recovered.SiloAddress); + var effect = Assert.Single(recovered.Effects); + Assert.Equal(logicalId, effect.LogicalId); + Assert.Equal(1, effect.Count); + Assert.Equal(0, recovered.InboxCount); + Assert.Single(fixture.Cluster.Silos); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Functional/PublicDurableMessagingBehaviorTests.cs b/test/Orleans.DurableMessaging.Tests/Functional/PublicDurableMessagingBehaviorTests.cs new file mode 100644 index 00000000000..3ccb12d3536 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Functional/PublicDurableMessagingBehaviorTests.cs @@ -0,0 +1,1219 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.DurableMessaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DurableMessagingClusterCollection +{ + public const string Name = "Durable messaging cluster"; +} + +[Collection(DurableMessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class PublicDurableMessagingBehaviorTests : IAsyncLifetime +{ + private readonly DurableMessagingClusterFixture fixture = new(); + + public ValueTask InitializeAsync() => fixture.InitializeAsync(); + public ValueTask DisposeAsync() => fixture.DisposeAsync(); + + [Fact] + public void DefaultHosting_UsesBinaryJournalFormat() + { + Assert.Equal("orleans-binary", fixture.Storage.JournalFormatKey); + } + + [Fact] + public async Task ApplicationJournaledStateNamesDoNotCollideWithMessagingState() + { + var grain = NewGrain(); + + var snapshot = await grain.GetSnapshotAsync(); + + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(0, snapshot.OutboxCount); + } + + [Fact] + public async Task Deliver_AcceptedOnlyAfterInboxAndStableJobOwnershipAreDurable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var barrier = fixture.Storage.BlockWrite(journalId); + using var envelope = CreateEnvelope(receiver, NewMessage(1, "durability")); + + var delivery = DeliverAsync(receiver, envelope.Value); + await barrier.WaitUntilEnteredAsync(); + + Assert.False(delivery.IsCompleted); + var staged = fixture.GetSnapshot(receiver); + Assert.Equal(1, staged.InboxCount); + Assert.Empty(staged.Effects); + + barrier.Release(); + var result = await delivery; + Assert.Equal(DeliveryStatus.Accepted, result.Status); + DurableEndpointSnapshot completed; + try + { + completed = await fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Accepted message did not drain. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}, job={snapshot.InboxJobId}.", + exception); + } + Assert.Single(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.True(fixture.Storage.GetSuccessfulWriteCount(journalId) >= 2); + } + + [Fact] + public async Task Deliver_CancellationStopsWaitingForInboxGate() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var barrier = fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var firstEnvelope = CreateEnvelope(receiver, NewMessage(72, "holds-gate")); + using var secondEnvelope = CreateEnvelope(receiver, NewMessage(73, "canceled")); + var firstDelivery = DeliverAsync(receiver, firstEnvelope.Value); + await barrier.WaitUntilEnteredAsync(); + using var cancellation = new CancellationTokenSource(); + + var canceledDelivery = DeliverWithCancellationAsync( + receiver, + secondEnvelope.Value, + cancellation.Token); + Assert.False(canceledDelivery.IsCompleted); + cancellation.Cancel(); + + try + { + await Assert.ThrowsAnyAsync(() => canceledDelivery); + } + finally + { + barrier.Release(); + } + + Assert.Equal(DeliveryStatus.Accepted, (await firstDelivery).Status); + } + + [Fact] + public async Task ConcurrentWriteCannotCaptureInboxAcceptanceBeforeScheduling() + { + var receiver = NewGrain(); + using var schedule = fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + using var envelope = CreateEnvelope(receiver, NewMessage(74, "schedule-barrier")); + + var delivery = DeliverAsync(receiver, envelope.Value); + await schedule.WaitUntilEnteredAsync(); + + var exception = await Assert.ThrowsAsync( + () => fixture.WriteStateAsync(receiver).AsTask()); + Assert.Contains("waiting for job scheduling", exception.Message, StringComparison.Ordinal); + + schedule.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("schedule-barrier", Assert.Single(completed.Effects).Value); + } + + [Fact] + public async Task RecoveryDuringInboxScheduling_PreventsFalseAcceptance() + { + var receiver = NewGrain(); + using var schedule = fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + using var envelope = CreateEnvelope(receiver, NewMessage(77, "recovered-during-schedule")); + + var delivery = DeliverAsync(receiver, envelope.Value); + await schedule.WaitUntilEnteredAsync(); + await fixture.RevertStateAsync(receiver); + schedule.Continue(); + + var exception = await Assert.ThrowsAsync(() => delivery); + Assert.Contains("interrupted by state recovery", exception.Message, StringComparison.Ordinal); + Assert.Equal(0, fixture.GetSnapshot(receiver).InboxCount); + } + + [Fact] + public async Task FailedInboxAcceptance_RevertsEnvelopeAndOrphanedJobCannotProcess() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var depthBaseline = fixture.Metrics.GetDepth("orleans-durable-messaging-inbox-depth"); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var envelope = CreateEnvelope(receiver, NewMessage(2, "failed-acceptance")); + + await Assert.ThrowsAnyAsync( + () => DeliverAsync(receiver, envelope.Value)); + + var reverted = await receiver.GetSnapshotAsync(); + Assert.Equal(0, reverted.InboxCount); + Assert.Empty(reverted.Effects); + Assert.Equal(depthBaseline, fixture.Metrics.GetDepth("orleans-durable-messaging-inbox-depth")); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(reverted.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(depthBaseline, fixture.Metrics.GetDepth("orleans-durable-messaging-inbox-depth")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(depthBaseline, fixture.Metrics.GetDepth("orleans-durable-messaging-inbox-depth")); + } + + [Fact] + public async Task AmbiguousInboxAcceptanceCommit_PreservesAndProcessesRecoveredEnvelope() + { + var receiver = NewGrain(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + fixture.Storage.FailAfterWrite(journalId); + using var envelope = CreateEnvelope(receiver, NewMessage(76, "ambiguous-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope.Value)); + + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("ambiguous-acceptance", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task Inbox_PrecommitCrash_ReclaimsScheduledOrphanAfterRecovery() + { + const string jobName = "orleans.messaging.inbox-drain"; + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + await receiver.DeactivateOnNextRecoveryAsync(); + var barrier = fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var attemptBaseline = fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var completionBaseline = fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var orphanBaseline = fixture.Metrics.GetCount( + "orleans-durable-messaging-orphaned-jobs-reclaimed", + jobName); + using var envelope = CreateEnvelope(receiver, NewMessage(3, "inbox-orphan")); + + var delivery = DeliverAsync(receiver, envelope.Value); + await barrier.WaitUntilEnteredAsync(); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-job-attempts-started", + attemptBaseline + 1); + + Assert.Equal( + orphanBaseline, + fixture.Metrics.GetCount("orleans-durable-messaging-orphaned-jobs-reclaimed", jobName)); + barrier.Fail(); + await Assert.ThrowsAnyAsync(() => delivery); + + var recovered = await fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + await fixture.Metrics.WaitForCountAsync( + "orleans-durable-messaging-orphaned-jobs-reclaimed", + orphanBaseline + 1, + jobName); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Empty(recovered.Effects); + Assert.Equal( + orphanBaseline + 1, + fixture.Metrics.GetCount("orleans-durable-messaging-orphaned-jobs-reclaimed", jobName)); + } + + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndOutgoingAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = Guid.NewGuid(); + using var envelope = CreateEnvelope( + receiver, + new DurableTestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope.Value); + var receiverState = await fixture.WaitForEffectCountAsync(receiver, 1); + var sinkState = await fixture.WaitForEffectCountAsync(sink, 1); + receiverState = await fixture.WaitForOutboxCountAsync(receiver, 0); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new DurableEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(0, receiverState.OutboxCount); + Assert.Equal(effect, Assert.Single(sinkState.Effects)); + + var duplicate = await DeliverAsync(receiver, envelope.Value); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Equal(1, Assert.Single((await sink.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task HandlerFailure_RollsBackEffectCompletionAndOutgoingThenDeadLetters() + { + var receiver = NewGrain(); + var sink = NewGrain(); + using var envelope = CreateEnvelope( + receiver, + new DurableTestMessage(Guid.NewGuid(), 9, "rollback", sink.GetGrainId(), ThrowAfterStaging: true)); + + var accepted = await DeliverAsync(receiver, envelope.Value); + var state = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Value.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty((await sink.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task HandlerSelectionFailure_IsDeadLettered() + { + var receiver = NewGrain(); + using var envelope = CreateEnvelope( + receiver, + NewMessage(80, "selection-failure"), + "messages/selection-failure"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var state = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Empty(state.Effects); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Contains("Injected handler selection failure", deadLetter.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task HandlerSelection_CannotStageOutboundMessages() + { + var receiver = NewGrain(); + using var envelope = CreateEnvelope( + receiver, + NewMessage(81, "selection-mutation"), + "messages/selection-mutation"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var state = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Empty(state.Effects); + Assert.Equal(0, state.OutboxCount); + Assert.Empty(state.OutboxDeadLetters); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Contains("selection is read-only", deadLetter.Reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task HandlerCannotCommitBeforeInboxCompletion() + { + var receiver = NewGrain(); + var message = new DurableTestMessage( + Guid.NewGuid(), + 10, + "premature-commit", + CommitDuringHandling: true); + using var envelope = CreateEnvelope(receiver, message); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var state = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Empty(state.Effects); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Contains("cannot be committed", deadLetter.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task HandlerCannotDeleteStateBeforeInboxCompletion() + { + var receiver = NewGrain(); + var message = new DurableTestMessage( + Guid.NewGuid(), + 11, + "premature-delete", + DeleteDuringHandling: true); + using var envelope = CreateEnvelope(receiver, message); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var state = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Empty(state.Effects); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Contains("cannot be committed or deleted", deadLetter.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task RecoveryDuringHandler_LeavesMessageRetryable() + { + var receiver = NewGrain(); + using var handler = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/recover-handler"); + using var envelope = CreateEnvelope( + receiver, + NewMessage(78, "recover-handler"), + "messages/recover-handler"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await handler.WaitUntilEnteredAsync(); + await fixture.RevertStateAsync(receiver); + handler.Release(); + + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task RecoveryDuringHandlerFailure_DiscardsStaleFailureAccounting() + { + var receiver = NewGrain(); + using var handler = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/recover-handler-failure"); + using var envelope = CreateEnvelope( + receiver, + NewMessage(79, "recover-handler-failure") with { ThrowOnceAfterStaging = true }, + "messages/recover-handler-failure"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await handler.WaitUntilEnteredAsync(); + await fixture.RevertStateAsync(receiver); + handler.Release(); + + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetention() + { + var receiver = NewGrain(); + using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/blocked-duplicate"); + using var envelope = CreateEnvelope(receiver, NewMessage(11, "duplicate"), "messages/blocked-duplicate"); + + var first = await DeliverAsync(receiver, envelope.Value); + await barrier.WaitUntilEnteredAsync(); + var second = DeliverAsync(receiver, envelope.Value); + + Assert.Equal(DeliveryStatus.Accepted, first.Status); + Assert.False(second.IsCompleted); + Assert.Equal(1, fixture.GetSnapshot(receiver).InboxCount); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await second).Status); + var state = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.MaxConcurrentHandlers); + } + + [Fact] + public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() + { + var receiver = NewGrain(); + using var envelope = CreateEnvelope(receiver, NewMessage(13, "reactivation")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var before = await fixture.WaitForEffectCountAsync(receiver, 1); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope.Value)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task ReorderedDistinctAndDuplicateMessages_ConvergeByApplicationSequence() + { + var receiver = NewGrain(); + var messages = new[] + { + NewMessage(3, "third"), + NewMessage(1, "first"), + NewMessage(2, "second"), + }; + + foreach (var message in messages) + { + using var envelope = CreateEnvelope(receiver, message); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope.Value)).Status); + } + + var state = await fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal([1, 2, 3], state.Effects.Select(static effect => effect.Sequence)); + Assert.All(state.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ConcurrentDelivery_WaitsWhileHandlersRemainSequential() + { + var receiver = NewGrain(); + using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/sequential"); + using var first = CreateEnvelope(receiver, NewMessage(21, "first"), "messages/sequential"); + using var second = CreateEnvelope(receiver, NewMessage(22, "second"), "messages/sequential"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first.Value)).Status); + await WaitForBarrierAsync(receiver, barrier); + var secondDelivery = DeliverAsync(receiver, second.Value); + Assert.False(secondDelivery.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + DurableEndpointSnapshot state; + try + { + state = await fixture.WaitForEffectCountAsync(receiver, 2); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Second message did not complete. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Equal(1, state.MaxConcurrentHandlers); + Assert.Equal([21, 22], state.Effects.Select(static effect => effect.Sequence)); + } + + [Fact] + public async Task MalformedTypedBody_DeadLettersAndDoesNotBlockLaterValidMessage() + { + var receiver = NewGrain(); + using var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed.Value)).Status); + var poisoned = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Empty(poisoned.Effects); + var deadLetter = Assert.Single(poisoned.InboxDeadLetters); + Assert.Equal(malformed.Value.MessageId, deadLetter.MessageId); + Assert.Contains(nameof(DurableTestMessage), deadLetter.Reason, StringComparison.Ordinal); + + using var valid = CreateEnvelope(receiver, NewMessage(41, "valid-after-poison"), "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valid.Value)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("valid-after-poison", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + [Fact] + public async Task InboxDeadLetterRemoval_IsDurable() + { + var receiver = NewGrain(); + using var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed.Value)).Status); + _ = await fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.True(await receiver.RemoveInboxDeadLetterAsync( + malformed.Value.SenderId, + malformed.Value.MessageId)); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.False(await receiver.RemoveInboxDeadLetterAsync( + malformed.Value.SenderId, + malformed.Value.MessageId)); + + await receiver.RequestDeactivationAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task StagedOutboxWithoutCommit_IsRemovedOnReactivationAndNeverDispatched() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var message = NewMessage(51, "uncommitted"); + + await sender.StageWithoutCommitAsync(receiver.GetGrainId(), "messages/uncommitted", message); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await sender.RequestDeactivationAsync(); + var reactivated = await sender.GetSnapshotAsync(); + + Assert.Equal(0, reactivated.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task CommittedOutbox_DeactivationBeforeLocalFollowUp_RecoversDurableJobOwnership() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAndDeactivateAsync( + receiver.GetGrainId(), + "messages/outbox-crash-window", + NewMessage(52, "durable-wakeup")); + await receiverWrite.WaitUntilEnteredAsync(); + var committed = fixture.GetSnapshot(sender); + + Assert.Equal(1, committed.OutboxCount); + Assert.False(string.IsNullOrEmpty(committed.OutboxJobId)); + + receiverWrite.Release(); + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + var recovered = await fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId + && snapshot.OutboxCount == 0 + && snapshot.OutboxJobId is null); + + Assert.Equal("durable-wakeup", Assert.Single(delivered.Effects).Value); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.OutboxCount); + Assert.Null(recovered.OutboxJobId); + } + + [Fact] + public async Task DuplicateOutboxEnqueue_PersistsOneStableJobOwnership() + { + var sender = NewGrain(); + var receiver = NewGrain(); + + await sender.SendDuplicateAsync( + receiver.GetGrainId(), + "messages/duplicate-outbox-enqueue", + NewMessage(54, "duplicate-enqueue")); + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal( + 1, + fixture.JobManagerProbe.GetSuccessCount( + "orleans.messaging.outbox-flush", + sender.GetGrainId())); + } + + [Fact] + public async Task OutboxSchedulingFailure_AbortsCommitAndRetryUsesStableOwnership() + { + var sender = NewGrain(); + var receiver = NewGrain(); + fixture.JobManagerProbe.FailAfterNext("orleans.messaging.outbox-flush"); + + await Assert.ThrowsAsync( + () => sender.SendAsync( + receiver.GetGrainId(), + "messages/schedule-retry", + NewMessage(55, "schedule-retry"))); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + await sender.RetryWriteStateAsync(); + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + + var effect = Assert.Single(delivered.Effects); + Assert.Equal("schedule-retry", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal( + 2, + fixture.JobManagerProbe.GetAttemptCount( + "orleans.messaging.outbox-flush", + sender.GetGrainId())); + Assert.Equal( + 2, + fixture.JobManagerProbe.GetSuccessCount( + "orleans.messaging.outbox-flush", + sender.GetGrainId())); + } + + [Fact] + public async Task FailedAtomicWrite_ReloadExposesNeitherGrainEffectNorOutgoingMessage() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId())); + + await Assert.ThrowsAnyAsync( + () => sender.SendAsync(receiver.GetGrainId(), "messages/write-failure", NewMessage(53, "failed-write"))); + await sender.RequestDeactivationAsync(); + + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + Assert.Equal( + 1, + fixture.JobManagerProbe.GetSuccessCount( + "orleans.messaging.outbox-flush", + sender.GetGrainId())); + } + + [Fact] + public async Task DeleteThenWrite_DiscardsPendingOutboxWithoutPoisoningNextWrite() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await sender.StageWithoutCommitAsync( + receiver.GetGrainId(), + "messages/delete-then-write", + NewMessage(75, "delete-then-write")); + + await sender.DeleteThenWriteStateAsync(); + + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task Outbox_PrecommitCrash_ReclaimsScheduledOrphanAfterRecovery() + { + const string jobName = "orleans.messaging.outbox-flush"; + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + await sender.DeactivateOnNextRecoveryAsync(); + var barrier = fixture.Storage.BlockWrite(JournalId.FromGrainId(sender.GetGrainId())); + var attemptBaseline = fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var completionBaseline = fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var orphanBaseline = fixture.Metrics.GetCount( + "orleans-durable-messaging-orphaned-jobs-reclaimed", + jobName); + + var send = sender.SendAsync( + receiver.GetGrainId(), + "messages/outbox-orphan", + NewMessage(54, "outbox-orphan")); + await barrier.WaitUntilEnteredAsync(); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-job-attempts-started", + attemptBaseline + 1); + + Assert.Equal( + orphanBaseline, + fixture.Metrics.GetCount("orleans-durable-messaging-orphaned-jobs-reclaimed", jobName)); + barrier.Fail(); + await Assert.ThrowsAnyAsync(() => send); + + await sender.RequestDeactivationAsync(); + var recovered = await sender.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await fixture.Metrics.WaitForCountAsync( + "orleans-durable-messaging-orphaned-jobs-reclaimed", + orphanBaseline + 1, + jobName); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(0, recovered.OutboxCount); + Assert.Null(recovered.OutboxJobId); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + Assert.Equal( + orphanBaseline + 1, + fixture.Metrics.GetCount("orleans-durable-messaging-orphaned-jobs-reclaimed", jobName)); + } + + [Fact] + public async Task OutboxJobClearWriteFailure_RevertsOwnershipAndRetryCleansUp() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var journalId = JournalId.FromGrainId(sender.GetGrainId()); + fixture.Storage.FailWrite(journalId, matchingWrite: 3); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/outbox-clear-retry", + NewMessage(56, "outbox-clear-retry")); + _ = await fixture.WaitForEffectCountAsync(receiver, 1); + var cleaned = await fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + static snapshot => snapshot.OutboxCount == 0 && snapshot.OutboxJobId is null); + + Assert.Equal(0, cleaned.OutboxCount); + Assert.Null(cleaned.OutboxJobId); + await sender.RequestDeactivationAsync(); + var recovered = await sender.GetSnapshotAsync(); + Assert.Null(recovered.OutboxJobId); + Assert.Equal(0, recovered.OutboxCount); + } + + [Fact] + public async Task InboxJobClearWriteFailure_RevertsThenRecoversAfterActivationLoss() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/inbox-clear-retry"); + using var envelope = CreateEnvelope(receiver, NewMessage(57, "inbox-clear-retry"), "messages/inbox-clear-retry"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await handler.WaitUntilEnteredAsync(); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId()), matchingWrite: 2); + fixture.DeactivateOnNextRecovery(receiver); + handler.Release(); + + var recovered = await fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + var cleaned = await fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && snapshot.InboxJobId is null); + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(cleaned.Effects).Count); + Assert.Empty(cleaned.InboxDeadLetters); + Assert.Null(cleaned.InboxJobId); + } + + [Fact] + public async Task DeliveryIntoEmptyInbox_ReplacesStalePersistedJobOwnership() + { + var receiver = NewGrain(); + var staleJobId = $"stale-{Guid.NewGuid():N}"; + await receiver.SetInboxJobIdAsync(staleJobId); + using var handler = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/stale-owner"); + using var envelope = CreateEnvelope(receiver, NewMessage(58, "stale-owner"), "messages/stale-owner"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await handler.WaitUntilEnteredAsync(); + var accepted = fixture.GetSnapshot(receiver); + + Assert.NotNull(accepted.InboxJobId); + Assert.NotEqual(staleJobId, accepted.InboxJobId); + + handler.Release(); + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("stale-owner", Assert.Single(completed.Effects).Value); + } + + [Fact] + public async Task Inbox_StaleGenerationCompletesWithoutClearingNewerOwner() + { + var receiver = NewGrain(); + const string route = "messages/stale-inbox-generation"; + using var handler = fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + using var envelope = CreateEnvelope(receiver, NewMessage(60, "newer-inbox-owner"), route); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + await handler.WaitUntilEnteredAsync(); + var owned = fixture.GetSnapshot(receiver); + Assert.False(string.IsNullOrEmpty(owned.InboxJobId)); + var completionBaseline = fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var manager = fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = receiver.GetGrainId(), + JobName = "orleans.messaging.inbox-drain", + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = "0" + } + }, + TestContext.Current.CancellationToken); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(owned.InboxJobId, fixture.GetSnapshot(receiver).InboxJobId); + handler.Release(); + Assert.Equal("newer-inbox-owner", Assert.Single((await fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + [Fact] + public async Task Outbox_StaleGenerationCompletesWithoutClearingNewerOwner() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/stale-outbox-generation", + NewMessage(61, "newer-outbox-owner")); + await receiverWrite.WaitUntilEnteredAsync(); + var owned = fixture.GetSnapshot(sender); + Assert.False(string.IsNullOrEmpty(owned.OutboxJobId)); + var completionBaseline = fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var manager = fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + try + { + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = sender.GetGrainId(), + JobName = "orleans.messaging.outbox-flush", + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = "0" + } + }, + TestContext.Current.CancellationToken); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(owned.OutboxJobId, fixture.GetSnapshot(sender).OutboxJobId); + } + finally + { + receiverWrite.Release(); + } + + Assert.Equal("newer-outbox-owner", Assert.Single((await fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + [Fact] + public async Task Outbox_JobVisibleDuringRecovery_PollsUntilCommittedOwnerIsRestored() + { + const string jobName = "orleans.messaging.outbox-flush"; + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/recovery-visibility", + NewMessage(62, "recovery-visibility")); + await receiverWrite.WaitUntilEnteredAsync(); + var owned = fixture.GetSnapshot(sender); + var ownershipId = Assert.IsType(owned.OutboxJobId); + var recoveryRead = fixture.Storage.BlockRead(JournalId.FromGrainId(sender.GetGrainId())); + var recovery = fixture.RevertStateAsync(sender).AsTask(); + await recoveryRead.WaitUntilEnteredAsync(); + var handlerBaseline = fixture.Metrics.GetCount("orleans-durablejobs-handler-executions-started"); + var completionBaseline = fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var orphanBaseline = fixture.Metrics.GetCount( + "orleans-durable-messaging-orphaned-jobs-reclaimed", + jobName); + var manager = fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + try + { + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = sender.GetGrainId(), + JobName = jobName, + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = ownershipId + } + }, + TestContext.Current.CancellationToken); + await fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-handler-executions-started", + handlerBaseline + 1); + + Assert.Equal( + orphanBaseline, + fixture.Metrics.GetCount("orleans-durable-messaging-orphaned-jobs-reclaimed", jobName)); + Assert.Equal(completionBaseline, fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed")); + } + finally + { + recoveryRead.Release(); + await recovery; + receiverWrite.Release(); + } + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("recovery-visibility", Assert.Single(delivered.Effects).Value); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + } + + [Fact] + public async Task InboxSchedulingFailure_RevertsAcceptanceAndRetryDoesNotStrandMessage() + { + var receiver = NewGrain(); + using var envelope = CreateEnvelope(receiver, NewMessage(59, "inbox-schedule-retry")); + fixture.JobManagerProbe.FailAfterNext("orleans.messaging.inbox-drain"); + + await Assert.ThrowsAsync( + () => DeliverAsync(receiver, envelope.Value)); + Assert.Equal(0, (await receiver.GetSnapshotAsync()).InboxCount); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var completed = await fixture.WaitForEffectCountAsync(receiver, 1); + + var effect = Assert.Single(completed.Effects); + Assert.Equal("inbox-schedule-retry", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal( + 2, + fixture.JobManagerProbe.GetAttemptCount( + "orleans.messaging.inbox-drain", + receiver.GetGrainId())); + Assert.Equal( + 2, + fixture.JobManagerProbe.GetSuccessCount( + "orleans.messaging.inbox-drain", + receiver.GetGrainId())); + } + + [Fact] + public async Task NullBodyAndContext_DecodeSuccessfullyAndTypedHandlersReceiveNull() + { + var receiver = NewGrain(); + using var referenceEnvelope = CreateEnvelope( + receiver, + body: null, + route: "nullable/reference", + builder => builder + .WithContextValue("null-reference", null) + .WithContextValue("null-value", null)); + + Assert.True(referenceEnvelope.Value.Data.TryGetBody(out var referenceBody)); + Assert.Null(referenceBody); + Assert.True(referenceEnvelope.Value.Data.TryGetContextValue("null-reference", out var referenceContext)); + Assert.Null(referenceContext); + Assert.True(referenceEnvelope.Value.Data.TryGetContextValue("null-value", out var valueContext)); + Assert.Null(valueContext); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, referenceEnvelope.Value)).Status); + + using var valueEnvelope = CreateEnvelope(receiver, body: null, route: "nullable/value"); + Assert.True(valueEnvelope.Value.Data.TryGetBody(out var valueBody)); + Assert.Null(valueBody); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valueEnvelope.Value)).Status); + + var completed = await fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.NullReferenceMessageCalls == 1 + && snapshot.NullNullableValueMessageCalls == 1); + Assert.Equal(1, completed.NullReferenceMessageCalls); + Assert.Equal(1, completed.NullNullableValueMessageCalls); + } + + [Fact] + public async Task BlockedInboxHandler_DoesNotStopIndependentOutboxAndInboxPumps() + { + var blocked = NewGrain(); + var independentSender = NewGrain(); + var independentReceiver = NewGrain(); + using var barrier = fixture.HandlerProbe.Arm(blocked.GetGrainId(), "messages/blocked-pump"); + using var blockedEnvelope = CreateEnvelope(blocked, NewMessage(61, "blocked"), "messages/blocked-pump"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(blocked, blockedEnvelope.Value)).Status); + await barrier.WaitUntilEnteredAsync(); + await independentSender.SendAsync( + independentReceiver.GetGrainId(), + "messages/independent", + NewMessage(62, "independent")); + var independent = await fixture.WaitForEffectCountAsync(independentReceiver, 1); + + Assert.Equal("independent", Assert.Single(independent.Effects).Value); + Assert.Empty(fixture.GetSnapshot(blocked).Effects); + barrier.Release(); + Assert.Equal("blocked", Assert.Single((await fixture.WaitForEffectCountAsync(blocked, 1)).Effects).Value); + } + + [Fact] + public async Task DuplicateExactRouteRegistration_ThrowsAndPreservesLookupAndDispatch() + { + var receiver = NewGrain(); + const string route = "exact/duplicate"; + + var registration = await receiver.RegisterDuplicateExactRouteHandlersAsync(route); + + Assert.Equal( + "A handler is already registered for exact route 'exact/duplicate'.", + registration.ExceptionMessage); + Assert.True(registration.LookupRetainedFirstHandler); + + using var envelope = CreateEnvelope(receiver, NewMessage(69, "first-handler"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope.Value)).Status); + var state = await fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.FirstExactRouteHandlerCalls == 1); + Assert.Equal(1, state.FirstExactRouteHandlerCalls); + Assert.Equal(0, state.ReplacementExactRouteHandlerCalls); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task RouteLookup_RejectsInvalidRouteKeys(string? route) + { + var result = await NewGrain().ValidateRouteLookupAsync(route); + + Assert.Equal("routeKey", result.HasHandlerParameterName); + Assert.Equal("routeKey", result.TryGetHandlerParameterName); + } + + [Fact] + public async Task RouteNotFound_IsRejectedWithoutInboxPersistence() + { + var receiver = NewGrain(); + using var envelope = CreateEnvelope(receiver, NewMessage(71, "missing"), "unknown/route"); + + var result = await DeliverAsync(receiver, envelope.Value); + + Assert.Equal(DeliveryStatus.RouteNotFound, result.Status); + Assert.Equal("No handler for route 'unknown/route'", result.Message); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + } + + [Fact] + public async Task Deliver_RejectsEnvelopeAddressedToAnotherGrain() + { + var receiver = NewGrain(); + var declaredReceiver = NewGrain(); + using var envelope = CreateEnvelope( + declaredReceiver, + NewMessage(74, "wrong-receiver")); + + var exception = await Assert.ThrowsAsync( + () => DeliverAsync(receiver, envelope.Value)); + + Assert.Contains(declaredReceiver.GetGrainId().ToString(), exception.Message, StringComparison.Ordinal); + Assert.Contains(receiver.GetGrainId().ToString(), exception.Message, StringComparison.Ordinal); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(declaredReceiver, envelope.Value)).Status); + var delivered = await fixture.WaitForEffectCountAsync(declaredReceiver, 1); + Assert.Single(delivered.Effects); + } + + [Fact] + public async Task RouteNotFound_OutboxRetriesThenDeadLettersWithoutReceiverPersistence() + { + var sender = NewGrain(); + var receiver = NewGrain(); + + await sender.SendAsync( + receiver.GetGrainId(), + "unknown/outbox-route", + NewMessage(72, "undeliverable")); + var senderState = await fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.Equal(0, senderState.OutboxCount); + var deadLetter = Assert.Single(senderState.OutboxDeadLetters); + Assert.Equal("unknown/outbox-route", deadLetter.Route); + Assert.Equal(3, deadLetter.AttemptCount); + Assert.Contains("No handler", deadLetter.Reason, StringComparison.Ordinal); + var receiverState = await receiver.GetSnapshotAsync(); + Assert.Equal(0, receiverState.InboxCount); + Assert.Empty(receiverState.Effects); + Assert.Empty(receiverState.InboxDeadLetters); + } + + [Fact] + public async Task OutboxDeadLetterRemoval_IsDurable() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = await sender.SendAsync( + receiver.GetGrainId(), + "unknown/removable-outbox-route", + NewMessage(73, "removable-undeliverable")); + _ = await fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.True(await sender.RemoveOutboxDeadLetterAsync(messageId)); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + Assert.False(await sender.RemoveOutboxDeadLetterAsync(messageId)); + + await sender.RequestDeactivationAsync(); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + private IDurableMessagingTestGrain NewGrain() => + fixture.Client.GetGrain(Guid.NewGuid()); + + private static DurableTestMessage NewMessage(int sequence, string value) => + new(Guid.NewGuid(), sequence, value); + + private static Task DeliverAsync( + IDurableMessagingTestGrain receiver, + DurableEnvelope envelope) => + DeliverWithCancellationAsync(receiver, envelope, TestContext.Current.CancellationToken); + + private static async Task DeliverWithCancellationAsync( + IDurableMessagingTestGrain receiver, + DurableEnvelope envelope, + CancellationToken cancellationToken) => + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + + private static async Task WaitForBarrierAsync( + IDurableMessagingTestGrain receiver, + HandlerProbe.Barrier barrier) + { + try + { + await barrier.WaitUntilEnteredAsync(); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Handler did not start. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, maxHandlers={snapshot.MaxConcurrentHandlers}, deadLetters={string.Join(" | ", snapshot.InboxDeadLetters.Select(static item => item.Reason))}.", + exception); + } + } + + private EnvelopeLease CreateEnvelope( + IDurableMessagingTestGrain receiver, + DurableTestMessage message, + string route = "messages/record") => + CreateEnvelope(receiver, (object)message, route); + + private EnvelopeLease CreateEnvelope( + IDurableMessagingTestGrain receiver, + object body, + string route) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + var builder = new DurableEnvelopeBuilder(sessions, sender).To(receiver.GetGrainId(), route); + var envelope = body switch + { + DurableTestMessage message => builder.WithBody(message).Build(), + string text => builder.WithBody(text).Build(), + _ => throw new ArgumentException($"Unsupported test body type {body.GetType()}.", nameof(body)), + }; + return new EnvelopeLease(envelope); + } + + private EnvelopeLease CreateEnvelope( + IDurableMessagingTestGrain receiver, + T body, + string route, + Action? configure = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + var builder = new DurableEnvelopeBuilder(sessions, sender).To(receiver.GetGrainId(), route); + configure?.Invoke(builder); + return new EnvelopeLease(builder.WithBody(body).Build()); + } + + private sealed class EnvelopeLease(DurableEnvelope value) : IDisposable + { + public DurableEnvelope Value { get; } = value; + public void Dispose() + { + } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Hosting/PublicDurableMessagingRegistrationTests.cs b/test/Orleans.DurableMessaging.Tests/Hosting/PublicDurableMessagingRegistrationTests.cs new file mode 100644 index 00000000000..d642eb562e9 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Hosting/PublicDurableMessagingRegistrationTests.cs @@ -0,0 +1,223 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.DurableMessaging.Configuration; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Hosting; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public sealed class PublicDurableMessagingRegistrationTests +{ + [Fact] + public void AddDurableMessaging_RegistersPublicScopedContractsAndInboxExtensionKey() + { + var services = new ServiceCollection(); + + services.AddDurableMessaging(); + + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(IDurableInbox)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(IDurableOutbox)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(IDurableMessagingDiagnostics)); + Assert.Contains( + services, + descriptor => descriptor.ServiceType == typeof(IGrainExtension) + && Equals(descriptor.ServiceKey, typeof(IDurableInboxExtension))); + } + + [Fact] + public void AddDurableMessaging_PreservesCallerSuppliedTimeProviderAndAppliesOptions() + { + var expectedTime = new FixedTimeProvider(new DateTimeOffset(2040, 2, 3, 4, 5, 6, TimeSpan.Zero)); + var services = new ServiceCollection(); + services.AddSingleton(expectedTime); + + services.AddDurableMessaging(options => + { + options.MaxCapacity = 17; + options.InboxBatchSize = 4; + options.OutboxBatchSize = 5; + }); + using var provider = services.BuildServiceProvider(); + + Assert.Same(expectedTime, provider.GetRequiredService()); + var configured = provider.GetRequiredService>().Value; + Assert.Equal(17, configured.MaxCapacity); + Assert.Equal(4, configured.InboxBatchSize); + Assert.Equal(5, configured.OutboxBatchSize); + } + + [Fact] + public void AddDurableMessaging_SelectsBinaryJournalFormat() + { + var services = new ServiceCollection(); + services.AddOptions(); + services.AddDurableMessaging(); + using var provider = services.BuildServiceProvider(); + + Assert.Equal( + "orleans-binary", + provider.GetRequiredService>().Value.JournalFormatKey); + } + + [Fact] + public void ActivationValidator_RejectsReentrantGrainTypes() + { + var validatorType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableMessagingActivationValidator", + throwOnError: true)!; + var validate = validatorType.GetMethod( + "Validate", + BindingFlags.Static | BindingFlags.Public)!; + var context = Substitute.For(); + context.GrainInstance.Returns(new ReentrantTestGrain()); + + var exception = Assert.Throws( + () => validate.Invoke(null, [context])); + + var diagnostic = Assert.IsType(exception.InnerException); + Assert.Contains("non-reentrant", diagnostic.Message, StringComparison.Ordinal); + Assert.Contains(typeof(ReentrantTestGrain).ToString(), diagnostic.Message, StringComparison.Ordinal); + } + + [Fact] + public void ActivationValidator_RejectsAlwaysInterleaveMethods() + { + var validatorType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableMessagingActivationValidator", + throwOnError: true)!; + var validate = validatorType.GetMethod( + "Validate", + BindingFlags.Static | BindingFlags.Public)!; + var context = Substitute.For(); + context.GrainInstance.Returns(new InterleavableTestGrain()); + + var exception = Assert.Throws( + () => validate.Invoke(null, [context])); + + var diagnostic = Assert.IsType(exception.InnerException); + Assert.Contains("interleavable method", diagnostic.Message, StringComparison.Ordinal); + Assert.Contains(nameof(IInterleavableBase.PingAsync), diagnostic.Message, StringComparison.Ordinal); + } + + [Fact] + public void ActivationValidator_RejectsStatelessWorkers() + { + var validatorType = typeof(IDurableInbox).Assembly.GetType( + "Orleans.DurableMessaging.DurableMessagingActivationValidator", + throwOnError: true)!; + var validate = validatorType.GetMethod( + "Validate", + BindingFlags.Static | BindingFlags.Public)!; + var context = Substitute.For(); + context.GrainInstance.Returns(new StatelessWorkerTestGrain()); + + var exception = Assert.Throws( + () => validate.Invoke(null, [context])); + + var diagnostic = Assert.IsType(exception.InnerException); + Assert.Contains("one activation", diagnostic.Message, StringComparison.Ordinal); + Assert.Contains("stateless worker", diagnostic.Message, StringComparison.Ordinal); + } + + [Fact] + public void AddDurableMessaging_InvalidOptionsFailThroughOptionsContract() + { + var services = new ServiceCollection(); + services.AddDurableMessaging(options => options.MaxCapacity = 0); + using var provider = services.BuildServiceProvider(); + + var exception = Assert.Throws( + () => provider.GetRequiredService>().Value); + + Assert.Contains("DurableInboxOptions validation failed", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task AddDurableMessaging_DoesNotReplaceUnrelatedConstructionErrors() + { + var services = new ServiceCollection(); + services.AddDurableMessaging(); + services.AddScoped(); + await using var provider = services.BuildServiceProvider(); + var extensionType = services.Single(descriptor => descriptor.ServiceType.Name == "DurableInboxExtension").ServiceType; + + var exception = Assert.Throws(() => provider.GetRequiredService(extensionType)); + + Assert.DoesNotContain("observer support", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ExternalConsumerAssembly_HasNoFriendAccessToDurableMessaging() + { + var sourceAssembly = typeof(IDurableInbox).Assembly; + var consumerName = typeof(PublicDurableMessagingRegistrationTests).Assembly.GetName().Name; + var friendDeclarations = sourceAssembly + .GetCustomAttributesData() + .Where(attribute => attribute.AttributeType.FullName == "System.Runtime.CompilerServices.InternalsVisibleToAttribute") + .Select(attribute => attribute.ConstructorArguments[0].Value?.ToString()) + .ToArray(); + + Assert.DoesNotContain(friendDeclarations, declaration => + declaration?.StartsWith(consumerName!, StringComparison.Ordinal) == true); + } + + private sealed class FixedTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => utcNow; + } + + [Orleans.Concurrency.Reentrant] + private sealed class ReentrantTestGrain + { + } + + public interface IInterleavableBase + { + [Orleans.Concurrency.AlwaysInterleave] + Task PingAsync(); + } + + public interface IInterleavableTestGrain : IGrain, IInterleavableBase + { + } + + private sealed class InterleavableTestGrain : IInterleavableTestGrain + { + public Task PingAsync() => Task.CompletedTask; + } + + [Orleans.Concurrency.StatelessWorker] + private sealed class StatelessWorkerTestGrain + { + } + + private class RollbackOnlyStateManager : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + public void RegisterState(string name, IJournaledState state) { } + public virtual void RegisterObserver(IJournaledStateObserver observer) => + throw new NotSupportedException(); + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } + + private sealed class FullyCapableStateManager : RollbackOnlyStateManager + { + public override void RegisterObserver(IJournaledStateObserver observer) { } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Orleans.DurableMessaging.Tests.csproj b/test/Orleans.DurableMessaging.Tests/Orleans.DurableMessaging.Tests.csproj new file mode 100644 index 00000000000..645c402f453 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Orleans.DurableMessaging.Tests.csproj @@ -0,0 +1,21 @@ + + + $(TestTargetFrameworks) + true + enable + enable + $(NoWarn);ORLEANSEXP005 + + + + + + + + + + + + + + diff --git a/test/Orleans.DurableMessaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.DurableMessaging.Tests/Support/ControlledDurableJobManager.cs new file mode 100644 index 00000000000..9df7a89b8a9 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/ControlledDurableJobManager.cs @@ -0,0 +1,130 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging.Tests.Support; + +public sealed class DurableJobManagerProbe +{ + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _attempts = []; + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; + private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); + + public void FailNext(string jobName) => + _failures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void FailAfterNext(string jobName) => + _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public int GetAttemptCount(string jobName, GrainId target) => + _attempts.TryGetValue((jobName, target), out var count) ? count : 0; + + public int GetSuccessCount(string jobName, GrainId target) => + _successes.TryGetValue((jobName, target), out var count) ? count : 0; + + public ScheduleBarrier BlockNext(string jobName) + { + var barrier = new ScheduleBarrier(); + if (!_scheduleBarriers.TryAdd(jobName, barrier)) + { + throw new InvalidOperationException($"A scheduling barrier is already armed for '{jobName}'."); + } + + return barrier; + } + + internal void OnAttempt(ScheduleJobRequest request) => + _attempts.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnSuccess(ScheduleJobRequest request) => + _successes.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal bool ShouldFail(string jobName) + => TryConsumeFailure(_failures, jobName); + + internal bool ShouldFailAfterSchedule(string jobName) + => TryConsumeFailure(_postScheduleFailures, jobName); + + internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) + { + if (!_scheduleBarriers.TryRemove(jobName, out var barrier)) + { + return; + } + + barrier.Entered.TrySetResult(); + await barrier.Release.Task.WaitAsync(cancellationToken); + } + + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + string jobName) + { + while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) + { + if (failures.TryUpdate(jobName, remaining - 1, remaining)) + { + return true; + } + } + + return false; + } + + public sealed class ScheduleBarrier : IDisposable + { + internal TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Continue() => Release.TrySetResult(); + + public void Dispose() => Continue(); + } +} + +internal sealed class ControlledDurableJobManager( + ILocalDurableJobManager inner, + DurableJobManagerProbe probe) : ILocalDurableJobManager +{ + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + probe.OnAttempt(request); + if (probe.ShouldFail(request.JobName)) + { + throw new IOException($"Injected durable job scheduling failure for '{request.JobName}'."); + } + + await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + var result = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnSuccess(request); + if (probe.ShouldFailAfterSchedule(request.JobName)) + { + throw new IOException( + $"Injected durable job scheduling response failure for '{request.JobName}'."); + } + + return result; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + inner.CancelAsync(job, cancellationToken); + + public static void Decorate(IServiceCollection services, DurableJobManagerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ILocalDurableJobManager)); + var factory = descriptor.ImplementationFactory + ?? throw new InvalidOperationException("The durable job manager registration must use an implementation factory."); + services.Remove(descriptor); + services.AddSingleton( + serviceProvider => new ControlledDurableJobManager( + (ILocalDurableJobManager)factory(serviceProvider), + probe)); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/ControlledJournalStorageProvider.cs b/test/Orleans.DurableMessaging.Tests/Support/ControlledJournalStorageProvider.cs new file mode 100644 index 00000000000..3af19b4b700 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/ControlledJournalStorageProvider.cs @@ -0,0 +1,194 @@ +using System.Buffers; +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.DurableMessaging.Tests.Support; + +public sealed class ControlledJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private VolatileJournalStorageProvider? _inner; + private readonly ConcurrentDictionary _readPlans = new(); + private readonly ConcurrentDictionary _writePlans = new(); + private readonly ConcurrentDictionary _postWritePlans = new(); + private readonly ConcurrentDictionary _successfulWrites = new(); + + public string? JournalFormatKey { get; private set; } + + public void Configure(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + JournalFormatKey = options.Value.JournalFormatKey; + _inner ??= new VolatileJournalStorageProvider(options); + } + + public IJournalStorage CreateStorage(JournalId journalId) => + new ControlledJournalStorage(this, journalId, Inner.CreateStorage(journalId)); + + public IAsyncEnumerable ListAsync( + JournalId prefix = default, + CancellationToken cancellationToken = default) => + Inner.ListAsync(prefix, cancellationToken); + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The controlled journal storage provider has not been configured."); + + public WriteBarrier BlockWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + var plan = new WritePlan(matchingWrite, fail: false); + if (!_writePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public WriteBarrier BlockRead(JournalId journalId, int matchingRead = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingRead); + var plan = new WritePlan(matchingRead, fail: false); + if (!_readPlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A read plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public void FailWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_writePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + } + + public void FailAfterWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_postWritePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + } + + public int GetSuccessfulWriteCount(JournalId journalId) => + _successfulWrites.TryGetValue(journalId, out var count) ? count : 0; + + private async ValueTask BeforeWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_writePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _writePlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + if (plan.Fail) + { + throw new IOException($"Injected journal write failure for '{journalId}'."); + } + + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async ValueTask BeforeReadAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_readPlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _readPlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private void OnWriteSucceeded(JournalId journalId) => + _successfulWrites.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + + private void AfterWrite(JournalId journalId) + { + if (!_postWritePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _postWritePlans.TryRemove(new KeyValuePair(journalId, plan)); + throw new IOException($"Injected post-commit journal response failure for '{journalId}'."); + } + + internal sealed class WritePlan(int target, bool fail) + { + public int Target { get; } = target; + public bool Fail { get; } = fail; + public int Seen; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class WriteBarrier + { + private readonly WritePlan _plan; + + internal WriteBarrier(WritePlan plan) => _plan = plan; + + public Task WaitUntilEnteredAsync() => _plan.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => _plan.Release.TrySetResult(); + public void Fail() => _plan.Release.TrySetException(new IOException("Injected blocked journal write failure.")); + } + + private sealed class ControlledJournalStorage( + ControlledJournalStorageProvider owner, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => inner.IsCompactionRequested; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + await owner.BeforeReadAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReadAsync(consumer, cancellationToken).ConfigureAwait(false); + } + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) => + inner.CreateIfNotExistsAsync(metadata, cancellationToken); + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId); + owner.AfterWrite(journalId); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId); + owner.AfterWrite(journalId); + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => + inner.DeleteAsync(cancellationToken); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingClusterFixture.cs b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingClusterFixture.cs new file mode 100644 index 00000000000..89c01e01de7 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingClusterFixture.cs @@ -0,0 +1,162 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.DurableMessaging.Configuration; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Support; + +public class DurableMessagingClusterFixture : IAsyncLifetime +{ + public DurableMessagingClusterFixture() : this(1) + { + } + + protected DurableMessagingClusterFixture(int initialSilos) + { + Clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + Storage = new ControlledJournalStorageProvider(); + Metrics = new DurableMessagingMetricProbe(); + JobManagerProbe = new DurableJobManagerProbe(); + HandlerProbe = new HandlerProbe(); + SnapshotProbe = new SnapshotProbe(); + var clusterId = $"durable-messaging-{Guid.NewGuid():N}"; + var serviceId = $"durable-messaging-service-{Guid.NewGuid():N}"; + var builder = new InProcessTestClusterBuilder((short)initialSilos); + builder.ConfigureClient(clientBuilder => + clientBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + })); + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + }); + siloBuilder.Services.AddSingleton(Clock); + siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + siloBuilder.Services.AddSingleton(HandlerProbe); + siloBuilder.Services.AddSingleton(SnapshotProbe); + siloBuilder.UseInMemoryDurableJobs(); + siloBuilder.AddDurableMessaging(ConfigureOptions); + siloBuilder.ConfigureServices(services => + ControlledDurableJobManager.Decorate(services, JobManagerProbe)); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.AddSingleton(Storage); + siloBuilder.Services.AddSingleton(serviceProvider => + { + Storage.Configure(serviceProvider.GetRequiredService>()); + return Storage; + }); + siloBuilder.Services.AddSingleton( + serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); + }); + Cluster = builder.Build(); + } + + public InProcessTestCluster Cluster { get; } + public IClusterClient Client => Cluster.Client!; + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public DurableMessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + + public Task WaitForEffectCountAsync(IDurableMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.Effects.Sum(static effect => effect.Count) >= expected); + + public Task WaitForInboxCountAsync(IDurableMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.InboxCount == expected); + + public Task WaitForDeadLetterCountAsync(IDurableMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Count + snapshot.OutboxDeadLetters.Count >= expected); + + public Task WaitForOutboxCountAsync(IDurableMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + + public DurableEndpointSnapshot GetSnapshot(IDurableMessagingTestGrain grain) => + GetGrainInstance(grain).GetSnapshotForTest(); + + public ValueTask RevertStateAsync(IDurableMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .RevertPendingChangesAsync(TestContext.Current.CancellationToken); + + public ValueTask WriteStateAsync(IDurableMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken); + + public void DeactivateOnNextRecovery(IDurableMessagingTestGrain grain) => + GetGrainInstance(grain).DeactivateOnNextRecoveryForTest(); + + private IGrainContext GetGrainContext(IDurableMessagingTestGrain grain) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + return context; + } + + private DurableMessagingTestGrain GetGrainInstance(IDurableMessagingTestGrain grain) => + GetGrainContext(grain).GrainInstance as DurableMessagingTestGrain + ?? throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' has an unexpected implementation."); + + protected virtual void ConfigureOptions(DurableInboxOptions options) + { + options.MaxCapacity = 2; + options.DeduplicationWindow = TimeSpan.FromMinutes(10); + options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxProcessingAttempts = 1; + options.MaxDeliveryAttempts = 3; + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); + options.InboxBatchSize = 8; + options.OutboxBatchSize = 8; + } + + public ValueTask InitializeAsync() => new(Cluster.DeployAsync()); + public async ValueTask DisposeAsync() + { + await Cluster.DisposeAsync(); + Metrics.Dispose(); + } +} + +public sealed class MultiSiloDurableMessagingClusterFixture : DurableMessagingClusterFixture +{ + public MultiSiloDurableMessagingClusterFixture() : base(2) + { + } +} + +public sealed class DedupeExpiryClusterFixture : DurableMessagingClusterFixture +{ +} + +public sealed class InboxCapacityClusterFixture : DurableMessagingClusterFixture +{ + protected override void ConfigureOptions(DurableInboxOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 1; + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingMetricProbe.cs b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingMetricProbe.cs new file mode 100644 index 00000000000..7aec77119a5 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingMetricProbe.cs @@ -0,0 +1,118 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Support; + +public sealed class DurableMessagingMetricProbe : IDisposable +{ + private readonly ConcurrentDictionary<(string Instrument, string JobName), long> _measurements = []; + private readonly ConcurrentDictionary _gauges = []; + private readonly object _lock = new(); + private TaskCompletionSource _changed = CreateSignal(); + private readonly MeterListener _listener; + + public DurableMessagingMetricProbe() + { + _listener = new MeterListener + { + InstrumentPublished = static (instrument, listener) => + { + if (instrument.Meter.Name == "Microsoft.Orleans" + && instrument.Name is "orleans-durable-messaging-orphaned-jobs-reclaimed" + or "orleans-durablejobs-job-attempts-started" + or "orleans-durablejobs-handler-executions-started" + or "orleans-durablejobs-jobs-completed" + or "orleans-durable-messaging-inbox-depth" + or "orleans-durable-messaging-outbox-depth") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + public long GetCount(string instrument, string jobName = "") => + _measurements.TryGetValue((instrument, jobName), out var count) ? count : 0; + + public long GetDepth(string instrument) + { + _listener.RecordObservableInstruments(); + return _gauges.TryGetValue(instrument, out var value) ? value : 0; + } + + public Task WaitForCountAsync( + string instrument, + long expected, + string jobName = "") => + WaitForCountWithCancellationAsync( + instrument, + expected, + jobName, + TestContext.Current.CancellationToken); + + public async Task WaitForCountWithCancellationAsync( + string instrument, + long expected, + string jobName, + CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + while (GetCount(instrument, jobName) < expected) + { + Task changed; + lock (_lock) + { + if (GetCount(instrument, jobName) >= expected) + { + return; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement( + Instrument instrument, + long measurement, + ReadOnlySpan> tags, + object? state) + { + if (instrument.Name is "orleans-durable-messaging-inbox-depth" or "orleans-durable-messaging-outbox-depth") + { + _gauges[instrument.Name] = measurement; + return; + } + + var jobName = ""; + foreach (var tag in tags) + { + if (tag.Key == "job_name") + { + jobName = tag.Value as string ?? ""; + break; + } + } + + _measurements.AddOrUpdate( + (instrument.Name, jobName), + measurement, + (_, current) => current + measurement); + lock (_lock) + { + _changed.TrySetResult(); + _changed = CreateSignal(); + } + } + + private static TaskCompletionSource CreateSignal() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingTestGrains.cs b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingTestGrains.cs new file mode 100644 index 00000000000..936a2a26ecb --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/DurableMessagingTestGrains.cs @@ -0,0 +1,504 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.DurableMessaging; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.DurableMessaging.Tests.Support; + +public interface IDurableMessagingTestGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, DurableTestMessage message); + Task SendDuplicateAsync(GrainId target, string route, DurableTestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, DurableTestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, DurableTestMessage message); + Task DeleteThenWriteStateAsync(); + Task RetryWriteStateAsync(); + Task RevertStateAsync(); + Task SetInboxJobIdAsync(string jobId); + Task DeactivateOnNextRecoveryAsync(); + Task RegisterDuplicateExactRouteHandlersAsync(string route); + Task ValidateRouteLookupAsync(string? route); + Task RemoveInboxDeadLetterAsync(GrainId senderId, Guid messageId); + Task RemoveOutboxDeadLetterAsync(Guid messageId); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); +} + +[GenerateSerializer, Immutable] +public sealed record DuplicateRouteRegistrationResult( + [property: Id(0)] string ExceptionMessage, + [property: Id(1)] bool LookupRetainedFirstHandler); + +[GenerateSerializer, Immutable] +public sealed record RouteLookupValidationResult( + [property: Id(0)] string HasHandlerParameterName, + [property: Id(1)] string TryGetHandlerParameterName); + +[GenerateSerializer, Immutable] +public sealed record DurableTestMessage( + [property: Id(0)] Guid LogicalId, + [property: Id(1)] int Sequence, + [property: Id(2)] string Value, + [property: Id(3)] GrainId? ForwardTo = null, + [property: Id(4)] bool ThrowAfterStaging = false, + [property: Id(5)] bool CommitDuringHandling = false, + [property: Id(6)] bool DeleteDuringHandling = false, + [property: Id(7)] bool ThrowOnceAfterStaging = false); + +[GenerateSerializer, Immutable] +public sealed record DurableEffect( + [property: Id(0)] Guid LogicalId, + [property: Id(1)] int Count, + [property: Id(2)] int Sequence, + [property: Id(3)] string Value); + +[GenerateSerializer, Immutable] +public sealed record DurableEndpointSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] string SiloAddress, + [property: Id(2)] int InboxCount, + [property: Id(3)] int OutboxCount, + [property: Id(4)] int MaxConcurrentHandlers, + [property: Id(5)] IReadOnlyList Effects, + [property: Id(6)] IReadOnlyList InboxDeadLetters, + [property: Id(7)] IReadOnlyList OutboxDeadLetters, + [property: Id(8)] string? InboxJobId, + [property: Id(9)] int ProcessedMessageCount, + [property: Id(10)] int FirstExactRouteHandlerCalls, + [property: Id(11)] int ReplacementExactRouteHandlerCalls, + [property: Id(12)] string? OutboxJobId, + [property: Id(13)] int NullReferenceMessageCalls, + [property: Id(14)] int NullNullableValueMessageCalls); + +[GenerateSerializer, Immutable] +public sealed record DurableDeadLetterSnapshot( + [property: Id(0)] Guid MessageId, + [property: Id(1)] string Route, + [property: Id(2)] string Reason, + [property: Id(3)] int AttemptCount, + [property: Id(4)] DateTimeOffset DeadLetteredAt); + +[GrainType("durable-messaging-public-test")] +public sealed class DurableMessagingTestGrain : DurableGrain, IDurableMessagingTestGrain, IJournaledStateObserver +{ + private readonly IDurableInbox _inbox; + private readonly IDurableOutbox _outbox; + private readonly IDurableMessagingDiagnostics _diagnostics; + private readonly IDurableDictionary _effects; + private readonly IDurableDictionary<(GrainId SenderId, Guid MessageId), DateTimeOffset> _processedMessages; + private readonly SerializerSessionPool _sessions; + private readonly IDurableValue _inboxJobId; + private readonly IDurableValue _outboxJobId; + private readonly ILocalSiloDetails _siloDetails; + private readonly HandlerProbe _handlerProbe; + private readonly SnapshotProbe _snapshotProbe; + private readonly Guid _activationId = Guid.NewGuid(); + private int _activeHandlers; + private int _maxConcurrentHandlers; + private int _firstExactRouteHandlerCalls; + private int _replacementExactRouteHandlerCalls; + private int _nullReferenceMessageCalls; + private int _nullNullableValueMessageCalls; + private int _handlerSelectionCalls; + private int _mutatingSelectionCalls; + private bool _deactivateOnNextRecovery; + private readonly HashSet _failedOnce = []; + + public DurableMessagingTestGrain( + IDurableInbox inbox, + IDurableOutbox outbox, + IDurableMessagingDiagnostics diagnostics, + [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("inbox")] IDurableValue applicationInboxState, + [FromKeyedServices("__orleans.durable-messaging.inbox-processed")] IDurableDictionary<(GrainId SenderId, Guid MessageId), DateTimeOffset> processedMessages, + [FromKeyedServices("__orleans.durable-messaging.inbox-job-id")] IDurableValue inboxJobId, + [FromKeyedServices("__orleans.durable-messaging.outbox-job-id")] IDurableValue outboxJobId, + SerializerSessionPool sessions, + ILocalSiloDetails siloDetails, + HandlerProbe handlerProbe, + SnapshotProbe snapshotProbe) + { + _inbox = inbox; + _outbox = outbox; + _diagnostics = diagnostics; + _effects = effects; + ArgumentNullException.ThrowIfNull(applicationInboxState); + _processedMessages = processedMessages; + _inboxJobId = inboxJobId; + _outboxJobId = outboxJobId; + _sessions = sessions; + _siloDetails = siloDetails; + _handlerProbe = handlerProbe; + _snapshotProbe = snapshotProbe; + StateManager.RegisterObserver(this); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + _inbox.RegisterHandler(new ThrowingSelectionHandler(this)); + _inbox.RegisterHandler(new MutatingSelectionHandler(this)); + _inbox.RegisterHandler("nullable/reference", new NullReferenceMessageHandler(this)); + _inbox.RegisterHandler("nullable/value", new NullNullableValueMessageHandler(this)); + _inbox.RegisterHandler(new TypedMessageHandler(this)); + return base.OnActivateAsync(cancellationToken); + } + + public async Task SendAsync(GrainId target, string route, DurableTestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.MessageId; + } + + public async Task SendDuplicateAsync(GrainId target, string route, DurableTestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.MessageId; + } + + public async Task SendAndDeactivateAsync(GrainId target, string route, DurableTestMessage message) + { + var messageId = await SendAsync(target, route, message); + DeactivateOnIdle(); + return messageId; + } + + public Task StageWithoutCommitAsync(GrainId target, string route, DurableTestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + return Task.FromResult(envelope.MessageId); + } + + public async Task DeleteThenWriteStateAsync() + { + await StateManager.DeleteStateAsync(CancellationToken.None); + await WriteStateAsync(); + } + + public async Task RetryWriteStateAsync() => await WriteStateAsync(); + + public async Task RevertStateAsync() => await StateManager.RevertPendingChangesAsync(CancellationToken.None); + + public async Task SetInboxJobIdAsync(string jobId) + { + _inboxJobId.Value = jobId; + await WriteStateAsync(); + } + + public Task DeactivateOnNextRecoveryAsync() + { + _deactivateOnNextRecovery = true; + return Task.CompletedTask; + } + + public Task RegisterDuplicateExactRouteHandlersAsync(string route) + { + var first = new CountingHandler(() => _firstExactRouteHandlerCalls++); + var replacement = new CountingHandler(() => _replacementExactRouteHandlerCalls++); + _inbox.RegisterHandler(route, first); + var exception = GetDuplicateRegistrationException(route, replacement); + var retained = _inbox.TryGetHandler(route, out var cached) && ReferenceEquals(first, cached); + return Task.FromResult(new DuplicateRouteRegistrationResult(exception.Message, retained)); + } + + public Task ValidateRouteLookupAsync(string? route) + { + var hasHandlerParameterName = GetRouteLookupExceptionParameterName(() => _inbox.HasHandler(route!)); + var tryGetHandlerParameterName = GetRouteLookupExceptionParameterName(() => _inbox.TryGetHandler(route!, out _)); + return Task.FromResult(new RouteLookupValidationResult( + hasHandlerParameterName, + tryGetHandlerParameterName)); + } + + public async Task RemoveInboxDeadLetterAsync(GrainId senderId, Guid messageId) + { + if (!_diagnostics.RemoveInboxDeadLetter(senderId, messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public async Task RemoveOutboxDeadLetterAsync(Guid messageId) + { + if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal DurableEndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); + + internal void DeactivateOnNextRecoveryForTest() => _deactivateOnNextRecovery = true; + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + public void OnWriteStarted() + { + } + + public void OnWriteCompleted() => PublishSnapshot(); + public void OnRecoveryCompleted() + { + PublishSnapshot(); + if (_deactivateOnNextRecovery) + { + _deactivateOnNextRecovery = false; + DeactivateOnIdle(); + } + } + + private DurableEnvelope CreateEnvelope(GrainId target, string route, DurableTestMessage message) => + new DurableEnvelopeBuilder(_sessions, this.GetGrainId()) + .To(target, route) + .WithBody(message) + .Build(); + + private async ValueTask HandleAsync( + DurableTestMessage message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + var active = Interlocked.Increment(ref _activeHandlers); + _maxConcurrentHandlers = Math.Max(_maxConcurrentHandlers, active); + try + { + if (_handlerProbe.TryGet(this.GetGrainId(), context.Envelope.RouteKey, out var gate)) + { + gate.Entered.TrySetResult(); + await gate.Continue.Task.WaitAsync(cancellationToken); + } + + _effects.TryGetValue(message.LogicalId, out var prior); + _effects[message.LogicalId] = new DurableEffect( + message.LogicalId, + (prior?.Count ?? 0) + 1, + message.Sequence, + message.Value); + + if (message.CommitDuringHandling) + { + await WriteStateAsync(cancellationToken); + } + + if (message.DeleteDuringHandling) + { + await StateManager.DeleteStateAsync(cancellationToken); + } + + if (message.ForwardTo is { } target) + { + var outgoing = context.CreateEnvelope() + .To(target, "messages/forwarded") + .WithBody(message with { ForwardTo = null, ThrowAfterStaging = false }) + .Build(); + context.Send(outgoing); + } + + if (message.ThrowAfterStaging + || (message.ThrowOnceAfterStaging && _failedOnce.Add(message.LogicalId))) + { + throw new InvalidOperationException($"Injected handler failure for {message.LogicalId}."); + } + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + + private void PublishSnapshot() => _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + + private DurableEndpointSnapshot CreateSnapshot() => + new( + _activationId, + _siloDetails.SiloAddress.ToParsableString(), + _inbox.Count, + _outbox.Count, + _maxConcurrentHandlers, + _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), + _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), + _diagnostics.OutboxDeadLetters.Select(ToSnapshot).ToArray(), + _inboxJobId.Value, + _processedMessages.Count, + _firstExactRouteHandlerCalls, + _replacementExactRouteHandlerCalls, + _outboxJobId.Value, + _nullReferenceMessageCalls, + _nullNullableValueMessageCalls); + + private static DurableDeadLetterSnapshot ToSnapshot(DurableDeadLetter deadLetter) => + new( + deadLetter.Message.MessageId, + deadLetter.Message.RouteKey, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private sealed class TypedMessageHandler(DurableMessagingTestGrain owner) : IInboxHandler + { + bool IInboxHandler.CanHandle(IInboxHandlerContext context) => + context.Envelope.RouteKey.StartsWith("messages/", StringComparison.Ordinal) + || context.Envelope.RouteKey == "typed"; + + public ValueTask HandleAsync( + DurableTestMessage? message, + IInboxHandlerContext context, + CancellationToken cancellationToken) => + owner.HandleAsync( + message ?? throw new InvalidOperationException("A durable test message is required."), + context, + cancellationToken); + } + + private sealed class ThrowingSelectionHandler(DurableMessagingTestGrain owner) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) + { + if (!string.Equals(context.Envelope.RouteKey, "messages/selection-failure", StringComparison.Ordinal)) + { + return false; + } + + if (Interlocked.Increment(ref owner._handlerSelectionCalls) == 2) + { + throw new InvalidOperationException("Injected handler selection failure."); + } + + return true; + } + + public ValueTask HandleAsync( + DurableTestMessage? message, + IInboxHandlerContext context, + CancellationToken cancellationToken) => + owner.HandleAsync( + message ?? throw new InvalidOperationException("A durable test message is required."), + context, + cancellationToken); + } + + private sealed class MutatingSelectionHandler(DurableMessagingTestGrain owner) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) + { + if (!string.Equals(context.Envelope.RouteKey, "messages/selection-mutation", StringComparison.Ordinal)) + { + return false; + } + + if (Interlocked.Increment(ref owner._mutatingSelectionCalls) > 1) + { + var outgoing = context.CreateEnvelope() + .To(context.GrainId, "messages/record") + .WithBody(new DurableTestMessage(Guid.NewGuid(), 81, "selection-side-effect")) + .Build(); + context.Send(outgoing); + } + + return true; + } + + public ValueTask HandleAsync( + DurableTestMessage? message, + IInboxHandlerContext context, + CancellationToken cancellationToken) => + owner.HandleAsync( + message ?? throw new InvalidOperationException("A durable test message is required."), + context, + cancellationToken); + } + + private sealed class NullReferenceMessageHandler(DurableMessagingTestGrain owner) : IInboxHandler + { + public ValueTask HandleAsync( + string? message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + if (message is not null) + { + throw new InvalidOperationException("Expected a null reference message."); + } + + owner._nullReferenceMessageCalls++; + return default; + } + } + + private sealed class NullNullableValueMessageHandler(DurableMessagingTestGrain owner) : IInboxHandler + { + public ValueTask HandleAsync( + int? message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + if (message is not null) + { + throw new InvalidOperationException("Expected a null nullable value message."); + } + + owner._nullNullableValueMessageCalls++; + return default; + } + } + + private sealed class CountingHandler(Action onCall) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) => true; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + onCall(); + return default; + } + } + + private InvalidOperationException GetDuplicateRegistrationException(string route, IInboxHandler replacement) + { + try + { + _inbox.RegisterHandler(route, replacement); + } + catch (InvalidOperationException exception) + { + return exception; + } + + throw new InvalidOperationException("Duplicate exact route registration did not throw."); + } + + private static string GetRouteLookupExceptionParameterName(Func lookup) + { + try + { + lookup(); + } + catch (ArgumentException exception) + { + return exception.ParamName + ?? throw new InvalidOperationException("Invalid route lookup exception did not identify its parameter."); + } + + throw new InvalidOperationException("Invalid route lookup did not throw."); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/HandlerProbe.cs b/test/Orleans.DurableMessaging.Tests/Support/HandlerProbe.cs new file mode 100644 index 00000000000..bb8b710c10b --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/HandlerProbe.cs @@ -0,0 +1,49 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging.Tests.Support; + +public sealed class HandlerProbe +{ + private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); + + public Barrier Arm(GrainId grainId, string route) + { + var barrier = new Barrier(this, (grainId, route)); + if (!_barriers.TryAdd((grainId, route), barrier)) + { + throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); + } + + return barrier; + } + + public bool TryGet(GrainId grainId, string route, out Barrier barrier) => + _barriers.TryGetValue((grainId, route), out barrier!); + + public sealed class Barrier : IDisposable + { + private readonly HandlerProbe _owner; + private readonly (GrainId GrainId, string Route) _key; + + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) + { + _owner = owner; + _key = key; + } + + internal TaskCompletionSource Entered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Continue { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => Continue.TrySetResult(); + + public void Dispose() + { + Release(); + _owner._barriers.TryRemove(new KeyValuePair<(GrainId GrainId, string Route), Barrier>(_key, this)); + } + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbe.cs new file mode 100644 index 00000000000..57a82537992 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbe.cs @@ -0,0 +1,96 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.DurableMessaging.Tests.Support; + +public sealed class SnapshotProbe +{ + private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _waiters = new(); + + internal int WaiterListCount => _waiters.Count; + + public async Task WaitAsync( + GrainId grainId, + Func predicate, + TimeSpan? timeout = null) + { + if (_latest.TryGetValue(grainId, out var current) && predicate(current)) + { + return current; + } + + var waiter = new Waiter(predicate); + List waiters; + while (true) + { + waiters = _waiters.GetOrAdd(grainId, static _ => []); + lock (waiters) + { + if (!_waiters.TryGetValue(grainId, out var currentWaiters) + || !ReferenceEquals(waiters, currentWaiters)) + { + continue; + } + + if (_latest.TryGetValue(grainId, out current) && predicate(current)) + { + RemoveWaiterListIfEmpty(grainId, waiters); + return current; + } + + waiters.Add(waiter); + break; + } + } + + try + { + return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); + } + finally + { + lock (waiters) + { + waiters.Remove(waiter); + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + } + + public void Publish(GrainId grainId, DurableEndpointSnapshot snapshot) + { + _latest[grainId] = snapshot; + if (!_waiters.TryGetValue(grainId, out var waiters)) + { + return; + } + + lock (waiters) + { + foreach (var waiter in waiters.ToArray()) + { + if (waiter.Predicate(snapshot)) + { + waiters.Remove(waiter); + waiter.Completion.TrySetResult(snapshot); + } + } + } + } + + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) + { + if (waiters.Count == 0) + { + _waiters.TryRemove(new KeyValuePair>(grainId, waiters)); + } + } + + private sealed class Waiter(Func predicate) + { + public Func Predicate { get; } = predicate; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbeTests.cs b/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbeTests.cs new file mode 100644 index 00000000000..97b0039e4f4 --- /dev/null +++ b/test/Orleans.DurableMessaging.Tests/Support/SnapshotProbeTests.cs @@ -0,0 +1,88 @@ +using Orleans.Runtime; +using Xunit; + +namespace Orleans.DurableMessaging.Tests.Support; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableMessaging")] +public class SnapshotProbeTests +{ + [Fact] + public async Task WaitAsync_TimeoutRemovesWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "timeout"); + var predicateCalls = 0; + + await Assert.ThrowsAsync( + () => probe.WaitAsync( + grainId, + _ => + { + predicateCalls++; + return false; + }, + TimeSpan.Zero)); + + Assert.Equal(0, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 0)); + + Assert.Equal(0, predicateCalls); + } + + [Fact] + public async Task WaitAsync_SuccessRemovesWaiterList() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "success"); + var wait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + var snapshot = CreateSnapshot(inboxCount: 1); + + probe.Publish(grainId, snapshot); + + Assert.Same(snapshot, await wait); + Assert.Equal(0, probe.WaiterListCount); + } + + [Fact] + public async Task WaitAsync_TimeoutRetainsListWithActiveWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "shared-list"); + var activeWait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + + await Assert.ThrowsAsync( + () => probe.WaitAsync(grainId, static _ => false, TimeSpan.Zero)); + + Assert.Equal(1, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 1)); + + await activeWait; + Assert.Equal(0, probe.WaiterListCount); + } + + private static DurableEndpointSnapshot CreateSnapshot(int inboxCount) => + new( + Guid.Empty, + string.Empty, + inboxCount, + 0, + 0, + [], + [], + [], + null, + 0, + 0, + 0, + null, + 0, + 0); +} diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 4ebe0e5c7cd..07941003165 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -628,11 +628,13 @@ await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask( } [Fact] - public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailures() + public async Task StateManager_WriteStateAsync_FencesWritesAfterRepeatedRecoveryFailures() { var storage = new CapturingStorage(); var sut = CreateTestSystem(storage: storage); var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("first", 1); @@ -652,15 +654,25 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu var secondRecoveryFailure = new IOException("Expected second recovery failure."); storage.NextReadException = secondRecoveryFailure; var recoveryException = await Assert.ThrowsAsync( - () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + () => sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask() .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); Assert.Same(secondRecoveryFailure, recoveryException); - await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase); + var initializeException = await Assert.ThrowsAsync( + () => sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Contains("fenced", initializeException.Message, StringComparison.OrdinalIgnoreCase); + + await sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.True(dictionary.ContainsKey("first")); Assert.False(dictionary.ContainsKey("second")); + Assert.Equal(2, observer.RecoveryCompletedCount); } [Fact] @@ -688,6 +700,172 @@ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() Assert.Equal(1, value.Value); } + [Fact] + public async Task StateManager_RevertAgainstEmptyJournalResetsUncommittedState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + value.Value = 42; + + await sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, value.Value); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(0, recoveredValue.Value); + } + + [Fact] + public async Task StateManager_ObserverPreparationParticipatesInAtomicCommit() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new RecordingStateObserver(() => value.Value = 42); + sut.Manager.RegisterObserver(observer); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(42, recoveredValue.Value); + } + + [Fact] + public async Task StateManager_RegisterObserverAfterInitializationIsRejected() + { + var sut = CreateTestSystem(); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + var exception = Assert.Throws( + () => sut.Manager.RegisterObserver(new RecordingStateObserver())); + + Assert.Contains("after initialization", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StateManager_FailedWriteOmitsObserverCompletionAndRecoveryRestoresState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + value.Value = 1; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + storage.NextAppendException = new InconsistentStateException("Expected write conflict."); + value.Value = 2; + await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(2, observer.WriteStartedCount); + Assert.Equal(1, observer.WriteCompletedCount); + Assert.Equal(2, observer.RecoveryCompletedCount); + Assert.Equal(1, value.Value); + } + + [Fact] + public async Task StateManager_NoOpWritePairsObserverBoundary() + { + var sut = CreateTestSystem(); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); + } + + [Fact] + public async Task StateManager_FinalizationRunsAfterEveryPreparationAndBeforeCapture() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var finalizer = new FinalizingStateObserver(() => Assert.Equal(42, value.Value)); + sut.Manager.RegisterObserver(finalizer); + sut.Manager.RegisterObserver(new RecordingStateObserver(() => value.Value = 42)); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.True(finalizer.Finalized); + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(42, recoveredValue.Value); + } + + [Fact] + public async Task StateManager_RecoveryObserverRunsOnceAfterAllStates() + { + var storage = new CapturingStorage(); + var initial = CreateTestSystem(storage: storage); + var initialFirst = new DurableValue("first", initial.Manager, CreateValueCodec()); + var initialSecond = new DurableValue("second", initial.Manager, CreateValueCodec()); + await initial.Lifecycle.OnStart(TestContext.Current.CancellationToken); + initialFirst.Value = 1; + initialSecond.Value = 2; + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + var sut = CreateTestSystem(storage: storage); + var first = new DurableValue("first", sut.Manager, CreateValueCodec()); + var second = new DurableValue("second", sut.Manager, CreateValueCodec()); + var observer = new RecoveryStateObserver(() => + { + Assert.Equal(1, first.Value); + Assert.Equal(2, second.Value); + }); + sut.Manager.RegisterObserver(observer); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + Assert.Equal(1, observer.RecoveryStartedCount); + Assert.Equal(1, observer.RecoveryCompletedCount); + } + + [Fact] + public async Task StateManager_RecoveryRequestNotifiesObserverBeforeReadStarts() + { + var storage = new BlockingRecoveryStorage(); + var sut = CreateTestSystem(storage: storage); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + var recovery = sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask(); + + Assert.Equal(1, observer.RecoveryRequestedCount); + await storage.RecoveryReadStarted.Task.WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); + storage.AllowRecoveryRead.SetResult(); + await recovery; + } + + [Fact] + public async Task StateManager_DeleteNotifiesObserverAfterSuccessfulDeletion() + { + var sut = CreateTestSystem(); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + await sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, observer.DeleteCompletedCount); + } + [Fact] public async Task StateManager_FencesStateOperationsUntilFailedRevertIsRetriedSuccessfully() { @@ -1991,6 +2169,74 @@ public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationT public ValueTask DeleteAsync(CancellationToken cancellationToken) => default; } + private sealed class RecordingStateObserver(Action? prepare = null) : IJournaledStateObserver + { + public List WriteCalls { get; } = []; + public int WriteStartedCount { get; private set; } + public int WriteCompletedCount { get; private set; } + public int RecoveryCompletedCount { get; private set; } + public int DeleteCompletedCount { get; private set; } + public int RecoveryStartedCount { get; private set; } + public int RecoveryRequestedCount { get; private set; } + + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + WriteCalls.Add("Preparing"); + prepare?.Invoke(); + return default; + } + + public void OnWriteStarted() + { + WriteCalls.Add("Started"); + WriteStartedCount++; + } + + public void OnWriteCompleted() + { + WriteCalls.Add("Completed"); + WriteCompletedCount++; + } + + public void OnRecoveryCompleted() => RecoveryCompletedCount++; + public void OnRecoveryRequested() => RecoveryRequestedCount++; + public void OnRecoveryStarted() => RecoveryStartedCount++; + public void OnDeleteCompleted() => DeleteCompletedCount++; + } + + private sealed class FinalizingStateObserver(Action finalize) : IJournaledStateObserver + { + public bool Finalized { get; private set; } + + public ValueTask OnWriteFinalizingAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + finalize(); + Finalized = true; + return default; + } + + public void OnWriteStarted() { } + public void OnWriteCompleted() { } + public void OnRecoveryCompleted() { } + } + + private sealed class RecoveryStateObserver(Action recovered) : IJournaledStateObserver + { + public int RecoveryStartedCount { get; private set; } + public int RecoveryCompletedCount { get; private set; } + + public void OnWriteStarted() { } + public void OnWriteCompleted() { } + public void OnRecoveryStarted() => RecoveryStartedCount++; + public void OnRecoveryCompleted() + { + recovered(); + RecoveryCompletedCount++; + } + } + private sealed class CapturingStorage : IJournalStorage { private readonly object _lock = new();