diff --git a/.specify/feature.json b/.specify/feature.json index b61121639..ded451f4d 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1,3 @@ { - "feature_directory": "specs/091-domain-owned-apis" + "feature_directory": "specs/091-groundwork-diagnostics-persistence" } diff --git a/AGENTS.md b/AGENTS.md index 40189611a..cc4ae44f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,5 +161,5 @@ New work should move toward this rule: For additional context about technologies, project structure, shell commands, contracts, and validation scenarios for the active work unit, read -`specs/092-domain-owned-apis/plan.md`. +`specs/091-groundwork-diagnostics-persistence/plan.md`. diff --git a/Directory.Packages.props b/Directory.Packages.props index c98359b29..45ece2d75 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -19,11 +19,12 @@ - - - - - + + + + + + diff --git a/EXTENSION_POINTS.md b/EXTENSION_POINTS.md index 7fb753987..20ce59db8 100644 --- a/EXTENSION_POINTS.md +++ b/EXTENSION_POINTS.md @@ -157,6 +157,7 @@ not automatic constitution violations. | Domain | Catalog | |---|---| +| Diagnostics persistence lifecycle (provider-neutral drain, bridge, observer, and replacement registration) | [`src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md`](src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md) | | Structured logs (log store, live feed, sink, source provider) | [`src/Elsa/Diagnostics/StructuredLogs/EXTENSION_POINTS.md`](src/Elsa/Diagnostics/StructuredLogs/EXTENSION_POINTS.md) | | OpenTelemetry (telemetry store, live feed, ingestor, redactor, source registry, provider, collector config) | [`src/Elsa/Diagnostics/OpenTelemetry/EXTENSION_POINTS.md`](src/Elsa/Diagnostics/OpenTelemetry/EXTENSION_POINTS.md) | diff --git a/Elsa.Server.slnx b/Elsa.Server.slnx index 075e3ba02..3871d869d 100644 --- a/Elsa.Server.slnx +++ b/Elsa.Server.slnx @@ -68,12 +68,15 @@ + - + + + @@ -316,6 +319,9 @@ + + + @@ -325,6 +331,9 @@ + + + diff --git a/NuGet.config b/NuGet.config index 4d7ea3e72..49a8e8e35 100644 --- a/NuGet.config +++ b/NuGet.config @@ -17,6 +17,7 @@ + @@ -44,7 +45,9 @@ + + @@ -58,6 +61,7 @@ + diff --git a/docs/maps/extension-point-map.md b/docs/maps/extension-point-map.md index a40beaaa3..cef221096 100644 --- a/docs/maps/extension-point-map.md +++ b/docs/maps/extension-point-map.md @@ -6,9 +6,9 @@ Records Markdown catalog facts from `EXTENSION_POINTS.md` files. It does not val ## Summary -- Catalog files discovered: 58 -- Source catalogs discovered: 57 -- Source catalogs indexed from root: 57 +- Catalog files discovered: 59 +- Source catalogs discovered: 58 +- Source catalogs indexed from root: 58 - Discovered source catalogs not linked from root index: 0 - Root-indexed catalogs missing on disk: 0 @@ -40,6 +40,7 @@ Records Markdown catalog facts from `EXTENSION_POINTS.md` files. It does not val | [src/Elsa/Api/FastEndpoints/EXTENSION_POINTS.md](../../src/Elsa/Api/FastEndpoints/EXTENSION_POINTS.md) | Elsa.Api.FastEndpoints | Elsa.Api | Feature-inheritance point
Secure-by-construction endpoint base classes
Implementable contributor interfaces
Per-shell security feature | `FastEndpointsFeatureBase`
`IFastEndpointsConfigurator` (CShells contract)
`IFastEndpointFilter`
`ApiSecurityFeature` / `ApiSecurityOptions` | Feature inheritance point (abstract base implementing CShells `IFastEndpointsShellFeature`).
Contributor (fan-in; every registered configurator is applied when the shell maps its FastEndpoints).
Contributor (fan-in; excludes endpoints from a shell's mapped set).
Shell feature with bindable options (`AllowAnonymous`, default `false`; `ShellName` is populated from the shell settings). | 0 | yes | | [src/Elsa/Caching/Memory/EXTENSION_POINTS.md](../../src/Elsa/Caching/Memory/EXTENSION_POINTS.md) | Elsa.Caching.Memory | Elsa.Caching | Overridable contracts | `ICacheManager` *(Core — `Elsa.Caching.Core`)*
`IChangeTokenSignaler` *(Core — `Elsa.Caching.Core`)*
`IChangeTokenSignalInvoker` *(Core — `Elsa.Caching.Core`)* | 0 | yes | | | [src/Elsa/Diagnostics/OpenTelemetry/EXTENSION_POINTS.md](../../src/Elsa/Diagnostics/OpenTelemetry/EXTENSION_POINTS.md) | Elsa.Diagnostics.OpenTelemetry | Elsa.Diagnostics | Overridable contracts
Ingestion & transport (not extension points)
Deferred
Notes | `IOpenTelemetryStore` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`IOpenTelemetryLiveFeed` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`IOpenTelemetryIngestor` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`IOpenTelemetryRedactor` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`IOpenTelemetrySourceRegistry` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`IOpenTelemetryProvider` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)*
`ICollectorConfigurationProvider` *(Core — `Elsa.Diagnostics.OpenTelemetry.Core`)* | 0 | yes | | +| [src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md](../../src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md) | Elsa.Diagnostics.Persistence | Elsa.Diagnostics | Overridable contracts
Implementable contributor interfaces
Events
Production bridges and registration | `IDiagnosticsDrainTarget` *(Feature contract — `Elsa.Diagnostics.Persistence`)*
`IDiagnosticsPersistenceObserver` *(Feature contract — `Elsa.Diagnostics.Persistence`)* | Adapter / Bridge; single Replacement.
Adapter / Bridge; single Replacement. | 0 | yes | | [src/Elsa/Diagnostics/StructuredLogs/EXTENSION_POINTS.md](../../src/Elsa/Diagnostics/StructuredLogs/EXTENSION_POINTS.md) | Elsa.Diagnostics.StructuredLogs | Elsa.Diagnostics | Overridable contracts
Persistence
Notes | `IStructuredLogStore` *(Core — `Elsa.Diagnostics.StructuredLogs.Core`)*
`IStructuredLogLiveFeed` *(Core — `Elsa.Diagnostics.StructuredLogs.Core`)*
`IStructuredLogLivePublisher` *(Core — `Elsa.Diagnostics.StructuredLogs.Core`)*
`IStructuredLogSink` *(Core — `Elsa.Diagnostics.StructuredLogs.Core`)*
`IStructuredLogSourceProvider` *(Core — `Elsa.Diagnostics.StructuredLogs.Core`)*
Groundwork diagnostic records
Temporary EF Core compatibility override | 0 | yes | | | [src/Elsa/Events/EXTENSION_POINTS.md](../../src/Elsa/Events/EXTENSION_POINTS.md) | Elsa.Events | Elsa.Events | Implementable contributor interfaces
Publishing strategies | `IEventHandler` *(Core — `Elsa.Events.Core`)* where `T : IEvent`
`IEvent` *(Core — `Elsa.Events.Core`)*
`IEventMiddleware` *(Core — `Elsa.Events.Core`)* | Contributor (event subscriber — handles a specific event type).
Marker interface — implement to define an event type. Typically a `sealed record`.
Contributor (event pipeline middleware). Composes the event dispatch pipeline. | 1 | yes | | [src/Elsa/Expressions/Api/EXTENSION_POINTS.md](../../src/Elsa/Expressions/Api/EXTENSION_POINTS.md) | Elsa.Expressions.Api | Elsa.Expressions | Descriptor Sources | 0 | yes | | | diff --git a/specs/091-groundwork-diagnostics-persistence/checklists/requirements.md b/specs/091-groundwork-diagnostics-persistence/checklists/requirements.md new file mode 100644 index 000000000..adf10fd74 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Durable Diagnostics Persistence + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-13 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details beyond the ratified persistence boundary and named program dependency +- [x] Focused on operator, host-owner, and maintainer outcomes +- [x] Written so behavioral requirements can be reviewed independently of code structure +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No `[NEEDS CLARIFICATION]` markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria describe externally verifiable outcomes +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] Technology names appear only where they define the ratified provider boundary or a removal criterion + +## Notes + +- Ready for implementation planning. The existing diagnostics workload report remains the canonical detailed inventory; this specification links behavior into one executable work unit rather than duplicating that inventory. diff --git a/specs/091-groundwork-diagnostics-persistence/contracts/diagnostics-persistence.md b/specs/091-groundwork-diagnostics-persistence/contracts/diagnostics-persistence.md new file mode 100644 index 000000000..c83984e41 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/contracts/diagnostics-persistence.md @@ -0,0 +1,49 @@ +# Contract: Diagnostics Persistence Mapping + +This contract maps Elsa-owned operations to Groundwork capabilities. It is a behavioral contract, not a public promise of Groundwork types in Elsa core assemblies. + +## Structured Logs + +| Elsa operation | Required behavior | Groundwork realization | +|---|---|---| +| Append | Durable, ordered, idempotent batch commit; authoritative cursor | Append diagnostic records under explicit binding and operation identity | +| Read recent | Exact filters, bounded newest selection, oldest-to-newest response | Indexed bounded record query with deterministic tie-breaks | +| Capture after cursor | Binding validation, stable commit order, exact-once page, bounded advancement | Provider position query plus opaque Elsa cursor codec | +| Trim | Keep exact newest count, including zero; high-water never rewinds | Idempotent bounded record retention operation | +| Inspect/readiness | Visible schema/capability/operational failures | Groundwork validation plus adapter binding checks | + +Cursor-unavailable is reserved for malformed, trimmed, or foreign cursors. Cancellation, schema, provider, and serialization failures must not be translated to cursor-unavailable or empty results. + +## OpenTelemetry + +| Elsa operation | Required behavior | Groundwork realization | +|---|---|---| +| Write normalized batch | Nonblocking capture followed by durable, idempotent acknowledgement | Elsa drain plus one or more transactional/bounded Groundwork operations | +| Query resources | Scoped filters and deterministic bounded catalog result | Groundwork document query over resource catalog | +| Query traces | Inclusive ranges, exact filters, stable ordering and limits | Indexed diagnostic-record query over trace summaries | +| Get trace | Exact summary, spans, resources, and related logs | Bounded coordinated queries by scoped trace/resource identities | +| Query metrics | Instrument catalog plus filtered points and latest-per-key semantics | Document catalog query plus indexed metric-point record query | +| Query logs | Exact declared filters, ordering, and limit | Indexed telemetry-log record query | +| Get diagnostics | Exact counts/high-water/drop information | Scoped Groundwork inspection plus Elsa lifecycle counters | +| Catalog retention | Deterministic least-recently-seen capacity | Bounded document mutation ordered by last-seen/tie-breaker | + +## Composition and deployment + +- The host selects one Groundwork provider and supplies provider-neutral store/session access. +- Each concrete diagnostics Groundwork adapter owns and contributes its schema declaration; the shared Elsa drain/lifecycle project remains Groundwork-free. +- Schema validation and application can run before application startup through the shared Groundwork CLI/tooling. +- Readiness fails on missing schema, drift, capability mismatch, or unavailable storage. +- Registration replaces only `IStructuredLogStore` and `IOpenTelemetryStore`; live feeds, redaction, parsing, authorization, and domain policy stay Elsa-owned. +- Provider choice must not create provider-specific Elsa store classes or EF migration sets. + +## Cross-provider conformance + +SQLite, SQL Server, PostgreSQL, and MongoDB must execute the same fixture and assertions for ordering, filters, inclusive boundaries, latest selection, exact counts, retention, scope isolation, restart, concurrency, cancellation, acknowledgement loss, operation-identity conflict, readiness, and bounded execution evidence. + +## Removal gate + +Diagnostics EF projects may be deleted only when all shared conformance tests, provider readiness tests, execution-plan checks, lifecycle tests, and the ratified performance matrix pass. The final architecture test must prove: + +1. no diagnostics project references EF Core or an EF diagnostics project; +2. no diagnostics EF context/entity/configuration/migration/registration remains; and +3. diagnostics core projects do not reference Groundwork. diff --git a/specs/091-groundwork-diagnostics-persistence/data-model.md b/specs/091-groundwork-diagnostics-persistence/data-model.md new file mode 100644 index 000000000..b5df42390 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/data-model.md @@ -0,0 +1,80 @@ +# Data Model: Durable Diagnostics Persistence + +All identities below are scoped by an explicit `StorageScope`. Provider table/collection names are resolved by Groundwork naming and routing policy; Elsa does not embed provider-specific identifiers in its public contracts. + +## Storage binding + +- `StorageScope`: opaque host-selected isolation boundary +- `SourceId`: diagnostics source binding +- `StreamKind`: structured logs, traces, spans, metric points, or telemetry logs +- `StreamId`: stable logical stream identity derived from scope, source, and kind +- `SchemaVersion`: declared adapter schema version + +Validation rejects missing scope/source, unsupported schema, and a cursor or operation identity bound to a different tuple. + +## Immutable record streams + +### Common record envelope + +- Provider-issued durable sequence/position +- Operation identity and item identity for idempotency +- Commit timestamp +- Occurrence timestamp +- Stable tie-break fields required by the domain query +- Filterable metadata declared by the adapter +- Canonical payload + +### Structured Log record + +Adds logical display sequence, level, category, event, source, and correlation filter fields. The opaque Elsa replay cursor binds version, scope, source, stream, provider position, and record anchor; it exposes none of those values as a supported parsing contract. + +### Trace summary and span records + +Trace summary records support bounded latest/history queries. Span records support exact trace-detail lookup by trace identity, with stable span ordering. + +### Metric point records + +Metric points bind an instrument identity and support inclusive time ranges plus stable latest-per-logical-key selection where required by the existing contract. + +### Telemetry log records + +Telemetry logs support inclusive time, severity, trace, resource, and text-related filters exactly as declared by `OpenTelemetryLogFilter`. + +## Mutable catalogs + +### Resource catalog entry + +- Scoped resource identity +- Resource attributes/payload +- `LastSeenUtc` +- Stable last-seen tie-breaker +- Revision/concurrency token + +### Instrument catalog entry + +- Scoped instrument identity +- Resource identity +- Name, unit, description, and instrument metadata +- `LastSeenUtc` +- Stable last-seen tie-breaker +- Revision/concurrency token + +Catalog upsert is idempotent. Capacity enforcement keeps exactly the newest configured entries by `(LastSeenUtc, tie-breaker)` within one scope. Capacity zero removes every current entry without changing immutable history high-water metadata. + +## Capture batch and operation ledger + +A capture batch contains one bounded normalized set for one storage binding. Its operation identity is stable across retries and restart. Reusing the identity for different canonical input fails. A committed operation returns the original authoritative outcome after acknowledgement loss. + +Retention operations use the same identity/fingerprint rule and record exact affected counts. Lifetime high-water and cursor sequence never rewind when records are trimmed. + +## Lifecycle state + +The Elsa drain moves through `Created -> Running -> Closing -> Draining -> Stopped/TimedOut`. Producers are nonblocking while running. Every accepted acknowledgement completes exactly once as committed or failed. Loss counters distinguish queue overflow, retry exhaustion, shutdown timeout, writes after closure, durable retention deletion, and subscriber delivery loss. + +## Relationships + +- One storage scope owns many source/stream bindings. +- One resource catalog entry can be referenced by traces, spans, metric instruments, metric points, and telemetry logs. +- One instrument catalog entry owns many metric point records. +- One trace summary owns many span records and may relate to telemetry log records. +- One replay position belongs to exactly one structured-log binding. diff --git a/specs/091-groundwork-diagnostics-persistence/evidence/red-test-baseline.md b/specs/091-groundwork-diagnostics-persistence/evidence/red-test-baseline.md new file mode 100644 index 000000000..a1734d1b2 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/evidence/red-test-baseline.md @@ -0,0 +1,58 @@ +# User Story 1 Red-Test Baseline + +Recorded at `2026-07-14T00:47:54Z`, before implementing T016-T021. + +- Branch head: `9d0218ab8cfa439f390253ad90032a578b34ff67` +- `origin/main`: `b7ad33fcc005c256011e905687f0f31286915f93` +- Merge base: `b7ad33fcc005c256011e905687f0f31286915f93` +- Configuration: `Release`, restored dependency graph reused with `--no-restore` +- Working tree under test: T012-T014 test changes plus the task ledger; no T016-T021 production implementation changes + +## T012 — Structured Logs replay conformance + +```text +dotnet test tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork.Tests.csproj -c Release --no-restore --filter FullyQualifiedName~GroundworkStructuredLogReplayTests --logger 'console;verbosity=minimal' +``` + +Result: 6 discovered, 5 passed, 1 failed. + +The only failure was +`GroundworkStructuredLogReplayTests.Operational_query_failure_is_not_reported_as_cursor_unavailable`. +It expected an exact `StructuredLogsException`, but the current adapter translated the provider +failure to `StructuredLogReplayCursorUnavailableException`. This is the intended T017 red: invalid, +foreign, and trimmed cursors must remain non-disclosing, while operational query failures must remain +distinguishable. + +## T013 — OpenTelemetry restart conformance + +```text +dotnet test tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj -c Release --no-restore --filter FullyQualifiedName~GroundworkOpenTelemetryRestartTests --logger 'console;verbosity=minimal' +``` + +Result: 1 discovered, 0 passed, 1 failed. + +The only failure was +`GroundworkOpenTelemetryRestartTests.Durable_state_survives_store_restart_for_every_signal_kind` +with the explicit message: + +```text +Expected T013 red: T020 has not implemented or wired GroundworkOpenTelemetryStore. +``` + +The test compiles against `IOpenTelemetryStore` and already exercises resources, trace summaries, +spans, instruments, metric points, and log records across a store restart. T020 must replace the +failing factory with the real Groundwork adapter without weakening the contract. + +## T014 — Durable operation conformance + +```text +dotnet test tests/Elsa/Diagnostics/Persistence/Tests/Elsa.Diagnostics.Persistence.Tests.csproj -c Release --no-restore --filter FullyQualifiedName~DiagnosticsDurableOperationConformanceTests --logger 'console;verbosity=minimal' +``` + +Result: 5 discovered, 5 passed, 0 failed. + +The specialized SQLite Groundwork diagnostic-record primitive already satisfies the prerequisite +append idempotency, operation-identity conflict, acknowledgement-loss, cancellation-boundary, +concurrent-writer, malformed-payload, and oversized-batch rejection contract. This green baseline is +intentional: T016-T021 may build adapters on that primitive, and T022 will generalize the evidence +across all required providers. diff --git a/specs/091-groundwork-diagnostics-persistence/oracle-inventory.md b/specs/091-groundwork-diagnostics-persistence/oracle-inventory.md new file mode 100644 index 000000000..daf5c42f4 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/oracle-inventory.md @@ -0,0 +1,44 @@ +# Temporary EF diagnostics behavior oracle + +This inventory is a deletion aid, not a second contract. The provider-neutral spec and conformance tests remain authoritative; EF implementations stay temporarily so each behavior below has a concrete parity oracle until T053–T055. + +| Behavior | EF source/oracle | Provider-neutral destination | +|---|---|---| +| Bounded nonblocking queue, drop-oldest overflow, single reader | `ChannelDrainingStoreBase` plus both resilience suites | `DiagnosticsDrainLoadTests` | +| Stable retry batch, nine bounded attempts, exponential backoff | `ChannelDrainingStoreBase` | `DiagnosticsDrainLifecycleTests` and failure fixtures | +| Retry exhaustion accounts loss and later batches recover | Both EF resilience suites | `DiagnosticsDrainLoadTests` | +| Explicit start, graceful completion, tolerant partial-start shutdown | EF startup tasks, shell terminators, and store tests | `DiagnosticsDrainLifecycleTests` / `DiagnosticsDrainShutdownTests` | +| Final retention after the queued tail | `CompleteDrainingCoreAsync` | `DiagnosticsDrainShutdownTests` | +| Bounded async-disposal fallback | `ChannelDrainingStoreBase.DisposeAsync` | `DiagnosticsDrainShutdownTests` | +| Accepted Structured Log acknowledgement completes on commit or explicit failure | `EfCoreStructuredLogStore` pending-completion registry | Shared drain acknowledgement assertions | +| Structured Log committed result carries the authoritative replay cursor | `EfCoreStructuredLogStore.PersistBatchAsync` | Structured Logs Groundwork conformance (T012–T017) | +| OpenTelemetry dropped counts remain queryable | `EfCoreOpenTelemetryStore` counters | Shared observability plus OpenTelemetry adapter mapping (T037/T041) | +| Provider logging cannot recursively enter the captured diagnostics stream | EF feature `NullLoggerFactory` wiring | Pull-only shared counters and adapter-specific instrumentation exclusion | +| Subscriber backpressure is a live-feed loss, not persistence overflow | `InMemoryStructuredLogLiveFeed` and `InMemoryOpenTelemetryLiveFeed` | Existing domain drop signals mapped to `SubscriberDelivery` without moving fan-out | + +No current EF test is deleted or weakened in this slice. The provider-specific query, replay, retention, composition, and migration oracles remain mapped by T012–T057 before deletion. + +## Exact temporary evidence paths + +These paths are the concrete deletion checklist for T010. A later task may add stronger Groundwork +evidence, but it must not remove an EF oracle below until its mapped conformance behavior is green. + +| Area | Structured Logs EF evidence | OpenTelemetry EF evidence | +|---|---|---| +| Shared draining implementation | `src/Elsa/Persistence/EFCore/Storage/ChannelDrainingStoreBase.cs` | `src/Elsa/Persistence/EFCore/Storage/ChannelDrainingStoreBase.cs` | +| Drain resilience tests | `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStoreResilienceTests.cs` | `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/EfCoreOpenTelemetryStoreResilienceTests.cs` | +| Drain startup task | `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Tasks/StartStructuredLogDrainingStartupTask.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Tasks/StartOpenTelemetryDrainingStartupTask.cs` | +| Drain shell terminator | `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Tasks/StopStructuredLogDrainingShellTerminator.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Tasks/StopOpenTelemetryDrainingShellTerminator.cs` | +| Subscriber-loss live feed | `src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs` | +| Query and replay implementation | `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Storage/EfCoreStructuredLogStore.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Storage/EfCoreOpenTelemetryStore.cs` | +| Query and replay tests | `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStoreTests.cs` (`GetRecent*`, `ReplayPages*`, `ReadAfter*`) | `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/EfCoreOpenTelemetryStoreTests.cs` (`QueryTraces*`, `QueryMetricsAndLogs*`) | +| Retention and retry tests | `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStoreTests.cs` (`TrimToZero*`, `DrainPrunes*`, `CompleteDrainingPrunes*`) and `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStorePruneRetryTests.cs` | `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/EfCoreOpenTelemetryStoreTests.cs` (`DrainPrunesHighVolumeSignalsToConfiguredCapacities`) | +| Composition implementation | `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/EFCoreStructuredLogsPersistenceFeatureBase.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/EFCoreOpenTelemetryPersistenceFeatureBase.cs` | +| Composition tests | `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/SqliteStructuredLogsPersistenceFeatureTests.cs` | `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/SqliteOpenTelemetryPersistenceFeatureTests.cs` | +| Migration/schema implementation | `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Sqlite/Migrations/20260618004216_Initial.cs` and `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Sqlite/Migrations/StructuredLogsDbContextModelSnapshot.cs` | `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Sqlite/Migrations/20260623005000_Initial.cs` and `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Sqlite/Migrations/OpenTelemetryDbContextModelSnapshot.cs` | + +The exact feature tests above are the closest composition/migration-startup evidence: each SQLite +feature inherits the EF migration startup path and separately proves the store, lifecycle tasks, and +DbContext factory registrations. There is no dedicated migration-shape test today; the migration and +model-snapshot files therefore remain explicit source oracles until provider schema validation replaces +them. diff --git a/specs/091-groundwork-diagnostics-persistence/plan.md b/specs/091-groundwork-diagnostics-persistence/plan.md new file mode 100644 index 000000000..2cfba71d4 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/plan.md @@ -0,0 +1,102 @@ +# Implementation Plan: Durable Diagnostics Persistence + +**Branch**: `091-groundwork-diagnostics-persistence` | **Date**: 2026-07-13 | **Spec**: [spec.md](spec.md) + +**Input**: Replace the Structured Logs and OpenTelemetry EF Core implementations with Groundwork implementations while preserving provider-neutral diagnostics contracts. + +## Summary + +Deliver one first-party Groundwork persistence family for Structured Logs and OpenTelemetry. High-volume immutable history uses Groundwork diagnostic-record streams; mutable resource and instrument catalogs use bounded Groundwork document storage. Elsa continues to own capture, retry, overload, shutdown, and live-delivery policy through a shared diagnostics drain component. The work proves exact behavior and bounded provider execution on SQLite, SQL Server, PostgreSQL, and MongoDB before removing every diagnostics EF Core project, registration, package, context, entity, configuration, and migration. + +## Technical Context + +**Language/Version**: C# / .NET 10 (`net10.0`) + +**Primary Dependencies**: Elsa diagnostics core contracts, Groundwork diagnostic-record and document packages, Microsoft.Extensions dependency injection/options/hosting + +**Storage**: Groundwork diagnostic-record streams plus ordinary documents on SQLite, SQL Server, PostgreSQL, and MongoDB + +**Testing**: xUnit, shared provider-neutral conformance suites, provider integration fixtures/containers, architecture and dependency tests, deterministic performance oracle + +**Target Platform**: Cross-platform ASP.NET Core hosts and pre-start deployment pipelines + +**Project Type**: Modular .NET libraries with host-composed persistence adapters + +**Performance Goals**: Groundwork p95 no worse than 1.25x the ratified EF oracle, throughput at least 80% of the oracle, and p99 no worse than 2x for the approved matrix + +**Constraints**: Core diagnostics assemblies remain free of Groundwork; all scale-bearing operations execute provider-side; explicit scope on every operation; non-recursive, payload-free instrumentation; bounded queues and shutdown; no production EF data migration + +**Scale/Scope**: Structured Logs plus OpenTelemetry resources, traces, spans, metric instruments, metric points, and log records across four providers, including schema validation/application and complete diagnostics EF removal + +## Constitution Check + +The constitution is still draft/provisional. This plan treats it as the current quality gate without claiming ratification. + +| Gate | Result | Design consequence | +|---|---|---| +| Domain core, default behavior, and infrastructure stay distinct | PASS | Existing `.Core` contracts remain provider-neutral; Groundwork references live only in persistence projects. | +| Domain-owned vocabulary and extension seams | PASS | Elsa exposes diagnostics contracts and lifecycle policy; Groundwork supplies infrastructure primitives rather than Elsa-specific business contracts. | +| One contract, one replacement path | PASS | Each Groundwork adapter replaces the existing store contract through existing composition seams; registration tests reject ambiguous defaults. | +| Provider-independent invariants | PASS | A single highest-seam suite defines ordering, scope, restart, idempotency, retention, and failure semantics for all providers. | +| Provider decomposition | PASS | Elsa adapters depend on provider-neutral Groundwork capabilities; hosts supply provider sessions/factories and schema tooling. | +| CQS and explicit operation semantics | PASS | Reads do not silently mutate; append, catalog upsert, and retention are explicit commands with authoritative outcomes. | +| Golden-test continuity before replacement | PASS | EF and in-memory behavior are retained as temporary oracles until conformance and performance gates pass. | +| Feature and branch-covered tests | PASS | Composition, disabled/enabled branches, readiness, shutdown, and provider selection receive explicit tests. | +| Public feature/nonsealed and logic/sealed guidance | PASS | Activatable feature types remain extensible; concrete infrastructure logic is sealed unless a documented seam requires otherwise. | +| Wrapped infrastructure failures | PASS | Operational failures remain visible through domain-facing exceptions; cursor invalidity is the only unavailable-cursor path. | +| Singleton exception requires ownership and lifecycle proof | PASS WITH DOCUMENTED EXCEPTION | The shared Elsa drain is singleton per host because it owns a bounded channel and host lifecycle. Tests prove one instance, explicit start/stop, drain-before-dispose, and complete acknowledgements. | + +Post-design re-check: the proposed contracts preserve the same boundaries. No constitution violation requires a waiver. + +## Project Structure + +### Documentation (this feature) + +```text +specs/091-groundwork-diagnostics-persistence/ +├── spec.md +├── plan.md +├── research.md +├── data-model.md +├── quickstart.md +├── contracts/ +│ └── diagnostics-persistence.md +└── tasks.md +``` + +### Source Code (repository root) + +```text +src/Elsa/Diagnostics/ +├── Persistence/ +│ └── Elsa.Diagnostics.Persistence.csproj +├── StructuredLogs/ +│ └── Persistence/Groundwork/ +└── OpenTelemetry/ + ├── Endpoints/OpenTelemetry/Logs/ + └── Persistence/Groundwork/ + +tests/Elsa/Diagnostics/ +├── Persistence/Tests/ +├── StructuredLogs/Persistence/Groundwork/Tests/ +├── StructuredLogs/Persistence/Conformance/Tests/ +├── OpenTelemetry/Persistence/Groundwork/Tests/ +└── OpenTelemetry/Persistence/Conformance/Tests/ +``` + +The exact conformance-project split may reuse existing test projects when that keeps references acyclic; the invariant is one reusable behavior corpus exercised by every supported provider. + +**Structure Decision**: Add a small Groundwork-free diagnostics-owned persistence helper project for shared capture/drain lifecycle, keep each Groundwork schema declaration and concrete adapter under its diagnostics domain, and keep core projects independent. Provider fixtures and conformance tests remain in `tests/Elsa/Diagnostics` so EF oracles can be removed without deleting the behavior authority. + +## Delivery Phases + +1. Extract and certify the Elsa-owned bounded drain/lifecycle component without changing store behavior. +2. Harden the existing Structured Logs Groundwork adapter against the full replay, scope, retention, restart, and failure contract. +3. Implement OpenTelemetry record streams and bounded resource/instrument catalogs, including the missing logs endpoint. +4. Add schema declaration, validation/application, readiness, composition, and four-provider conformance. +5. Run the ratified EF-versus-Groundwork performance matrix and resolve material regressions. +6. Remove diagnostics EF implementations and prove zero EF dependencies inside the diagnostics surface. + +## Complexity Tracking + +No constitution violation is accepted. The extra shared persistence project is a domain-owned lifecycle component used by two diagnostics adapters, not a new storage abstraction; it prevents the current EF-namespaced drain implementation from being copied into each Groundwork adapter. diff --git a/specs/091-groundwork-diagnostics-persistence/quickstart.md b/specs/091-groundwork-diagnostics-persistence/quickstart.md new file mode 100644 index 000000000..25dfc41ae --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/quickstart.md @@ -0,0 +1,44 @@ +# Quickstart: Durable Diagnostics Persistence + +## Planning validation + +```bash +rg -n "NEEDS CLARIFICATION|\[FEATURE|\[###|TODO|TBD" specs/091-groundwork-diagnostics-persistence +``` + +## Focused test progression + +```bash +dotnet test tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork.Tests.csproj +dotnet test tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj +``` + +The second project is created by this work unit. Add shared conformance and lifecycle projects to this list as the task phase fixes their exact paths. + +## Required provider certification + +Run the same highest-seam diagnostics fixture against: + +1. SQLite on a real database file; +2. SQL Server in the pinned integration container; +3. PostgreSQL in the pinned integration container; and +4. MongoDB in the pinned replica-set integration container. + +For each provider, retain test evidence for schema validate/apply, restart, concurrent writers, acknowledgement loss, exact retention, cross-scope isolation, and bounded plans for scale-bearing operations. + +## Performance gate + +Use the ratified EF-versus-Groundwork diagnostics workload and dataset, not a smoke profile, for promotion. Groundwork passes only when p95 is at most 1.25x EF, throughput is at least 80% of EF, and p99 is at most 2x EF. Record environment, provider versions, schema digest, workload digest, sample counts, database work/round trips, and raw artifacts. + +## Final dependency audit + +```bash +rg -n "EntityFrameworkCore|Persistence\.EFCore|DbContext|Migration" src/Elsa/Diagnostics tests/Elsa/Diagnostics +rg -n "Groundwork" src/Elsa/Diagnostics/*/Core +``` + +The first command may match temporary oracle code until the removal phase; it must have no diagnostics EF implementation/dependency matches at completion. The second command must remain empty throughout. + +## Delivery rule + +Work on the feature branch, implement by user-story slice with tests first, obtain an independent review, push the organization-owned branch, merge the reviewed PR, and leave the repository with all required checks green. diff --git a/specs/091-groundwork-diagnostics-persistence/research.md b/specs/091-groundwork-diagnostics-persistence/research.md new file mode 100644 index 000000000..ba946006d --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/research.md @@ -0,0 +1,83 @@ +# Research: Durable Diagnostics Persistence + +## Decision 1: Keep diagnostics core provider-neutral + +**Decision**: Preserve `IStructuredLogStore` and `IOpenTelemetryStore` as Elsa-owned contracts. Groundwork packages appear only in concrete persistence projects. + +**Rationale**: The host must be able to replace infrastructure without changing the diagnostics domain. This also keeps a later third-party EF implementation possible without carrying EF in the foundation repository. + +**Alternatives rejected**: Exposing Groundwork sessions or records from core contracts would make infrastructure vocabulary part of the domain API. + +## Decision 2: Use record streams for immutable history and documents for catalogs + +**Decision**: Store Structured Logs, trace summaries, spans, metric points, and telemetry log records in append-oriented diagnostic-record streams. Store resources and metric instruments as bounded keyed documents with explicit last-seen metadata. + +**Rationale**: Immutable signal history requires stable append order, replay positions, exact trimming, and idempotent append. Catalogs require keyed replacement and deterministic capacity enforcement rather than append replay. + +**Alternatives rejected**: Modeling every signal as an ordinary document loses the specialized replay/trim semantics. Modeling catalogs as streams makes current-state lookup and capacity enforcement unnecessarily indirect. + +## Decision 3: Extract one Elsa-owned drain lifecycle component + +**Decision**: Move the reusable bounded-channel, retry, acknowledgement, overload, and shutdown mechanism out of the EF namespace into `Elsa.Diagnostics.Persistence`. Adapters compose it rather than inherit provider-specific policy. + +**Rationale**: Those semantics belong to diagnostics capture, not EF or Groundwork. One implementation prevents policy drift between Structured Logs and OpenTelemetry. + +**Alternatives rejected**: Copying the existing EF base into each adapter repeats subtle lifecycle code. Moving the policy into Groundwork would invert domain ownership. + +The shared lifecycle project therefore has no Groundwork package reference. Structured Logs and OpenTelemetry each own their Groundwork schema declaration inside their concrete adapter project and contribute it to host composition independently. + +## Decision 4: Let host composition supply Groundwork access + +**Decision**: Groundwork adapters consume a provider-neutral factory/lease abstraction selected by the host. They do not expose four provider-specific Elsa store implementations. + +**Rationale**: Elsa owns the adapter behavior once; Groundwork owns relational/document-provider realization. This is the intended separation between store contracts and concrete implementation. + +**Alternatives rejected**: Four Elsa adapters would recreate the provider matrix that this program is eliminating. A global static session would obscure scope and disposal. + +## Decision 5: Separate cursor invalidity from operational failure + +**Decision**: Translate only malformed, trimmed, or wrong-binding replay positions to `StructuredLogReplayCursorUnavailableException`. Provider, schema, serialization, and cancellation failures remain visible as their actual domain-facing failure. + +**Rationale**: Returning cursor-unavailable for storage outages makes an investigation misleading and defeats readiness signals. + +**Alternatives rejected**: Catch-all cursor translation and empty-result fallback both conceal operational faults. + +## Decision 6: Bound filtered replay without client evaluation + +**Decision**: Compile filters, ordering, page limits, and cursor advancement into declared Groundwork operations. A page may advance over scanned nonmatches, but application code may not fetch an unbounded history to filter it. + +**Rationale**: Empty filtered pages can be legitimate while the durable cursor advances. Provider-side bounds protect memory and give identical semantics at scale. + +**Alternatives rejected**: Repeated unrestricted reads followed by LINQ filtering are not scale-safe and yield provider-dependent work. + +## Decision 7: Use deterministic least-recently-seen catalog retention + +**Decision**: Resource and instrument entries carry last-seen time plus a stable tie-breaker. Capacity enforcement retains the newest configured entries and deletes the exact remainder within the caller's storage scope. + +**Rationale**: This gives bounded catalogs, deterministic tests, and restart stability without generic reduce support. + +**Alternatives rejected**: In-memory LRU state is lost on restart; time-only ordering is ambiguous when timestamps collide. + +## Decision 8: Add the missing OpenTelemetry logs query endpoint + +**Decision**: Expose the existing `QueryLogsAsync` product capability alongside resources, traces, trace detail, and metrics, using the same authorization and validation conventions. + +**Rationale**: The core/provider contract already promises log queries, but the HTTP query surface is incomplete. + +**Alternatives rejected**: Deferring the endpoint would leave conformance success invisible to users and retain an acknowledged product gap. + +## Decision 9: Require provider execution evidence + +**Decision**: The four-provider suite must capture the selected physical operation/index evidence for scale-bearing reads and retention, in addition to comparing results. + +**Rationale**: Result equality does not prove the absence of broad scans or client-side evaluation. + +**Alternatives rejected**: Unit-only query-translation tests do not prove real provider plans. + +## Decision 10: Remove EF only after dual-oracle certification + +**Decision**: Retain the EF implementations as temporary behavior/performance oracles during implementation. Remove them in the final phase after shared conformance, four-provider readiness, and ratified performance gates pass. + +**Rationale**: The end state is zero EF in diagnostics, but deleting the oracle early would make subtle parity regressions harder to detect. + +**Alternatives rejected**: Immediate deletion optimizes repository appearance before behavioral replacement is proven. diff --git a/specs/091-groundwork-diagnostics-persistence/spec.md b/specs/091-groundwork-diagnostics-persistence/spec.md new file mode 100644 index 000000000..0e7fa476e --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/spec.md @@ -0,0 +1,149 @@ +# Feature Specification: Durable Diagnostics Persistence + +**Feature Branch**: `091-groundwork-diagnostics-persistence` + +**Created**: 2026-07-13 + +**Status**: Draft + +**Input**: Replace the Structured Logs and OpenTelemetry EF Core persistence implementations with one Groundwork-backed implementation family while keeping diagnostics core contracts independent of any persistence technology. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Resume durable diagnostics after interruption (Priority: P1) + +An operator reconnecting to a running or restarted Elsa host needs Structured Logs and OpenTelemetry history to remain available in a stable committed order, so an investigation does not lose or duplicate evidence when writers, readers, or hosts restart. + +**Why this priority**: Durable, trustworthy history is the reason to select a persistent diagnostics store. Incorrect replay or restart behavior makes the feature actively misleading. + +**Independent Test**: Append diagnostic records from multiple writers, restart the persistence session, reconnect from an earlier cursor, and verify that every committed record is returned exactly once in stable order while trimmed or foreign cursors fail without leaking binding details. + +**Acceptance Scenarios**: + +1. **Given** multiple writers commit Structured Log entries with equal display sequences and timestamps, **When** a reader replays from a committed cursor, **Then** every later committed entry appears exactly once in durable commit order. +2. **Given** a host restarts after committing Structured Logs and OpenTelemetry records, **When** an operator queries history, **Then** committed records, lifetime high-water values, catalog state, and valid replay positions remain available. +3. **Given** a replay position is malformed, trimmed, or belongs to another source, scope, or stream, **When** it is used, **Then** replay is rejected with one non-disclosing unavailable outcome. +4. **Given** storage is unavailable or its schema is invalid, **When** a diagnostic read or write is attempted, **Then** the operational failure remains visible and is not reported as an empty store or unavailable cursor. + +--- + +### User Story 2 - Query exact diagnostic history at scale (Priority: P1) + +An operator investigating a workflow needs bounded filters, deterministic ordering, trace detail, metrics, logs, exact counts, and retention to behave identically regardless of the host's supported database choice. + +**Why this priority**: A durable store is useful only when it can answer the diagnostics product's actual queries correctly without loading broad histories into application memory. + +**Independent Test**: Load deterministic Structured Logs and OpenTelemetry datasets, run every supported filter and boundary case against each supported provider, and compare results, order, counts, retention, and execution bounds against the shared behavior contract. + +**Acceptance Scenarios**: + +1. **Given** Structured Logs across levels, categories, sources, and equal timestamps, **When** recent or replay history is requested, **Then** filters, limits, cursor advancement, and oldest-to-newest response order are exact. +2. **Given** resources, repeated trace summaries, spans, instruments, metric points, and log records, **When** resource, trace, trace-detail, metric, or log queries are executed, **Then** all declared filters, inclusive ranges, latest-per-key behavior, tie-breaks, and result limits are exact. +3. **Given** a retention limit including zero, **When** retention runs, **Then** exactly the newest allowed records remain within the caller's scope while lifetime cursor and logical high-water metadata do not rewind. +4. **Given** identical identifiers in two storage scopes, **When** either scope appends, queries, counts, or trims records, **Then** the other scope's existence and data cannot be observed or affected. +5. **Given** resource and instrument catalogs grow beyond their configured capacities, **When** catalog retention runs, **Then** growth remains bounded by deterministic least-recently-seen retention without requiring broad client-side scans. + +--- + +### User Story 3 - Preserve capture under load and shutdown (Priority: P2) + +An application owner needs diagnostics capture to remain nonblocking, bounded, observable, and drainable during bursts and shutdown, so persistence latency cannot destabilize workflow execution or silently discard evidence. + +**Why this priority**: Provider correctness alone does not protect the host. The Elsa-owned capture and drain policy must continue to enforce explicit overload and shutdown behavior. + +**Independent Test**: Saturate the capture queue, inject transient and persistent storage failures, cancel writes, and perform graceful and timed-out shutdowns while verifying bounded memory, retry behavior, committed acknowledgements, loss accounting, and host responsiveness. + +**Acceptance Scenarios**: + +1. **Given** producers outpace durable persistence, **When** the configured queue capacity is exceeded, **Then** capture remains nonblocking, the documented items are shed, and loss is counted by reason. +2. **Given** a transient provider failure, **When** a batch is retried, **Then** it commits at most once and the caller receives the original durable outcome after acknowledgement loss. +3. **Given** retry exhaustion, **When** a batch cannot be persisted, **Then** the batch is accounted as lost, the drain loop remains alive, and a later batch can still succeed. +4. **Given** graceful host termination, **When** shutdown begins, **Then** new writes stop, queued items drain within the configured window, final retention is attempted, and the outcome is observable before storage services are disposed. +5. **Given** the shutdown deadline expires, **When** provider work remains incomplete, **Then** shutdown stays bounded and every accepted caller acknowledgement is completed with either a committed result or an explicit failure. + +--- + +### User Story 4 - Operate one provider model without EF migrations (Priority: P2) + +A host developer or DevOps engineer needs diagnostics persistence to use the same provider-neutral deployment model as the rest of Groundwork, so selecting SQLite, SQL Server, PostgreSQL, or MongoDB does not require maintaining feature-specific EF migration sets. + +**Why this priority**: Eliminating duplicated migration and provider implementations is the program-level maintenance outcome, but it follows behavioral parity and operational proof. + +**Independent Test**: Configure each supported provider, validate and apply the declared schema before startup, run the same diagnostics behavior suite, and verify that the diagnostics composition contains no EF Core implementation, registration, package, or migration dependency. + +**Acceptance Scenarios**: + +1. **Given** any supported provider, **When** deployment validation runs before the host serves traffic, **Then** all required streams, catalogs, indexes, scopes, and capabilities are validated or applied deterministically. +2. **Given** validation detects missing or drifted storage, **When** the host starts, **Then** readiness fails with actionable non-payload diagnostics instead of falling back to an empty or in-memory durable store. +3. **Given** parity and performance gates pass, **When** the diagnostics persistence feature is composed, **Then** it registers only the Groundwork concrete adapters while diagnostics core contracts remain provider-neutral. +4. **Given** the completed repository, **When** diagnostics projects and dependency graphs are inspected, **Then** no diagnostics EF Core projects, registrations, packages, contexts, entities, configurations, or migrations remain. + +### Edge Cases + +- An append commits but its acknowledgement is lost before reaching the Elsa drain loop. +- A trim commits but its acknowledgement is lost, then the same operation is retried after restart. +- Records share occurrence timestamps, logical sequences, trace identifiers, or resource identifiers. +- A filtered replay page contains no matching entries even though its scanned cursor advances. +- Retention removes the replay anchor while lifetime high-water state remains valid. +- Cancellation occurs before provider work, during provider work, and after commit but before acknowledgement. +- One storage scope is noisy enough to reach retention limits while another remains nearly empty. +- Provider instrumentation emits logs or traces while persisting the same diagnostics signal. +- A malformed payload, oversized batch, invalid range, or unsupported query reaches the adapter boundary. +- Startup is concurrent in multiple host processes against an empty database. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST preserve the existing provider-neutral Structured Logs and OpenTelemetry store contracts; core diagnostics modules MUST NOT depend on a concrete persistence implementation. +- **FR-002**: Structured Log append MUST complete only after durable acknowledgement and MUST return an authoritative opaque replay cursor bound to source, storage scope, and stream. +- **FR-003**: Structured Log history MUST preserve lifetime logical high-water, bounded recent queries, durable tail capture, bounded read-after pages, filtered cursor advancement, and exact count-based retention across restart. +- **FR-004**: OpenTelemetry persistence MUST support normalized batch writes and exact resource, trace, trace-detail, metric, and log queries, including declared case policies, inclusive ranges, deterministic ordering, and latest-per-logical-key selection. +- **FR-005**: Mutable resource and instrument catalogs MUST use bounded keyed storage with deterministic capacity enforcement; high-volume immutable signal history MUST use append-oriented record streams. +- **FR-006**: Every append, query, inspection, catalog operation, and trim MUST bind an explicit storage scope at the persistence boundary; cross-scope access MUST require a distinct privileged path and MUST NOT be the default. +- **FR-007**: All scale-bearing predicates, ordering, latest-per-key selection, exact counts, and retention selection MUST execute within a declared bounded provider operation; broad client-side evaluation is prohibited. +- **FR-008**: Retried append and trim operations MUST be idempotent across acknowledgement loss and restart, and reuse of an operation identity for a different request MUST fail without mutation. +- **FR-009**: The Elsa-owned capture layer MUST provide bounded queues, nonblocking producers, documented batch limits, cancellation-aware retry with bounded backoff, overload shedding, graceful drain, and a bounded shutdown fallback. +- **FR-010**: Loss and lifecycle observability MUST distinguish queue overflow, retry exhaustion, shutdown timeout, writes after closure, durable retention deletion, and subscriber delivery loss. +- **FR-011**: Diagnostics persistence instrumentation MUST be non-recursive and MUST NOT include payloads, secrets, tenant identifiers, trace identifiers, or record identifiers in low-cardinality labels. +- **FR-012**: Missing schema, schema drift, capability mismatch, and operational storage failures MUST fail validation or readiness visibly; they MUST NOT be converted into empty results or an in-memory persistence fallback. +- **FR-013**: SQLite, SQL Server, PostgreSQL, and MongoDB MUST pass one shared highest-seam behavior suite covering correctness, scope isolation, restart, concurrency, failure, acknowledgement loss, retention, and bounded execution evidence. +- **FR-014**: The system MUST expose the documented OpenTelemetry logs query surface so every query promised by the diagnostics contract is reachable through the product surface. +- **FR-015**: Provider validation and schema application MUST be executable before application startup using the shared deployment tooling. +- **FR-016**: EF Core diagnostics implementations, provider registrations, contexts, entities, configurations, packages, and migrations MUST be removed only after shared parity and performance gates pass. +- **FR-017**: Diagnostics core projects and public contracts MUST remain free of Groundwork references after the concrete Groundwork implementation becomes the sole first-party durable provider. + +### Key Entities + +- **Diagnostic Storage Binding**: The explicit tenant, storage scope, stream, and source identity governing one adapter instance. +- **Structured Log Record**: An immutable committed log entry with display metadata, filter fields, durable provider cursor, and an opaque Elsa replay cursor. +- **Telemetry Record Stream**: One append-ordered stream for trace summaries, spans, metric points, or telemetry log records. +- **Resource Catalog Entry**: Mutable keyed metadata for a telemetry-producing resource, including last-seen state used for bounded retention. +- **Instrument Catalog Entry**: Mutable keyed metadata for a metric instrument, including last-seen state used for bounded retention. +- **Capture Batch**: The bounded unit accepted by Elsa's drain policy and committed idempotently to one durable stream. +- **Replay Position**: A versioned opaque value binding a reader to its source, scope, stream, provider position, and record anchor. +- **Retention Operation**: An idempotent request to keep exactly the newest configured records or catalog entries within one scope. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A multi-writer restart and replay suite returns 100% of committed records exactly once and in stable order, with zero cross-scope or cross-source disclosure. +- **SC-002**: All four supported providers pass the same Structured Logs and OpenTelemetry behavior suite with identical results, ordering, exact counts, and retention outcomes. +- **SC-003**: Every scale-bearing query and retention scenario has provider execution evidence showing a declared bounded plan and no broad application-memory evaluation. +- **SC-004**: Under sustained producer overload, capture memory remains within configured queue bounds, producer calls do not wait for database I/O, and every loss is attributed to one documented reason. +- **SC-005**: Graceful shutdown either commits every accepted item or reports its explicit loss within the configured shutdown window; no caller acknowledgement remains incomplete. +- **SC-006**: Groundwork durable-operation p95 latency is no worse than 1.25 times the EF oracle, throughput is at least 80% of the oracle, and p99 latency is no worse than 2 times the oracle for the ratified workload matrix. +- **SC-007**: Startup and deployment validation reject 100% of tested missing-schema, drift, wrong-scope, and capability-mismatch cases before the host reports ready. +- **SC-008**: The completed diagnostics persistence and composition surface contains zero EF Core projects, registrations, packages, contexts, entities, configurations, or migrations, while core diagnostics assemblies contain zero Groundwork references. +- **SC-009**: One persisted diagnostic operation produces no recursive diagnostic append in the non-recursion conformance test. + +## Assumptions + +- Existing Structured Logs and OpenTelemetry domain contracts are the behavioral authority and remain stable unless a contradiction is proven by their highest-seam tests. +- The committed Groundwork diagnostic-record and ordinary-document primitives provide the provider execution foundation; this work owns Elsa adapters and composition rather than a second storage abstraction. +- The current Groundwork Structured Logs adapter is a useful starting point but is not considered complete until it passes the full failure, scope, provider, lifecycle, and operational requirements in this specification. +- Structured Log live fan-out, OpenTelemetry live fan-out, ingestion parsing, redaction, and authorization remain diagnostics-domain responsibilities outside durable storage. +- Resource and instrument catalog retention is capacity-based and least-recently-seen; cascading record deletion and generic aggregation are out of scope. +- Production EF data migration is out of scope because the selected Groundwork and Elsa persistence software is greenfield and unreleased. +- A dedicated third-party EF implementation repository may be created later, but it is outside this work unit. diff --git a/specs/091-groundwork-diagnostics-persistence/tasks.md b/specs/091-groundwork-diagnostics-persistence/tasks.md new file mode 100644 index 000000000..5a407ca09 --- /dev/null +++ b/specs/091-groundwork-diagnostics-persistence/tasks.md @@ -0,0 +1,208 @@ +# Tasks: Durable Diagnostics Persistence + +**Input**: Design documents from `specs/091-groundwork-diagnostics-persistence/` + +**Prerequisites**: `plan.md`, `spec.md`, `research.md`, `data-model.md`, `contracts/diagnostics-persistence.md` + +**Tests**: Tests are mandatory. For every behavior-changing implementation task, complete the listed test task first and observe the relevant assertion fail. + +**Organization**: Tasks are grouped by user story. Shared setup and provider fixtures are blocking foundations; each story then has an independently executable acceptance boundary. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: May run in parallel after its declared phase dependencies because it owns different files. +- **[Story]**: Maps the task to a user story in `spec.md`. +- Every task names the exact file or directory it owns. + +## Phase 1: Setup + +**Purpose**: Create project boundaries and registration points without changing active persistence behavior. + +- [X] T001 Create the shared diagnostics lifecycle library in `src/Elsa/Diagnostics/Persistence/Elsa.Diagnostics.Persistence.csproj` and add it to `Elsa.Server.slnx` +- [X] T002 [P] Create the OpenTelemetry Groundwork adapter project in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.csproj` +- [X] T003 [P] Create the shared lifecycle test project in `tests/Elsa/Diagnostics/Persistence/Tests/Elsa.Diagnostics.Persistence.Tests.csproj` +- [X] T004 [P] Create the OpenTelemetry Groundwork test project in `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj` +- [X] T005 Add all new projects to `Elsa.Server.slnx` and verify project references remain acyclic + +**Checkpoint**: New project boundaries build, but existing EF and Groundwork behavior remains unchanged. + +--- + +## Phase 2: Foundational Provider and Architecture Gates + +**Purpose**: Establish reusable conformance infrastructure and boundaries that block all story implementation. + +- [X] T006 Write failing architecture tests proving diagnostics core assemblies contain no Groundwork references and persistence registration has one unambiguous replacement path in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceArchitectureTests.cs` +- [X] T007 Implement only the composition abstractions needed by both adapters in `src/Elsa/Diagnostics/Persistence/`, catalog their semantics in `src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md`, and make T006 pass without exposing Groundwork types from diagnostics core +- [X] T008 Build a reusable four-provider fixture with real SQLite, SQL Server, PostgreSQL, and MongoDB lifecycle support in `tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsProviderFixture.cs` and instantiate every provider lease in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderLifecycleSmokeTests.cs` +- [X] T009 [P] Add reusable acknowledgement-loss, cancellation, restart, and operational-failure doubles in `tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsFailureFixtures.cs` +- [X] T010 [P] Record the temporary EF behavior-oracle inventory and exact parity mapping in `specs/091-groundwork-diagnostics-persistence/oracle-inventory.md` +- [X] T011 Add one shared diagnostics provider capability/readiness assertion helper in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderAssertions.cs` + +**Checkpoint**: All adapters can be tested through the same real-provider and failure harness; architecture boundaries are executable. + +--- + +## Phase 3: User Story 1 - Resume durable diagnostics after interruption (Priority: P1) 🎯 MVP + +**Goal**: Make Structured Logs and OpenTelemetry history durable, replayable, idempotent, and restart-safe with failures classified correctly. + +**Independent Test**: Commit all signal kinds from concurrent writers, restart the store, replay/query them in stable order, retry acknowledgement-lost operations, and reject malformed/trimmed/foreign cursors without hiding operational failures. + +### Tests for User Story 1 — write and observe failures first + +- [X] T012 [P] [US1] Expand Structured Logs conformance tests for tied ordering, filtered cursor advancement, restart, invalid binding, trimmed anchors, and operational failure visibility in `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogReplayTests.cs` +- [X] T013 [P] [US1] Add OpenTelemetry restart tests for resources, trace summaries, spans, instruments, metric points, and log records in `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryRestartTests.cs` +- [X] T014 [P] [US1] Add append idempotency, operation-identity conflict, acknowledgement-loss, cancellation-boundary, concurrent-writer, malformed-payload, and oversized-batch rejection-without-mutation tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDurableOperationConformanceTests.cs` +- [X] T015 [US1] Run T012-T014 against the current adapters and record the expected failing assertions in `specs/091-groundwork-diagnostics-persistence/evidence/red-test-baseline.md` before implementation + +### Implementation for User Story 1 + +- [X] T016 [US1] Harden durable append, stable commit ordering, cursor advancement, and exact cursor-failure translation in `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs` +- [X] T017 [US1] Bind replay cursors to version, storage scope, source, stream, provider position, and record anchor in `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkReplayCursorCodec.cs` +- [X] T018 [P] [US1] Define OpenTelemetry record-stream mappings and canonical serializers in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/` +- [X] T019 [P] [US1] Define resource and instrument catalog document mappings in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/` +- [ ] T020 [US1] Implement idempotent normalized batch writes and durable restart reads in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs` +- [ ] T021 [US1] Declare Structured Logs streams/indexes/ledger requirements in `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/StructuredLogsGroundworkStorageSchema.cs` and OpenTelemetry streams/catalogs/indexes/ledger requirements in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/OpenTelemetryGroundworkStorageSchema.cs` +- [ ] T022 [US1] Run the US1 conformance set against all four providers and store a summarized evidence manifest in `specs/091-groundwork-diagnostics-persistence/evidence/us1-provider-results.json` + +**Checkpoint**: Durable append, restart, replay, idempotency, and failure semantics work independently for every diagnostic signal on all four providers. + +--- + +## Phase 4: User Story 2 - Query exact diagnostic history at scale (Priority: P1) + +**Goal**: Execute every declared filter, range, ordering, latest-per-key, count, scope, and retention operation exactly and within provider-side bounds. + +**Independent Test**: Load a deterministic multi-scope dataset, compare every query and retention outcome across four providers, and verify declared indexes/plans with no broad client evaluation. + +### Tests for User Story 2 — write and observe failures first + +- [ ] T023 [P] [US2] Add the complete Structured Logs filter, limit, tie-break, count, retention-zero, and scope-isolation matrix in `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogQueryConformanceTests.cs` +- [ ] T024 [P] [US2] Add OpenTelemetry resource, trace, trace-detail, metric, and log query matrices with inclusive boundaries, stable ties, invalid-range rejection, and unsupported-query rejection without broad reads in `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryQueryConformanceTests.cs` +- [ ] T025 [P] [US2] Add deterministic resource/instrument catalog capacity and least-recently-seen retention tests in `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryCatalogTests.cs` +- [ ] T026 [P] [US2] Add cross-scope non-disclosure and exact retention tests for all signals in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsScopeAndRetentionConformanceTests.cs` +- [ ] T027 [US2] Add real-provider execution-plan/index evidence assertions for every scale-bearing query and mutation in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsBoundedExecutionTests.cs` + +### Implementation for User Story 2 + +- [ ] T028 [P] [US2] Implement bounded Structured Logs query and retention compilation in `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs` +- [ ] T029 [P] [US2] Implement bounded resource and instrument catalog queries/upserts/capacity enforcement in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/` +- [ ] T030 [US2] Implement bounded trace, span, metric-point, and telemetry-log record queries in `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs` +- [ ] T031 [US2] Add the missing authorized OpenTelemetry logs query endpoint in `src/Elsa/Diagnostics/OpenTelemetry/Endpoints/OpenTelemetry/Logs/Endpoint.cs` +- [ ] T032 [US2] Add endpoint binding and result tests in `tests/Elsa/Diagnostics/OpenTelemetry/Tests/OpenTelemetryLogsEndpointTests.cs` +- [ ] T033 [US2] Run the US2 query, scope, retention, and plan suite against all four providers and store a summarized evidence manifest in `specs/091-groundwork-diagnostics-persistence/evidence/us2-provider-results.json` + +**Checkpoint**: All persisted diagnostics queries and mutations are exact, scope-safe, and demonstrably bounded across all providers. + +--- + +## Phase 5: User Story 3 - Preserve capture under load and shutdown (Priority: P2) + +**Goal**: Give both Groundwork adapters one Elsa-owned bounded, nonblocking, retrying, observable, and drainable capture lifecycle. + +**Independent Test**: Saturate queues, inject transient and permanent faults, cancel at every boundary, lose acknowledgements, and stop gracefully or by timeout while proving bounded memory and complete caller outcomes. + +### Tests for User Story 3 — write and observe failures first + +- [X] T034 [P] [US3] Port and expand the EF drain oracle into provider-independent queue, retry, acknowledgement, closure, and disposal tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLifecycleTests.cs` +- [X] T035 [P] [US3] Add concurrent producer, overflow shedding, retry exhaustion, and later-batch recovery tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLoadTests.cs` +- [X] T036 [P] [US3] Add graceful drain, timeout fallback, final retention, and no-incomplete-acknowledgement tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainShutdownTests.cs` +- [X] T037 [P] [US3] Add non-recursive instrumentation, low-cardinality/no-payload telemetry, and production subscriber-delivery bridge classification tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObservabilityTests.cs` + +### Implementation for User Story 3 + +- [X] T038 [US3] Extract the bounded drain state machine from `src/Elsa/Persistence/EFCore/Storage/ChannelDrainingStoreBase.cs` into composed services under `src/Elsa/Diagnostics/Persistence/Draining/` +- [X] T039 [US3] Implement queue, retry, shutdown, retention, closure, and subscriber-delivery loss classification in `src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsPersistenceObservability.cs` and `src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsSubscriberDeliveryLossBridge.cs`, consuming the existing domain live-feed signals without moving fan-out into persistence +- [ ] T040 [P] [US3] Integrate the shared drain into `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs` +- [ ] T041 [P] [US3] Integrate the shared drain into `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs` +- [ ] T042 [US3] Register one explicit start/stop/drain-before-provider-disposal lifecycle in `src/Elsa/Diagnostics/Persistence/Extensions/ServiceCollectionExtensions.cs` +- [ ] T043 [US3] Run the complete load and shutdown suite and record queue bounds, loss totals, and completion outcomes in `specs/091-groundwork-diagnostics-persistence/evidence/us3-lifecycle-results.json` + +**Checkpoint**: Persistence latency cannot block producers or leave accepted acknowledgements unresolved; every loss and shutdown outcome is observable. + +--- + +## Phase 6: User Story 4 - Operate one provider model without EF migrations (Priority: P2) + +**Goal**: Compose, deploy, validate, and operate only Groundwork diagnostics persistence across all providers, then remove diagnostics EF completely. + +**Independent Test**: Validate/apply schema before startup, run the same behavior suite on every provider, fail readiness on drift, pass performance gates, and prove zero EF dependencies remain in diagnostics while core remains Groundwork-free. + +### Tests for User Story 4 — write and observe failures first + +- [ ] T044 [P] [US4] Add schema validate/apply, missing-schema, drift, capability-mismatch, and concurrent-start tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsSchemaDeploymentTests.cs` +- [ ] T045 [P] [US4] Add enabled/disabled/provider-selection and one-store registration tests in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceFeatureTests.cs` +- [ ] T046 [P] [US4] Add readiness tests proving provider/schema failures never fall back to empty or in-memory durable results in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceReadinessTests.cs` +- [ ] T047 [US4] Add final dependency/public-surface tests for zero diagnostics EF and zero core Groundwork references in `tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceArchitectureTests.cs` + +### Implementation for User Story 4 + +- [ ] T048 [US4] Contribute each concrete schema declaration to the shared Groundwork validate/apply CLI path from `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogsPersistenceFeature.cs` and `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryPersistenceFeature.cs` +- [ ] T049 [US4] Implement Groundwork persistence feature composition and readiness in `src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/` and `src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/` +- [ ] T050 [P] [US4] Add the ratified EF-versus-Groundwork diagnostics workload, datasets, environment manifest, and raw-artifact schema under `tools/performance/diagnostics/` +- [ ] T051 [US4] Run the non-promotable smoke profile, then the full ratified four-provider performance matrix, and store the decision artifact in `specs/091-groundwork-diagnostics-persistence/evidence/performance-decision.json` +- [ ] T052 [US4] For every material correctness or performance regression, record the failing gate, changed source/test paths, remediation commit, and rerun outcome in `specs/091-groundwork-diagnostics-persistence/evidence/performance-decision.json`, then repeat T044-T051 until every removal gate passes +- [ ] T053 [US4] Delete Structured Logs EF implementation projects and their tests under `src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/` and `tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/` while preserving provider-neutral conformance coverage +- [ ] T054 [US4] Delete OpenTelemetry EF implementation projects and their tests under `src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/` and `tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/` while preserving provider-neutral conformance coverage +- [ ] T055 [US4] Remove diagnostics EF project/registration/migration/package usages from `Elsa.Server.slnx` and affected diagnostics projects; remove EF entries from `Directory.Packages.props` only when the repository-wide dependency audit proves no remaining feature consumes them +- [ ] T056 [P] [US4] Update `src/Elsa/Diagnostics/StructuredLogs/README.md`, `src/Elsa/Diagnostics/StructuredLogs/EXTENSION_POINTS.md`, `src/Elsa/Diagnostics/OpenTelemetry/README.md`, and `src/Elsa/Diagnostics/OpenTelemetry/EXTENSION_POINTS.md` +- [ ] T057 [US4] Run the complete four-provider suite, solution build, architecture audit, and final `rg` zero-EF/zero-core-Groundwork checks documented in `specs/091-groundwork-diagnostics-persistence/quickstart.md` + +**Checkpoint**: Diagnostics has one first-party Groundwork persistence model, four-provider proof, pre-start deployment support, and zero EF implementation surface. + +--- + +## Phase 7: Cross-Cutting Completion and Ratification + +**Purpose**: Make the work durable for maintainers and agents, and land it through the approved operating model. + +- [ ] T058 [P] Refresh the narrowest affected dependency and extension-point maps using `tools/maps/generate-feature-dependency-map.sh` and `tools/maps/generate-extension-point-map.sh` +- [ ] T059 Reconcile all evidence, task completion, exceptions, and follow-up findings in `specs/091-groundwork-diagnostics-persistence/quickstart.md` and `docs/reports/unfinished-work.md` +- [ ] T060 Run an independent review of the exact branch HEAD against FR-001 through FR-017 and SC-001 through SC-009, then remediate every blocker +- [ ] T061 Push the organization-owned feature branch, open the reviewed PR linked to the diagnostics migration issue, obtain required checks, and merge it to `main` +- [ ] T062 Verify `main` and the remote issue/PR state, then record the final commit and evidence links in `specs/091-groundwork-diagnostics-persistence/quickstart.md` + +--- + +## Dependencies & Execution Order + +### Phase dependencies + +- Phase 1 has no dependencies. +- Phase 2 depends on Phase 1 and blocks every user story. +- US1 and US2 may start after Phase 2; US2 query implementations depend on the relevant US1 record/catalog mappings. +- US3 may start after Phase 2; execute T034-T039 immediately as the first independent implementation slice, then integrate T040-T041 after the relevant adapters exist. +- US4 schema/readiness test work may begin after Phase 2, but EF removal tasks T053-T055 depend on all US1-US3 gates plus T044-T052. +- Phase 7 depends on the desired user stories being complete; merge and final verification are strictly last. + +### Story dependencies + +- **US1**: No story dependency; supplies durable record/catalog foundations. +- **US2**: Uses US1 mappings but remains independently testable through deterministic query fixtures. +- **US3**: Owns capture lifecycle and can be developed with failure doubles before concrete adapters integrate it. +- **US4**: Integrates and removes the old implementation only after US1-US3 correctness and performance proof. + +### Parallel opportunities + +- T002-T004, T008-T010, and all test tasks explicitly marked `[P]` own different files. +- After Phase 2, one worker can harden Structured Logs, one can implement OpenTelemetry storage, and one can implement the shared drain, provided only one worker owns any shared project/solution file at a time. +- Provider certification executions may run in parallel only when their container/database names and artifact directories are isolated. +- Documentation/map refresh can run in parallel after the corresponding implementation paths stabilize. + +## Implementation Strategy + +1. Land Phase 1-2 as a small architecture/test-fixture checkpoint. +2. Complete the T034-T039 shared-drain tests and extraction as the first independently certified implementation slice. +3. Deliver US1 durable restart/replay and integrate the tested drain into each adapter as it becomes available. +4. Add US2 exact query/retention behavior and provider-plan evidence. +5. Complete US4 deployment/performance gates; only then delete EF. +6. Obtain exact-HEAD independent review, merge through Model B, and verify remote `main`. + +## Notes + +- Do not weaken provider-plan gates to make small fixtures pass; correct dataset selectivity, schema, or query shape. +- Do not replace bounded provider operations with identity fan-out, broad materialization, or client evaluation. +- Do not catch operational failures as cursor unavailability or empty results. +- Keep all scope, operation identity, and route/schema binding checks ahead of provider I/O when possible. +- Commit after each coherent approved work unit; do not commit personal `.agent-prefs/` files. diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Core/Exceptions/OpenTelemetryPersistenceException.cs b/src/Elsa/Diagnostics/OpenTelemetry/Core/Exceptions/OpenTelemetryPersistenceException.cs new file mode 100644 index 000000000..2b787561a --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Core/Exceptions/OpenTelemetryPersistenceException.cs @@ -0,0 +1,130 @@ +using System.Collections.ObjectModel; + +namespace Elsa.Diagnostics.OpenTelemetry.Core.Exceptions; + +/// Stable failure classifications exposed by OpenTelemetry persistence contracts. +public enum OpenTelemetryPersistenceFailureReason +{ + CapabilityUnavailable, + ConflictingOperation, + ExpiredOperation, + InvalidRecord, + CorruptData, + ProviderFailure +} + +/// +/// Provider-neutral base exception for failures crossing an OpenTelemetry persistence boundary. +/// Infrastructure details remain available only through . +/// +public abstract class OpenTelemetryPersistenceException : Exception +{ + protected OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason reason, + string operation, + string message, + IReadOnlyDictionary? context = null, + Exception? innerException = null) + : base(message, innerException) + { + ArgumentException.ThrowIfNullOrWhiteSpace(operation); + Reason = reason; + Operation = operation; + Context = new ReadOnlyDictionary( + new Dictionary(context ?? new Dictionary(), StringComparer.Ordinal)); + } + + public OpenTelemetryPersistenceFailureReason Reason { get; } + public string Operation { get; } + public IReadOnlyDictionary Context { get; } +} + +/// Reports that a required portable store capability is unavailable. +public sealed class OpenTelemetryPersistenceCapabilityException( + string operation, + string capability, + string message, + IReadOnlyDictionary? context = null) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.CapabilityUnavailable, + operation, + message, + With(context, "capability", capability)) +{ + private static IReadOnlyDictionary With( + IReadOnlyDictionary? context, + string key, + string value) + { + var result = new Dictionary(context ?? new Dictionary(), StringComparer.Ordinal) + { + [key] = value + }; + return result; + } +} + +/// Reports an operation-identity conflict without exposing the concrete store type. +public sealed class OpenTelemetryPersistenceConflictException( + string operation, + string message, + IReadOnlyDictionary context, + Exception innerException) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.ConflictingOperation, + operation, + message, + context, + innerException); + +/// Reports that a retry can no longer safely continue. +public sealed class OpenTelemetryPersistenceExpiredException( + string operation, + string message, + IReadOnlyDictionary context, + Exception innerException) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.ExpiredOperation, + operation, + message, + context, + innerException); + +/// Reports that normalized telemetry cannot be accepted by the portable persistence contract. +public sealed class OpenTelemetryPersistenceValidationException( + string operation, + string message, + IReadOnlyDictionary context, + Exception innerException) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.InvalidRecord, + operation, + message, + context, + innerException); + +/// Reports malformed or inconsistent durable OpenTelemetry persistence state. +public sealed class OpenTelemetryPersistenceDataException( + string operation, + string message, + IReadOnlyDictionary context, + Exception? innerException = null) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.CorruptData, + operation, + message, + context, + innerException); + +/// Reports an operational persistence failure without exposing the concrete provider. +public sealed class OpenTelemetryPersistenceUnavailableException( + string operation, + string message, + IReadOnlyDictionary context, + Exception innerException) + : OpenTelemetryPersistenceException( + OpenTelemetryPersistenceFailureReason.ProviderFailure, + operation, + message, + context, + innerException); diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Elsa.Diagnostics.OpenTelemetry.csproj b/src/Elsa/Diagnostics/OpenTelemetry/Elsa.Diagnostics.OpenTelemetry.csproj index 3f96ee2f7..c6403a1e4 100644 --- a/src/Elsa/Diagnostics/OpenTelemetry/Elsa.Diagnostics.OpenTelemetry.csproj +++ b/src/Elsa/Diagnostics/OpenTelemetry/Elsa.Diagnostics.OpenTelemetry.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Extensions/ServiceCollectionExtensions.cs b/src/Elsa/Diagnostics/OpenTelemetry/Extensions/ServiceCollectionExtensions.cs index d64043665..fcadfa373 100644 --- a/src/Elsa/Diagnostics/OpenTelemetry/Extensions/ServiceCollectionExtensions.cs +++ b/src/Elsa/Diagnostics/OpenTelemetry/Extensions/ServiceCollectionExtensions.cs @@ -2,6 +2,7 @@ using Elsa.Diagnostics.OpenTelemetry.Core.Options; using Elsa.Diagnostics.OpenTelemetry.Providers.InMemory; using Elsa.Diagnostics.OpenTelemetry.Services; +using Elsa.Diagnostics.Persistence.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -17,6 +18,7 @@ public static class ServiceCollectionExtensions /// public static IServiceCollection AddOpenTelemetryDiagnosticsServices(this IServiceCollection services, Action? configureOptions = null) { + services.AddDiagnosticsPersistenceObservability(); if (configureOptions != null) services.Configure(configureOptions); diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocumentSerializer.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocumentSerializer.cs new file mode 100644 index 000000000..657c331f1 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocumentSerializer.cs @@ -0,0 +1,265 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Groundwork.Documents.Store; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; + +/// Maps mutable resource and instrument catalogs to canonical keyed Groundwork documents. +public sealed class CatalogDocumentSerializer +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = false, + RespectRequiredConstructorParameters = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow + }; + + /// Creates a canonical keyed document request for a resource catalog value. + /// The resource cannot be represented canonically. + public SaveDocumentRequest ToSaveRequest(TelemetryResource resource, long? expectedRevision = null) + { + ArgumentNullException.ThrowIfNull(resource); + ValidateExpectedRevision(expectedRevision); + + return Map(() => + { + var id = RequiredText(resource.Id, nameof(resource.Id)); + var serviceName = RequiredText(resource.ServiceName, nameof(resource.ServiceName)); + EnumValue((int)resource.Status, nameof(resource.Status)); + + return CreateRequest( + CatalogDocuments.ResourceKind, + id, + new ResourceDocument( + id, + serviceName, + resource.ServiceInstanceId, + resource.TelemetrySdkLanguage, + Canonicalize(resource.Attributes), + resource.LastSeen.UtcTicks, + (int)resource.Status, + id), + expectedRevision); + }); + } + + /// Creates a canonical keyed document request for an instrument catalog value. + /// The caller supplies the observation time once and reuses it across retries. + /// The instrument cannot be represented canonically. + public SaveDocumentRequest ToSaveRequest( + MetricInstrument instrument, + DateTimeOffset lastSeen, + long? expectedRevision = null) + { + ArgumentNullException.ThrowIfNull(instrument); + ValidateExpectedRevision(expectedRevision); + + return Map(() => + { + var id = RequiredText(instrument.Id, nameof(instrument.Id)); + var resourceId = RequiredText(instrument.ResourceId, nameof(instrument.ResourceId)); + var name = RequiredText(instrument.Name, nameof(instrument.Name)); + EnumValue((int)instrument.Kind, nameof(instrument.Kind)); + + return CreateRequest( + CatalogDocuments.InstrumentKind, + id, + new InstrumentDocument( + id, + resourceId, + name, + instrument.Unit, + instrument.Description, + (int)instrument.Kind, + Canonicalize(instrument.Attributes), + lastSeen.UtcTicks, + id), + expectedRevision); + }); + } + + /// Restores a resource catalog value and its revision from a Groundwork document. + /// The document kind, schema, identity, or payload is invalid. + public ResourceCatalogEntry ToResource(DocumentEnvelope envelope) + { + ArgumentNullException.ThrowIfNull(envelope); + return Deserialize(envelope, CatalogDocuments.ResourceKind, (ResourceDocument document) => + { + ValidateIdentity(envelope, document.Id, document.RetentionTieBreaker); + return new ResourceCatalogEntry( + new( + RequiredText(document.Id, nameof(document.Id)), + RequiredText(document.ServiceName, nameof(document.ServiceName)), + document.ServiceInstanceId, + document.TelemetrySdkLanguage, + Restore(document.Attributes), + Utc(document.LastSeenUtcTicks), + EnumValue(document.Status, nameof(document.Status))), + envelope.Version); + }); + } + + /// Restores an instrument catalog value, retention metadata, and revision from a Groundwork document. + /// The document kind, schema, identity, or payload is invalid. + public InstrumentCatalogEntry ToInstrument(DocumentEnvelope envelope) + { + ArgumentNullException.ThrowIfNull(envelope); + return Deserialize(envelope, CatalogDocuments.InstrumentKind, (InstrumentDocument document) => + { + ValidateIdentity(envelope, document.Id, document.RetentionTieBreaker); + return new InstrumentCatalogEntry( + new( + RequiredText(document.Id, nameof(document.Id)), + RequiredText(document.ResourceId, nameof(document.ResourceId)), + RequiredText(document.Name, nameof(document.Name)), + document.Unit, + document.Description, + EnumValue(document.Kind, nameof(document.Kind)), + Restore(document.Attributes)), + Utc(document.LastSeenUtcTicks), + RequiredText(document.RetentionTieBreaker, nameof(document.RetentionTieBreaker)), + envelope.Version); + }); + } + + private static SaveDocumentRequest CreateRequest( + string documentKind, + string id, + TDocument document, + long? expectedRevision) + { + try + { + return new( + documentKind, + id, + CatalogDocuments.SchemaVersion, + JsonSerializer.Serialize(document, SerializerOptions), + expectedRevision); + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException) + { + throw new CatalogPayloadException("The OpenTelemetry catalog value cannot be serialized.", exception); + } + } + + private static SaveDocumentRequest Map(Func map) + { + try + { + return map(); + } + catch (CatalogPayloadException) + { + throw; + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException) + { + throw new CatalogPayloadException( + "The OpenTelemetry catalog value cannot be represented as a canonical payload.", + exception); + } + } + + private static void ValidateExpectedRevision(long? expectedRevision) => + ArgumentOutOfRangeException.ThrowIfLessThan(expectedRevision ?? 0, 0); + + private static TResult Deserialize( + DocumentEnvelope envelope, + string expectedKind, + Func map) + where TDocument : class + { + try + { + if (!StringComparer.Ordinal.Equals(envelope.DocumentKind, expectedKind)) + throw new CatalogPayloadException( + $"OpenTelemetry catalog document kind '{envelope.DocumentKind}' is not '{expectedKind}'."); + if (!StringComparer.Ordinal.Equals(envelope.SchemaVersion, CatalogDocuments.SchemaVersion)) + throw new CatalogPayloadException( + $"OpenTelemetry catalog schema '{envelope.SchemaVersion}' is not supported."); + if (envelope.Version < 0) + throw new CatalogPayloadException("The OpenTelemetry catalog revision cannot be negative."); + + var document = JsonSerializer.Deserialize(envelope.ContentJson, SerializerOptions) + ?? throw new CatalogPayloadException("The OpenTelemetry catalog payload is empty."); + return map(document); + } + catch (CatalogPayloadException) + { + throw; + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException or OverflowException) + { + throw new CatalogPayloadException("The OpenTelemetry catalog payload is malformed.", exception); + } + } + + private static void ValidateIdentity(DocumentEnvelope envelope, string? id, string? retentionTieBreaker) + { + if (!StringComparer.Ordinal.Equals(envelope.Id, id)) + throw new CatalogPayloadException("The OpenTelemetry catalog payload identity does not match its envelope."); + if (!StringComparer.Ordinal.Equals(envelope.Id, retentionTieBreaker)) + throw new CatalogPayloadException("The OpenTelemetry catalog retention tie-breaker does not match its identity."); + } + + private static AttributeDocument[] Canonicalize(IDictionary attributes) => + Required(attributes, nameof(attributes)) + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => new AttributeDocument(RequiredText(x.Key, "attribute key"), x.Value)) + .ToArray(); + + private static Dictionary Restore(AttributeDocument[]? attributes) + { + if (attributes is null) + throw new CatalogPayloadException("The OpenTelemetry catalog attributes are missing."); + + var result = new Dictionary(StringComparer.Ordinal); + foreach (var attribute in attributes) + { + var value = Required(attribute, "attribute"); + if (value.Key is null || !result.TryAdd(value.Key, value.Value)) + throw new CatalogPayloadException("The OpenTelemetry catalog contains an invalid or duplicate attribute key."); + } + return result; + } + + private static T Required(T? value, string field) where T : class => + value ?? throw new CatalogPayloadException($"The OpenTelemetry catalog field '{field}' is required."); + + private static string RequiredText(string? value, string field) => + !string.IsNullOrWhiteSpace(value) + ? value + : throw new CatalogPayloadException($"The OpenTelemetry catalog field '{field}' requires a non-empty value."); + + private static TEnum EnumValue(int value, string field) where TEnum : struct, Enum => + Enum.IsDefined(typeof(TEnum), value) + ? (TEnum)Enum.ToObject(typeof(TEnum), value) + : throw new CatalogPayloadException($"The OpenTelemetry catalog field '{field}' has an unsupported value."); + + private static DateTimeOffset Utc(long ticks) => new(ticks, TimeSpan.Zero); + + private sealed record AttributeDocument(string Key, string? Value); + + private sealed record ResourceDocument( + string Id, + string ServiceName, + string? ServiceInstanceId, + string? TelemetrySdkLanguage, + AttributeDocument[] Attributes, + long LastSeenUtcTicks, + int Status, + string RetentionTieBreaker); + + private sealed record InstrumentDocument( + string Id, + string ResourceId, + string Name, + string? Unit, + string? Description, + int Kind, + AttributeDocument[] Attributes, + long LastSeenUtcTicks, + string RetentionTieBreaker); +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocuments.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocuments.cs new file mode 100644 index 000000000..7f8120f64 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogDocuments.cs @@ -0,0 +1,17 @@ +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; + +/// Stable document identities and JSON paths for the OpenTelemetry catalogs. +public static class CatalogDocuments +{ + public const string ResourceKind = "openTelemetryResource"; + public const string InstrumentKind = "openTelemetryMetricInstrument"; + public const string SchemaVersion = "1"; + + public const string InstrumentResourceIdPath = "/resourceId"; + public const string ServiceNamePath = "/serviceName"; + public const string ResourceStatusPath = "/status"; + public const string InstrumentNamePath = "/name"; + public const string InstrumentKindPath = "/kind"; + public const string LastSeenPath = "/lastSeenUtcTicks"; + public const string RetentionTieBreakerPath = "/retentionTieBreaker"; +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogEntries.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogEntries.cs new file mode 100644 index 000000000..f3ca58b96 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogEntries.cs @@ -0,0 +1,13 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; + +/// A resource catalog value together with its durable optimistic-concurrency revision. +public sealed record ResourceCatalogEntry(TelemetryResource Resource, long Revision); + +/// An instrument catalog value together with its retention metadata and durable revision. +public sealed record InstrumentCatalogEntry( + MetricInstrument Instrument, + DateTimeOffset LastSeen, + string RetentionTieBreaker, + long Revision); diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogPayloadException.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogPayloadException.cs new file mode 100644 index 000000000..575b00ca9 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Catalogs/CatalogPayloadException.cs @@ -0,0 +1,13 @@ +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; + +/// Raised when an OpenTelemetry catalog document cannot be mapped to or from its durable payload. +public sealed class CatalogPayloadException : Exception +{ + public CatalogPayloadException(string message) : base(message) + { + } + + public CatalogPayloadException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.csproj b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.csproj new file mode 100644 index 000000000..24b9cc589 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + 0.0.1-local + + + + + + + + + + + + + diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryBinding.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryBinding.cs new file mode 100644 index 000000000..420fb3a83 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryBinding.cs @@ -0,0 +1,84 @@ +using System.Security.Cryptography; +using System.Text; +using Groundwork.Core.Scoping; +using Groundwork.DiagnosticRecords; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork; + +/// +/// Explicit persistence routing for one OpenTelemetry source. Core contracts remain provider-neutral; this +/// Groundwork leaf owns the translation to diagnostic scopes, streams, and a scoped document partition. +/// +public sealed class GroundworkOpenTelemetryBinding +{ + private GroundworkOpenTelemetryBinding(string tenantId, string scopeId, string sourceId, string routeIdentity) + { + TenantId = tenantId; + ScopeId = scopeId; + SourceId = sourceId; + TraceStreamId = $"open-telemetry:{routeIdentity}:traces"; + SpanStreamId = $"open-telemetry:{routeIdentity}:spans"; + MetricPointStreamId = $"open-telemetry:{routeIdentity}:metric-points"; + LogStreamId = $"open-telemetry:{routeIdentity}:logs"; + } + + public string TenantId { get; } + public string ScopeId { get; } + public string SourceId { get; } + public string TraceStreamId { get; } + public string SpanStreamId { get; } + public string MetricPointStreamId { get; } + public string LogStreamId { get; } + + /// Creates a binding whose stream names are deterministically derived from the source identity. + public static GroundworkOpenTelemetryBinding Create(string tenantId, string scopeId, string sourceId) + { + Validate(tenantId, nameof(tenantId)); + Validate(scopeId, nameof(scopeId)); + Validate(sourceId, nameof(sourceId)); + return new(tenantId, scopeId, sourceId, HashBinding(tenantId, scopeId, sourceId)); + } + + internal DiagnosticStorageScope DiagnosticScope + { + get + { + ValidateAll(); + return new(TenantId, ScopeId); + } + } + + /// The Groundwork document partition derived from tenant, scope, and source. + public StorageScope DocumentStorageScope + { + get + { + ValidateAll(); + return new($"otel-{HashBinding(TenantId, ScopeId, SourceId)}"); + } + } + + internal void ValidateAll() + { + Validate(TenantId, nameof(TenantId)); + Validate(ScopeId, nameof(ScopeId)); + Validate(SourceId, nameof(SourceId)); + Validate(TraceStreamId, nameof(TraceStreamId)); + Validate(SpanStreamId, nameof(SpanStreamId)); + Validate(MetricPointStreamId, nameof(MetricPointStreamId)); + Validate(LogStreamId, nameof(LogStreamId)); + + var streams = new[] { TraceStreamId, SpanStreamId, MetricPointStreamId, LogStreamId }; + if (streams.Distinct(StringComparer.Ordinal).Count() != streams.Length) + throw new ArgumentException("OpenTelemetry stream identities must be distinct.", nameof(TraceStreamId)); + } + + private static void Validate(string value, string parameterName) => + ArgumentException.ThrowIfNullOrWhiteSpace(value, parameterName); + + private static string HashBinding(string tenantId, string scopeId, string sourceId) + { + var input = Encoding.UTF8.GetBytes($"{tenantId.Length}:{tenantId}{scopeId.Length}:{scopeId}{sourceId.Length}:{sourceId}"); + return Convert.ToHexStringLower(SHA256.HashData(input)); + } +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs new file mode 100644 index 000000000..b2d30fec8 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStore.cs @@ -0,0 +1,739 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Elsa.Diagnostics.OpenTelemetry.Core.Contracts; +using Elsa.Diagnostics.OpenTelemetry.Core.Exceptions; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.OpenTelemetry.Core.Options; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; +using Elsa.Diagnostics.Persistence.Draining; +using Groundwork.DiagnosticRecords; +using Groundwork.Documents.Scoping; +using Groundwork.Documents.Store; +using Microsoft.Extensions.Options; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork; + +/// +/// Durable Groundwork implementation of the provider-neutral OpenTelemetry store contract. Immutable signals use +/// four append streams. The adapter rejects catalog identity writes and case-equivalence-dependent reads until +/// Groundwork supplies the portable comparison-key route tracked by issues #70 and #71. A durable capture ledger binds +/// a caller-owned drain batch identity to canonical input and tracks a bounded attempt for each stream independently. +/// +/// +/// The interim exact read surface is storage diagnostics plus unfiltered telemetry-log restart reads (with either no +/// time range or a closed inclusive range). Resource results, trace results/detail, metric results, catalog writes, and +/// every string-filtered log query fail before provider I/O. Those paths must not be enabled until #70/#71 are consumed. +/// +public sealed class GroundworkOpenTelemetryStore : IOpenTelemetryStore +{ + private const int MaxLedgerWriteAttempts = 8; + private const int LedgerSchemaVersion = 2; + private static readonly JsonSerializerOptions LedgerJsonOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = false, + RespectRequiredConstructorParameters = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow + }; + private static readonly string[] StreamKinds = ["traces", "spans", "metric-points", "logs"]; + + private readonly GroundworkOpenTelemetryStores _stores; + private readonly GroundworkOpenTelemetryBinding _binding; + private readonly DiagnosticStorageScope _scope; + private readonly DiagnosticStreamId _traceStream; + private readonly DiagnosticStreamId _spanStream; + private readonly DiagnosticStreamId _metricPointStream; + private readonly DiagnosticStreamId _logStream; + private readonly DiagnosticRecordStreamDefinition _traceDefinition; + private readonly DiagnosticRecordStreamDefinition _spanDefinition; + private readonly DiagnosticRecordStreamDefinition _metricPointDefinition; + private readonly DiagnosticRecordStreamDefinition _logDefinition; + private readonly CanonicalRecordSerializer _recordSerializer = new(); + private readonly TimeProvider _timeProvider; + private readonly int _traceCapacity; + private readonly int _spanCapacity; + private readonly int _metricPointCapacity; + private readonly int _logRecordCapacity; + private readonly int _maxQuerySize; + + public GroundworkOpenTelemetryStore( + GroundworkOpenTelemetryStores stores, + IOptions options, + GroundworkOpenTelemetryBinding binding, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(stores); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(binding); + ArgumentNullException.ThrowIfNull(stores.Traces); + ArgumentNullException.ThrowIfNull(stores.Spans); + ArgumentNullException.ThrowIfNull(stores.MetricPoints); + ArgumentNullException.ThrowIfNull(stores.Logs); + ArgumentNullException.ThrowIfNull(stores.Documents); + binding.ValidateAll(); + + if (stores.Documents.Access.Kind != DocumentStoreAccessKind.Scoped || + stores.Documents.Access.Scope != binding.DocumentStorageScope) + { + throw new ArgumentException( + "The OpenTelemetry document store is not bound to the tenant, scope, and source selected by the adapter binding.", + nameof(stores)); + } + + _stores = stores; + _binding = binding; + _scope = binding.DiagnosticScope; + _traceStream = new(binding.TraceStreamId); + _spanStream = new(binding.SpanStreamId); + _metricPointStream = new(binding.MetricPointStreamId); + _logStream = new(binding.LogStreamId); + _traceDefinition = OpenTelemetryRecordStreamDefinitions.CreateTraces(binding.TraceStreamId); + _spanDefinition = OpenTelemetryRecordStreamDefinitions.CreateSpans(binding.SpanStreamId); + _metricPointDefinition = OpenTelemetryRecordStreamDefinitions.CreateMetricPoints(binding.MetricPointStreamId); + _logDefinition = OpenTelemetryRecordStreamDefinitions.CreateLogs(binding.LogStreamId); + _timeProvider = timeProvider ?? TimeProvider.System; + var value = options.Value; + _traceCapacity = ClampCapacity(value.TraceCapacity); + _spanCapacity = ClampCapacity(value.SpanCapacity); + _metricPointCapacity = ClampCapacity(value.MetricPointCapacity); + _logRecordCapacity = ClampCapacity(value.LogRecordCapacity); + _maxQuerySize = ClampCapacity(value.MaxQuerySize); + } + + public ValueTask WriteAsync(OpenTelemetryBatch batch, CancellationToken cancellationToken = default) => + WriteAsync(DiagnosticsDrainBatchId.New(), batch, cancellationToken); + + /// + /// Commits one drain batch. The same identity may be retried with the same canonical input; a different identity + /// always represents an independent capture, even when its content is byte-for-byte identical. + /// + public async ValueTask WriteAsync( + DiagnosticsDrainBatchId batchId, + OpenTelemetryBatch batch, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(batch); + if (batchId.Value == Guid.Empty) + throw new ArgumentException("The diagnostics drain batch identity cannot be empty.", nameof(batchId)); + cancellationToken.ThrowIfCancellationRequested(); + RejectCatalogIdentityWrites(batch); + + var context = Context(("batchId", batchId.ToString())); + DiagnosticRecordInput[] traces; + DiagnosticRecordInput[] spans; + DiagnosticRecordInput[] points; + DiagnosticRecordInput[] logs; + try + { + traces = NormalizeTraces(batch.Traces); + spans = NormalizeRecords(batch.Spans, span => _recordSerializer.ToRecord(span.Id, span)); + points = NormalizeRecords(batch.MetricPoints, point => _recordSerializer.ToRecord(point.Id, point)); + logs = NormalizeRecords(batch.Logs, log => _recordSerializer.ToRecord(log.Id, log)); + } + catch (RecordPayloadException exception) + { + throw new OpenTelemetryPersistenceValidationException( + "write", + "The OpenTelemetry persistence operation contains a record that cannot be represented canonically.", + context, + exception); + } + + try + { + var fingerprint = CaptureFingerprint(traces, spans, points, logs); + var targets = new[] + { + new StreamTarget("traces", _stores.Traces, _traceStream, _traceDefinition, PhysicalizeRecordIds(batchId, "traces", traces)), + new StreamTarget("spans", _stores.Spans, _spanStream, _spanDefinition, PhysicalizeRecordIds(batchId, "spans", spans)), + new StreamTarget("metric-points", _stores.MetricPoints, _metricPointStream, _metricPointDefinition, PhysicalizeRecordIds(batchId, "metric-points", points)), + new StreamTarget("logs", _stores.Logs, _logStream, _logDefinition, PhysicalizeRecordIds(batchId, "logs", logs)) + }; + Preflight(batchId, targets); + await GetOrCreateCaptureOperationAsync(batchId, fingerprint, cancellationToken); + + foreach (var target in targets) + await AppendAsync(batchId, fingerprint, target, cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (OpenTelemetryPersistenceException) + { + throw; + } + catch (Exception exception) + { + throw TranslateFailure("write", context, exception); + } + } + + public ValueTask QueryResourcesAsync( + OpenTelemetryResourceFilter filter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(filter); + throw UnsupportedCaseIdentity( + "query-resources", + "Resource catalog results require case-insensitive resource identity."); + } + + public ValueTask QueryTracesAsync( + OpenTelemetryTraceFilter filter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(filter); + throw UnsupportedCaseIdentity( + "query-traces", + "Trace results require case-insensitive latest-per-trace identity."); + } + + public ValueTask GetTraceAsync( + string traceId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(traceId); + throw UnsupportedCaseIdentity( + "get-trace", + "Trace detail lookup requires case-insensitive trace identity and catalog joins."); + } + + public ValueTask QueryMetricsAsync( + OpenTelemetryMetricFilter filter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(filter); + throw UnsupportedCaseIdentity( + "query-metrics", + "Metric results require case-insensitive instrument identity and catalog joins."); + } + + public async ValueTask QueryLogsAsync( + OpenTelemetryLogFilter filter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(filter); + RejectUnsupportedTextFilter(filter.ServiceName, nameof(filter.ServiceName)); + RejectUnsupportedTextFilter(filter.Search, nameof(filter.Search)); + RejectUnsupportedTextFilter(filter.ResourceId, nameof(filter.ResourceId)); + RejectUnsupportedTextFilter(filter.TraceId, nameof(filter.TraceId)); + RejectUnsupportedTextFilter(filter.SpanId, nameof(filter.SpanId)); + RejectUnsupportedTextFilter(filter.Severity, nameof(filter.Severity)); + RejectOneSidedRange(filter.From, filter.To); + var take = ClampTake(filter.Take); + if (take == 0) + return new([], 0); + + try + { + var predicates = new List(); + AddRange(predicates, RecordFields.Timestamp, filter.From, filter.To); + var page = await _stores.Logs.QueryAsync(new( + _scope, + _logStream, + take, + new(RecordFields.Timestamp, DiagnosticSortDirection.Descending), + Predicate: All(predicates)), cancellationToken); + return new(page.Records.Select(_recordSerializer.ToLog).Reverse().ToArray(), 0); + } + catch (OperationCanceledException) + { + throw; + } + catch (OpenTelemetryPersistenceException) + { + throw; + } + catch (Exception exception) + { + throw TranslateFailure("query-logs", Context(), exception); + } + } + + public async ValueTask GetDiagnosticsAsync(CancellationToken cancellationToken = default) + { + try + { + var traces = await _stores.Traces.InspectAsync(new(_scope, _traceStream), cancellationToken); + var spans = await _stores.Spans.InspectAsync(new(_scope, _spanStream), cancellationToken); + var points = await _stores.MetricPoints.InspectAsync(new(_scope, _metricPointStream), cancellationToken); + var logs = await _stores.Logs.InspectAsync(new(_scope, _logStream), cancellationToken); + var resources = await QueryDocumentsAsync(CatalogDocuments.ResourceKind, 0, cancellationToken); + var instruments = await QueryDocumentsAsync(CatalogDocuments.InstrumentKind, 0, cancellationToken); + return new( + _traceCapacity, + _spanCapacity, + _metricPointCapacity, + _logRecordCapacity, + ToCount(resources.TotalCount), + ToCount(traces.RetainedCount.Value), + ToCount(spans.RetainedCount.Value), + ToCount(instruments.TotalCount), + ToCount(points.RetainedCount.Value), + ToCount(logs.RetainedCount.Value), + 0, + 0, + 0, + 0); + } + catch (OperationCanceledException) + { + throw; + } + catch (OpenTelemetryPersistenceException) + { + throw; + } + catch (Exception exception) + { + throw TranslateFailure("get-diagnostics", Context(), exception); + } + } + + private async Task AppendAsync( + DiagnosticsDrainBatchId batchId, + string fingerprint, + StreamTarget target, + CancellationToken cancellationToken) + { + if (target.Records.Count == 0) + return; + + var attempt = await GetOrCreateStreamAttemptAsync( + batchId, + fingerprint, + target, + cancellationToken); + if (attempt.Committed) + return; + + var operationId = OperationId(batchId, target.Kind, attempt.IssuedAt); + var request = DiagnosticRecordBatch.Create(_scope, target.Stream, operationId, target.Records); + await target.Store.AppendAsync(request, cancellationToken); + await MarkStreamCommittedAsync(batchId, fingerprint, target.Kind, CancellationToken.None); + cancellationToken.ThrowIfCancellationRequested(); + } + + private async Task GetOrCreateCaptureOperationAsync( + DiagnosticsDrainBatchId batchId, + string fingerprint, + CancellationToken cancellationToken) + { + var documentId = batchId.ToString(); + for (var attempt = 0; attempt < MaxLedgerWriteAttempts; attempt++) + { + var existing = await LoadOperationAsync(documentId, cancellationToken); + if (existing is not null) + { + ValidateFingerprint(existing.Operation, batchId, fingerprint); + return; + } + + var candidate = new CaptureOperation( + LedgerSchemaVersion, + documentId, + fingerprint, + _timeProvider.GetUtcNow(), + _binding.TenantId, + _binding.ScopeId, + _binding.SourceId, + new Dictionary(StringComparer.Ordinal)); + var result = await SaveOperationAsync(candidate, 0, cancellationToken); + if (result.Status == DocumentStoreWriteStatus.Saved) + return; + } + + throw new InvalidOperationException("The OpenTelemetry capture operation could not be claimed after concurrent retries."); + } + + private async Task GetOrCreateStreamAttemptAsync( + DiagnosticsDrainBatchId batchId, + string fingerprint, + StreamTarget target, + CancellationToken cancellationToken) + { + var documentId = batchId.ToString(); + for (var retry = 0; retry < MaxLedgerWriteAttempts; retry++) + { + var snapshot = await LoadRequiredOperationAsync(documentId, cancellationToken); + ValidateFingerprint(snapshot.Operation, batchId, fingerprint); + if (snapshot.Operation.Streams.TryGetValue(target.Kind, out var existing)) + { + if (!existing.Committed && _timeProvider.GetUtcNow() > existing.RetryUntil) + throw new DiagnosticOperationExpiredException( + DiagnosticOperationKind.Append, + OperationId(batchId, target.Kind, existing.IssuedAt)); + return existing; + } + + var issuedAt = _timeProvider.GetUtcNow(); + var candidate = new CaptureStreamAttempt( + issuedAt, + issuedAt + target.Definition.AppendIdempotencyWindow + target.Definition.MaxOperationClockSkew, + Committed: false); + var streams = new Dictionary(snapshot.Operation.Streams, StringComparer.Ordinal) + { + [target.Kind] = candidate + }; + var result = await SaveOperationAsync( + snapshot.Operation with { Streams = streams }, + snapshot.Version, + cancellationToken); + if (result.Status == DocumentStoreWriteStatus.Saved) + return candidate; + } + + throw new InvalidOperationException($"OpenTelemetry stream attempt '{target.Kind}' could not be claimed after concurrent retries."); + } + + private async Task MarkStreamCommittedAsync( + DiagnosticsDrainBatchId batchId, + string fingerprint, + string streamKind, + CancellationToken cancellationToken) + { + var documentId = batchId.ToString(); + for (var retry = 0; retry < MaxLedgerWriteAttempts; retry++) + { + var snapshot = await LoadRequiredOperationAsync(documentId, cancellationToken); + ValidateFingerprint(snapshot.Operation, batchId, fingerprint); + if (!snapshot.Operation.Streams.TryGetValue(streamKind, out var attempt)) + throw new InvalidOperationException($"OpenTelemetry stream attempt '{streamKind}' is missing from the capture ledger."); + if (attempt.Committed) + return; + + var streams = new Dictionary(snapshot.Operation.Streams, StringComparer.Ordinal) + { + [streamKind] = attempt with { Committed = true } + }; + var result = await SaveOperationAsync( + snapshot.Operation with { Streams = streams }, + snapshot.Version, + cancellationToken); + if (result.Status == DocumentStoreWriteStatus.Saved) + return; + } + + throw new InvalidOperationException($"OpenTelemetry stream attempt '{streamKind}' could not be completed after concurrent retries."); + } + + private async Task LoadOperationAsync( + string documentId, + CancellationToken cancellationToken) + { + var envelope = await _stores.Documents.LoadAsync( + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind, + documentId, + cancellationToken); + return envelope is null ? null : ReadOperation(envelope); + } + + private async Task LoadRequiredOperationAsync( + string documentId, + CancellationToken cancellationToken) => + await LoadOperationAsync(documentId, cancellationToken) + ?? throw new OpenTelemetryPersistenceDataException( + "write", + "The OpenTelemetry capture operation disappeared during an active write.", + Context(("batchId", documentId))); + + private Task SaveOperationAsync( + CaptureOperation operation, + long expectedVersion, + CancellationToken cancellationToken) => + _stores.Documents.SaveAsync(new( + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind, + operation.BatchId, + OpenTelemetryGroundworkStorageSchema.SchemaVersion, + JsonSerializer.Serialize(operation, LedgerJsonOptions), + expectedVersion), cancellationToken); + + private CaptureOperationSnapshot ReadOperation(DocumentEnvelope envelope) + { + try + { + if (!StringComparer.Ordinal.Equals(envelope.SchemaVersion, OpenTelemetryGroundworkStorageSchema.SchemaVersion)) + throw CorruptOperation(envelope.Id, "The OpenTelemetry capture operation schema is unsupported."); + if (!StringComparer.Ordinal.Equals(envelope.DocumentKind, OpenTelemetryGroundworkStorageSchema.OperationLedgerKind)) + throw CorruptOperation(envelope.Id, "The OpenTelemetry capture operation kind is invalid."); + var operation = JsonSerializer.Deserialize(envelope.ContentJson, LedgerJsonOptions) + ?? throw CorruptOperation(envelope.Id, "The OpenTelemetry capture operation is empty."); + if (operation.LedgerSchemaVersion != LedgerSchemaVersion || + !StringComparer.Ordinal.Equals(operation.BatchId, envelope.Id) || + !StringComparer.Ordinal.Equals(operation.TenantId, _binding.TenantId) || + !StringComparer.Ordinal.Equals(operation.ScopeId, _binding.ScopeId) || + !StringComparer.Ordinal.Equals(operation.SourceId, _binding.SourceId) || + operation.Streams is null || + operation.Streams.Keys.Except(StreamKinds, StringComparer.Ordinal).Any() || + operation.Streams.Values.Any(x => x is null || x.RetryUntil < x.IssuedAt)) + { + throw CorruptOperation(envelope.Id, "The OpenTelemetry capture operation does not match this storage binding."); + } + return new(operation, envelope.Version); + } + catch (Exception exception) when (exception is JsonException or NotSupportedException) + { + throw new OpenTelemetryPersistenceDataException( + "write", + "The OpenTelemetry capture operation is malformed.", + Context(("batchId", envelope.Id)), + exception); + } + } + + private static void ValidateFingerprint( + CaptureOperation operation, + DiagnosticsDrainBatchId batchId, + string fingerprint) + { + if (StringComparer.Ordinal.Equals(operation.Fingerprint, fingerprint)) + return; + throw new DiagnosticOperationConflictException( + DiagnosticOperationKind.Append, + new(operation.CreatedAt, batchId.ToString())); + } + + private void Preflight(DiagnosticsDrainBatchId batchId, IReadOnlyList targets) + { + var issuedAt = _timeProvider.GetUtcNow(); + foreach (var target in targets.Where(x => x.Records.Count > 0)) + { + var request = DiagnosticRecordBatch.Create( + _scope, + target.Stream, + OperationId(batchId, target.Kind, issuedAt), + target.Records); + DiagnosticRecordRequestValidator.Validate(request, target.Definition); + } + } + + private static void RejectCatalogIdentityWrites(OpenTelemetryBatch batch) + { + if (batch.Resources.Count > 0 || batch.Instruments.Count > 0) + { + throw UnsupportedCaseIdentity( + "write", + "Resource and instrument catalog writes require provider-neutral case-insensitive document identity."); + } + } + + private static DiagnosticOperationId OperationId( + DiagnosticsDrainBatchId batchId, + string streamKind, + DateTimeOffset issuedAt) => + new(issuedAt, $"otel-v2:{batchId}:{streamKind}"); + + private static DiagnosticRecordInput[] PhysicalizeRecordIds( + DiagnosticsDrainBatchId batchId, + string streamKind, + IReadOnlyList records) => + records.Select(record => record with + { + RecordId = $"otel-{Hash($"{batchId}:{streamKind}:{record.RecordId}")}" + }).ToArray(); + + private DiagnosticRecordInput[] NormalizeTraces(IReadOnlyCollection values) => + NormalizeRecords(values, trace => + { + var provisional = _recordSerializer.ToRecord(trace.TraceId, trace); + return provisional with { RecordId = $"trace-{Hash(provisional.Payload)}" }; + }); + + private static DiagnosticRecordInput[] NormalizeRecords( + IReadOnlyCollection values, + Func map) + { + ArgumentNullException.ThrowIfNull(values); + var result = new Dictionary(StringComparer.Ordinal); + foreach (var value in values) + { + var record = map(value); + if (result.TryGetValue(record.RecordId, out var existing)) + { + if (!RecordEquals(existing, record)) + throw new ArgumentException( + $"OpenTelemetry record id '{record.RecordId}' identifies conflicting payloads in one batch.", + nameof(values)); + continue; + } + result.Add(record.RecordId, record); + } + return result.Values.OrderBy(x => x.RecordId, StringComparer.Ordinal).ToArray(); + } + + private string CaptureFingerprint( + IReadOnlyList traces, + IReadOnlyList spans, + IReadOnlyList points, + IReadOnlyList logs) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + Append(hash, "elsa-open-telemetry-capture-v2"); + Append(hash, _binding.TenantId); + Append(hash, _binding.ScopeId); + Append(hash, _binding.SourceId); + Append(hash, DiagnosticRequestFingerprint.ForAppend(_scope, _traceStream, traces).Value); + Append(hash, DiagnosticRequestFingerprint.ForAppend(_scope, _spanStream, spans).Value); + Append(hash, DiagnosticRequestFingerprint.ForAppend(_scope, _metricPointStream, points).Value); + Append(hash, DiagnosticRequestFingerprint.ForAppend(_scope, _logStream, logs).Value); + return Convert.ToHexStringLower(hash.GetHashAndReset()); + } + +#pragma warning disable GW0004 + private Task QueryDocumentsAsync(string kind, int take, CancellationToken cancellationToken) => + _stores.Documents.QueryAsync(new PortableDocumentQuery(kind, take: take), cancellationToken); +#pragma warning restore GW0004 + + private static void AddRange( + List predicates, + string field, + DateTimeOffset? from, + DateTimeOffset? to) + { + if (from is not null && to is not null) + predicates.Add(DiagnosticRecordPredicate.RangeInclusive( + field, + DiagnosticFieldValue.Timestamp(from.Value), + DiagnosticFieldValue.Timestamp(to.Value))); + } + + private static DiagnosticRecordPredicate? All(IReadOnlyList predicates) => + predicates.Count switch + { + 0 => null, + 1 => predicates[0], + _ => new DiagnosticRecordPredicate.All(predicates) + }; + + private static void RejectUnsupportedTextFilter(string? value, string parameterName) + { + if (!string.IsNullOrWhiteSpace(value)) + throw new OpenTelemetryPersistenceCapabilityException( + "query-logs", + "portable-text-filter", + $"Filter '{parameterName}' requires the portable comparison-key or long-text query work tracked separately from restart persistence.", + Context(("filter", parameterName))); + } + + private static OpenTelemetryPersistenceCapabilityException UnsupportedCaseIdentity( + string operation, + string detail) => + new( + operation, + "portable-case-equivalence", + $"{detail} This operation is unavailable until the portable case-equivalence route is available."); + + private static void RejectOneSidedRange(DateTimeOffset? from, DateTimeOffset? to) + { + if ((from is null) != (to is null)) + throw new OpenTelemetryPersistenceCapabilityException( + "query-logs", + "one-sided-time-range", + "One-sided OpenTelemetry ranges require the portable query-at-scale capability."); + } + + private static OpenTelemetryPersistenceDataException CorruptOperation(string batchId, string message) => + new("write", message, Context(("batchId", batchId))); + + private static OpenTelemetryPersistenceException TranslateFailure( + string operation, + IReadOnlyDictionary context, + Exception exception) => exception switch + { + DiagnosticOperationConflictException => new OpenTelemetryPersistenceConflictException( + operation, + "The OpenTelemetry persistence operation conflicts with an existing request.", + context, + exception), + DiagnosticOperationExpiredException => new OpenTelemetryPersistenceExpiredException( + operation, + "The OpenTelemetry persistence operation can no longer be retried safely.", + context, + exception), + DiagnosticRecordValidationException => new OpenTelemetryPersistenceValidationException( + operation, + "The OpenTelemetry persistence operation contains an invalid record.", + context, + exception), + RecordPayloadException when operation == "write" => new OpenTelemetryPersistenceValidationException( + operation, + "The OpenTelemetry persistence operation contains a record that cannot be represented canonically.", + context, + exception), + RecordPayloadException => new OpenTelemetryPersistenceDataException( + operation, + "The OpenTelemetry persistence operation encountered a malformed durable record.", + context, + exception), + JsonException => new OpenTelemetryPersistenceDataException( + operation, + "The OpenTelemetry persistence operation encountered malformed durable data.", + context, + exception), + _ => new OpenTelemetryPersistenceUnavailableException( + operation, + "The OpenTelemetry persistence operation could not be completed.", + context, + exception) + }; + + private static IReadOnlyDictionary Context( + params (string Key, string Value)[] values) => + values.ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal); + + private int ClampTake(int? requested) => Math.Clamp(requested ?? _maxQuerySize, 0, _maxQuerySize); + + private static int ClampCapacity(int value) => Math.Max(1, value); + + private static int ToCount(long value) => value >= int.MaxValue ? int.MaxValue : (int)Math.Max(0, value); + + private static string Hash(string value) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private static void Append(IncrementalHash hash, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + Span length = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(length, bytes.Length); + hash.AppendData(length); + hash.AppendData(bytes); + } + + private static bool RecordEquals(DiagnosticRecordInput first, DiagnosticRecordInput second) + { + if (first.OccurredAt != second.OccurredAt || !StringComparer.Ordinal.Equals(first.Payload, second.Payload)) + return false; + var firstFields = first.Fields ?? new Dictionary>(); + var secondFields = second.Fields ?? new Dictionary>(); + return firstFields.Count == secondFields.Count && firstFields.All(field => + secondFields.TryGetValue(field.Key, out var values) && field.Value.SequenceEqual(values)); + } + + private sealed record CaptureOperation( + int LedgerSchemaVersion, + string BatchId, + string Fingerprint, + DateTimeOffset CreatedAt, + string TenantId, + string ScopeId, + string SourceId, + Dictionary Streams); + + private sealed record CaptureStreamAttempt( + DateTimeOffset IssuedAt, + DateTimeOffset RetryUntil, + bool Committed); + + private sealed record CaptureOperationSnapshot(CaptureOperation Operation, long Version); + + private sealed record StreamTarget( + string Kind, + IDiagnosticRecordStore Store, + DiagnosticStreamId Stream, + DiagnosticRecordStreamDefinition Definition, + IReadOnlyList Records); +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStores.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStores.cs new file mode 100644 index 000000000..ed2ef5094 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/GroundworkOpenTelemetryStores.cs @@ -0,0 +1,12 @@ +using Groundwork.DiagnosticRecords; +using Groundwork.Documents.Store; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork; + +/// The five provider sessions used by one explicitly bound OpenTelemetry adapter. +public sealed record GroundworkOpenTelemetryStores( + IDiagnosticRecordStore Traces, + IDiagnosticRecordStore Spans, + IDiagnosticRecordStore MetricPoints, + IDiagnosticRecordStore Logs, + IDocumentStore Documents); diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/OpenTelemetryGroundworkStorageSchema.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/OpenTelemetryGroundworkStorageSchema.cs new file mode 100644 index 000000000..4b1ade8d9 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/OpenTelemetryGroundworkStorageSchema.cs @@ -0,0 +1,215 @@ +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; +using Groundwork.Core.Indexing; +using Groundwork.Core.Intents; +using Groundwork.Core.Manifests; +using Groundwork.Core.PhysicalStorage; +using Groundwork.Core.Queries; +using Groundwork.DiagnosticRecords; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork; + +/// Complete Groundwork stream, physical catalog, index, and operation-ledger declarations for OpenTelemetry. +public static class OpenTelemetryGroundworkStorageSchema +{ + public const string SchemaVersion = "1.0.0"; + public const string OperationLedgerKind = "openTelemetryCaptureOperation"; + + public const string ByResourceStatusIndex = "by-resource-status"; + public const string ByInstrumentResourceIndex = "by-resource"; + public const string ByInstrumentKindIndex = "by-kind"; + public const string ByLastSeenIndex = "by-last-seen"; + public const string ByRetentionTieBreakerIndex = "by-retention-tie-breaker"; + public const string ByRetentionIndex = "by-retention"; + + public static IReadOnlySet RequiredCapabilities { get; } = + new HashSet(StringComparer.Ordinal) { "schema-history", "optimistic-concurrency" }; + + public static IReadOnlySet RequiredOperationLedgers { get; } = + new HashSet(StringComparer.Ordinal) + { + "diagnostic-append-idempotency", + "open-telemetry-capture-operation" + }; + + public static IReadOnlyList CreateStreams(GroundworkOpenTelemetryBinding binding) + { + ArgumentNullException.ThrowIfNull(binding); + binding.ValidateAll(); + return + [ + OpenTelemetryRecordStreamDefinitions.CreateTraces(binding.TraceStreamId), + OpenTelemetryRecordStreamDefinitions.CreateSpans(binding.SpanStreamId), + OpenTelemetryRecordStreamDefinitions.CreateMetricPoints(binding.MetricPointStreamId), + OpenTelemetryRecordStreamDefinitions.CreateLogs(binding.LogStreamId) + ]; + } + + /// + /// Creates scoped, dedicated document tables for the two mutable catalogs and capture-operation ledger. + /// Canonical JSON remains authoritative while declared projected columns service exact bounded lookups. + /// + public static StorageManifest CreateDocumentManifest() => new( + new StorageManifestIdentity("elsa-open-telemetry-diagnostics"), + new StorageManifestOwner("elsa.diagnostics.open-telemetry"), + new StorageManifestVersion(SchemaVersion), + [ + CatalogUnit( + CatalogDocuments.ResourceKind, + "OpenTelemetry resource catalog", + "elsa_open_telemetry_resources", + "elsa_open_telemetry_resource_indexes", + [ + Column(CatalogDocuments.ResourceStatusPath, PortablePhysicalType.Int32), + Column(CatalogDocuments.LastSeenPath, PortablePhysicalType.Int64), + Column(CatalogDocuments.RetentionTieBreakerPath, PortablePhysicalType.String, length: 512) + ], + [ + Logical(ByResourceStatusIndex, CatalogDocuments.ResourceStatusPath, IndexValueKind.Number), + Logical(ByLastSeenIndex, CatalogDocuments.LastSeenPath, IndexValueKind.Number), + Logical(ByRetentionTieBreakerIndex, CatalogDocuments.RetentionTieBreakerPath, IndexValueKind.Keyword) + ], + [ + Query("resources-by-status", ByResourceStatusIndex), + Query("resources-by-last-seen", ByLastSeenIndex, sortable: true), + Query("resources-by-retention-tie-breaker", ByRetentionTieBreakerIndex, sortable: true) + ]), + CatalogUnit( + CatalogDocuments.InstrumentKind, + "OpenTelemetry metric instrument catalog", + "elsa_open_telemetry_instruments", + "elsa_open_telemetry_instrument_indexes", + [ + Column(CatalogDocuments.InstrumentResourceIdPath, PortablePhysicalType.String, length: 512), + Column(CatalogDocuments.InstrumentKindPath, PortablePhysicalType.Int32), + Column(CatalogDocuments.LastSeenPath, PortablePhysicalType.Int64), + Column(CatalogDocuments.RetentionTieBreakerPath, PortablePhysicalType.String, length: 512) + ], + [ + Logical(ByInstrumentResourceIndex, CatalogDocuments.InstrumentResourceIdPath, IndexValueKind.Keyword), + Logical(ByInstrumentKindIndex, CatalogDocuments.InstrumentKindPath, IndexValueKind.Number), + Logical(ByLastSeenIndex, CatalogDocuments.LastSeenPath, IndexValueKind.Number), + Logical(ByRetentionTieBreakerIndex, CatalogDocuments.RetentionTieBreakerPath, IndexValueKind.Keyword) + ], + [ + Query("instruments-by-resource", ByInstrumentResourceIndex), + Query("instruments-by-kind", ByInstrumentKindIndex), + Query("instruments-by-last-seen", ByLastSeenIndex, sortable: true), + Query("instruments-by-retention-tie-breaker", ByRetentionTieBreakerIndex, sortable: true) + ]), + LedgerUnit() + ], + RequiredCapabilities, + [ + "Catalog identity writes and case-equivalence-dependent reads remain unavailable until Groundwork exposes the required portable comparison-key policy.", + "The capture-operation document binds a drain batch identity and tracks a bounded attempt for every non-empty record stream." + ]); + + private static StorageUnit CatalogUnit( + string documentKind, + string label, + string tableName, + string projectionName, + ProjectedColumnDefinition[] columns, + LogicalIndexDeclaration[] logicalIndexes, + BoundedQueryDeclaration[] queries) + { + var indexes = logicalIndexes + .Select(index => new PhysicalIndexDefinition( + index.Identity, + new[] { new PhysicalIndexColumnDefinition("storage_scope", 0) } + .Concat(index.Fields.Select((field, position) => + new PhysicalIndexColumnDefinition(Path(field.Path), position + 1))) + .ToArray())) + .Append(new PhysicalIndexDefinition( + ByRetentionIndex, + [ + new PhysicalIndexColumnDefinition("storage_scope", 0), + new PhysicalIndexColumnDefinition(Path(CatalogDocuments.LastSeenPath), 1), + new PhysicalIndexColumnDefinition(Path(CatalogDocuments.RetentionTieBreakerPath), 2) + ])) + .ToArray(); + var definition = PhysicalTableDefinition.DedicatedDocumentTable( + tableName, + indexes: indexes, + linkedProjectedColumns: columns, + linkedProjectionLogicalName: projectionName); + return Unit(documentKind, label, logicalIndexes, queries) with + { + PhysicalStorage = new( + StorageUnitProvisioningMode.Declared, + PhysicalStoragePolicy.Explicit(definition), + logicalIndexes, + queries) + }; + } + + private static StorageUnit LedgerUnit() => Unit( + OperationLedgerKind, + "OpenTelemetry capture operation ledger", + [], + []) with + { + PhysicalStorage = new( + StorageUnitProvisioningMode.Declared, + PhysicalStoragePolicy.Explicit(PhysicalTableDefinition.DedicatedDocumentTable( + "elsa_open_telemetry_capture_operations"))) + }; + +#pragma warning disable GW0001, GW0002, GW0003 + private static StorageUnit Unit( + string documentKind, + string label, + IReadOnlyList logicalIndexes, + IReadOnlyList boundedQueries) => new( + new StorageUnitIdentity(documentKind), + label, + StorageIntent.PortableDocument(), + LifecyclePolicy.Mutable, + IdentityPolicy.StringId(), + TenancyPolicy.Scoped, + ConcurrencyPolicy.Optimistic(), + SerializationPolicy.Json(), + logicalIndexes.Select(index => new IndexDeclaration( + index.Identity, + index.Fields, + index.ValueKind, + index.IsUnique, + boundedQueries.Any(query => query.IndexIdentity == index.Identity && query.SortSupport != QuerySortSupport.None), + index.MissingValueBehavior, + boundedQueries + .Where(query => query.IndexIdentity == index.Identity) + .SelectMany(query => query.Operations) + .ToHashSet())).ToArray(), + boundedQueries.Select(query => new PortableQueryDeclaration( + query.Identity, + query.IndexIdentity, + query.Operations, + query.SortSupport, + query.PagingSupport)).ToArray(), + PhysicalizationPolicy.Portable); +#pragma warning restore GW0001, GW0002, GW0003 + + private static ProjectedColumnDefinition Column( + string path, + PortablePhysicalType type, + int? length = null) => new(Path(path), Path(path), type, Length: length, IsNullable: false); + + private static LogicalIndexDeclaration Logical(string identity, string path, IndexValueKind kind) => new( + identity, + [new IndexField(Path(path))], + kind, + false, + MissingValueBehavior.Excluded); + + private static BoundedQueryDeclaration Query(string name, string indexName, bool sortable = false) => new( + name, + indexName, + new HashSet { PortableQueryOperation.Equal }, + sortable ? QuerySortSupport.Both : QuerySortSupport.None, + QueryPagingSupport.Offset, + BoundedQueryExecutionClass.ScaleBearing, + supportsTotalCount: true); + + private static string Path(string path) => path.TrimStart('/'); +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/CanonicalRecordSerializer.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/CanonicalRecordSerializer.cs new file mode 100644 index 000000000..4377c33a8 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/CanonicalRecordSerializer.cs @@ -0,0 +1,493 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Groundwork.DiagnosticRecords; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; + +/// +/// Maps immutable OpenTelemetry signals to canonical Groundwork diagnostic records and restores their domain models. +/// +public sealed class CanonicalRecordSerializer +{ + public const int SchemaVersion = 1; + + private const string TraceKind = "trace"; + private const string SpanKind = "span"; + private const string MetricPointKind = "metricPoint"; + private const string LogKind = "log"; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = false, + RespectRequiredConstructorParameters = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow + }; + + /// Creates the immutable record form of a trace summary. + /// The trace contains data that cannot be represented canonically. + public DiagnosticRecordInput ToRecord(string recordId, TelemetryTrace trace) + { + ArgumentException.ThrowIfNullOrWhiteSpace(recordId); + ArgumentNullException.ThrowIfNull(trace); + + return Map(() => + { + var traceId = RequiredText(trace.TraceId, nameof(trace.TraceId)); + var resourceIds = CanonicalSet(trace.ResourceIds, nameof(trace.ResourceIds), requireValue: true); + var workflowInstanceIds = CanonicalSet(trace.WorkflowInstanceIds, nameof(trace.WorkflowInstanceIds)); + EnumValue((int)trace.Status, nameof(trace.Status)); + + return new( + recordId, + trace.StartTime, + Serialize(TraceKind, new TracePayload( + traceId, + trace.RootSpanId, + trace.Name, + trace.StartTime.UtcTicks, + trace.EndTime.UtcTicks, + trace.Duration.Ticks, + (int)trace.Status, + resourceIds, + workflowInstanceIds, + trace.SpanCount)), + Fields( + (RecordFields.TraceId, Strings(traceId)), + (RecordFields.ResourceId, Strings(resourceIds)), + (RecordFields.WorkflowInstanceId, Strings(workflowInstanceIds)), + (RecordFields.Status, Int64((long)trace.Status)), + (RecordFields.StartTime, Timestamp(trace.StartTime)), + (RecordFields.EndTime, Timestamp(trace.EndTime)), + (RecordFields.Name, OptionalString(trace.Name)))); + }); + } + + /// Creates the immutable record form of a span. + /// The span contains data that cannot be represented canonically. + public DiagnosticRecordInput ToRecord(string recordId, TelemetrySpan span) + { + ArgumentException.ThrowIfNullOrWhiteSpace(recordId); + ArgumentNullException.ThrowIfNull(span); + + return Map(() => + { + var id = RequiredText(span.Id, nameof(span.Id)); + var traceId = RequiredText(span.TraceId, nameof(span.TraceId)); + var spanId = RequiredText(span.SpanId, nameof(span.SpanId)); + var resourceId = RequiredText(span.ResourceId, nameof(span.ResourceId)); + var name = RequiredText(span.Name, nameof(span.Name)); + var kind = RequiredText(span.Kind, nameof(span.Kind)); + EnumValue((int)span.Status, nameof(span.Status)); + + return new( + recordId, + span.StartTime, + Serialize(SpanKind, new SpanPayload( + id, + traceId, + spanId, + span.ParentSpanId, + resourceId, + name, + kind, + span.StartTime.UtcTicks, + span.EndTime.UtcTicks, + (int)span.Status, + span.StatusDescription, + Canonicalize(span.Attributes), + Required(span.Events, nameof(span.Events)).Select(ToPayload).ToArray(), + Required(span.Links, nameof(span.Links)).Select(ToPayload).ToArray())), + Fields( + (RecordFields.TraceId, Strings(traceId)), + (RecordFields.SpanId, Strings(spanId)), + (RecordFields.ParentSpanId, OptionalString(span.ParentSpanId)), + (RecordFields.ResourceId, Strings(resourceId)), + (RecordFields.Status, Int64((long)span.Status)), + (RecordFields.StartTime, Timestamp(span.StartTime)), + (RecordFields.EndTime, Timestamp(span.EndTime)), + (RecordFields.Name, Strings(name)))); + }); + } + + /// Creates the immutable record form of a metric point. + /// The point contains data that cannot be represented canonically. + public DiagnosticRecordInput ToRecord(string recordId, MetricPoint point) + { + ArgumentException.ThrowIfNullOrWhiteSpace(recordId); + ArgumentNullException.ThrowIfNull(point); + + return Map(() => + { + var id = RequiredText(point.Id, nameof(point.Id)); + var instrumentId = RequiredText(point.InstrumentId, nameof(point.InstrumentId)); + var instrumentName = RequiredText(point.InstrumentName, nameof(point.InstrumentName)); + var resourceId = RequiredText(point.ResourceId, nameof(point.ResourceId)); + + return new( + recordId, + point.Timestamp, + Serialize(MetricPointKind, new MetricPointPayload( + id, + instrumentId, + instrumentName, + resourceId, + point.Timestamp.UtcTicks, + ToBits(point.Value), + ToBits(point.Sum), + point.Count, + Canonicalize(point.Attributes), + point.TraceId, + point.SpanId)), + Fields( + (RecordFields.InstrumentId, Strings(instrumentId)), + (RecordFields.InstrumentName, Strings(instrumentName)), + (RecordFields.ResourceId, Strings(resourceId)), + (RecordFields.Timestamp, Timestamp(point.Timestamp)), + (RecordFields.TraceId, OptionalString(point.TraceId)), + (RecordFields.SpanId, OptionalString(point.SpanId)))); + }); + } + + /// Creates the immutable record form of an OTLP log record. + /// The log contains data that cannot be represented canonically. + public DiagnosticRecordInput ToRecord(string recordId, OtlpLogRecord log) + { + ArgumentException.ThrowIfNullOrWhiteSpace(recordId); + ArgumentNullException.ThrowIfNull(log); + + return Map(() => + { + var id = RequiredText(log.Id, nameof(log.Id)); + var resourceId = RequiredText(log.ResourceId, nameof(log.ResourceId)); + var severityText = Required(log.SeverityText, nameof(log.SeverityText)); + var body = Required(log.Body, nameof(log.Body)); + + return new( + recordId, + log.Timestamp, + Serialize(LogKind, new LogPayload( + id, + resourceId, + log.Timestamp.UtcTicks, + severityText, + log.SeverityNumber, + body, + log.TraceId, + log.SpanId, + Canonicalize(log.Attributes))), + Fields( + (RecordFields.ResourceId, Strings(resourceId)), + (RecordFields.Timestamp, Timestamp(log.Timestamp)), + (RecordFields.SeverityText, Strings(severityText)), + (RecordFields.SeverityNumber, OptionalInt64(log.SeverityNumber)), + (RecordFields.TraceId, OptionalString(log.TraceId)), + (RecordFields.SpanId, OptionalString(log.SpanId)), + (RecordFields.Body, Strings(body)))); + }); + } + + /// Restores a trace summary from a durable record. + /// The record payload is malformed or has an incompatible kind or schema. + public TelemetryTrace ToTrace(DiagnosticRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return Deserialize(record.Payload, TraceKind, x => new( + RequiredText(x.TraceId, nameof(x.TraceId)), + x.RootSpanId, + x.Name, + Utc(x.StartTimeUtcTicks), + Utc(x.EndTimeUtcTicks), + TimeSpan.FromTicks(x.DurationTicks), + EnumValue(x.Status, nameof(x.Status)), + CanonicalSet(x.ResourceIds, nameof(x.ResourceIds), requireValue: true), + CanonicalSet(x.WorkflowInstanceIds, nameof(x.WorkflowInstanceIds)), + x.SpanCount)); + } + + /// Restores a span from a durable record. + /// The record payload is malformed or has an incompatible kind or schema. + public TelemetrySpan ToSpan(DiagnosticRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return Deserialize(record.Payload, SpanKind, x => new( + RequiredText(x.Id, nameof(x.Id)), + RequiredText(x.TraceId, nameof(x.TraceId)), + RequiredText(x.SpanId, nameof(x.SpanId)), + x.ParentSpanId, + RequiredText(x.ResourceId, nameof(x.ResourceId)), + RequiredText(x.Name, nameof(x.Name)), + RequiredText(x.Kind, nameof(x.Kind)), + Utc(x.StartTimeUtcTicks), + Utc(x.EndTimeUtcTicks), + EnumValue(x.Status, nameof(x.Status)), + x.StatusDescription, + Restore(x.Attributes), + Required(x.Events, nameof(x.Events)).Select(ToModel).ToArray(), + Required(x.Links, nameof(x.Links)).Select(ToModel).ToArray())); + } + + /// Restores a metric point from a durable record. + /// The record payload is malformed or has an incompatible kind or schema. + public MetricPoint ToMetricPoint(DiagnosticRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return Deserialize(record.Payload, MetricPointKind, x => new( + RequiredText(x.Id, nameof(x.Id)), + RequiredText(x.InstrumentId, nameof(x.InstrumentId)), + RequiredText(x.InstrumentName, nameof(x.InstrumentName)), + RequiredText(x.ResourceId, nameof(x.ResourceId)), + Utc(x.TimestampUtcTicks), + FromBits(x.ValueBits), + FromBits(x.SumBits), + x.Count, + Restore(x.Attributes), + x.TraceId, + x.SpanId)); + } + + /// Restores an OTLP log record from a durable record. + /// The record payload is malformed or has an incompatible kind or schema. + public OtlpLogRecord ToLog(DiagnosticRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return Deserialize(record.Payload, LogKind, x => new( + RequiredText(x.Id, nameof(x.Id)), + RequiredText(x.ResourceId, nameof(x.ResourceId)), + Utc(x.TimestampUtcTicks), + Required(x.SeverityText, nameof(x.SeverityText)), + x.SeverityNumber, + Required(x.Body, nameof(x.Body)), + x.TraceId, + x.SpanId, + Restore(x.Attributes))); + } + + private static DiagnosticRecordInput Map(Func map) + { + try + { + return map(); + } + catch (RecordPayloadException) + { + throw; + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException) + { + throw new RecordPayloadException("The OpenTelemetry record cannot be represented as a canonical payload.", exception); + } + } + + private static string Serialize(string kind, TPayload value) + { + try + { + return JsonSerializer.Serialize(new RecordEnvelope(SchemaVersion, kind, value), SerializerOptions); + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException) + { + throw new RecordPayloadException("The OpenTelemetry record cannot be serialized.", exception); + } + } + + private static TModel Deserialize(string payload, string expectedKind, Func map) + where TPayload : class + { + try + { + var envelope = JsonSerializer.Deserialize>(payload, SerializerOptions) + ?? throw new RecordPayloadException("The OpenTelemetry record payload is empty."); + if (envelope.SchemaVersion != SchemaVersion) + throw new RecordPayloadException($"OpenTelemetry record schema '{envelope.SchemaVersion}' is not supported."); + if (!StringComparer.Ordinal.Equals(envelope.Kind, expectedKind)) + throw new RecordPayloadException($"OpenTelemetry record kind '{envelope.Kind}' is not '{expectedKind}'."); + if (envelope.Value is null) + throw new RecordPayloadException("The OpenTelemetry record payload has no value."); + return map(envelope.Value); + } + catch (RecordPayloadException) + { + throw; + } + catch (Exception exception) when (exception is ArgumentException or JsonException or NotSupportedException or OverflowException) + { + throw new RecordPayloadException("The OpenTelemetry record payload is malformed.", exception); + } + } + + private static Dictionary> Fields( + params (string Name, IReadOnlyList Values)[] fields) + { + var result = new Dictionary>(StringComparer.Ordinal); + foreach (var (name, values) in fields) + { + if (values.Count > 0) + result.Add(name, values); + } + return result; + } + + private static IReadOnlyList Strings(string value) => [DiagnosticFieldValue.String(value)]; + + private static IReadOnlyList Strings(IEnumerable values) => + values.Select(DiagnosticFieldValue.String).ToArray(); + + private static IReadOnlyList OptionalString(string? value) => + value is null ? [] : Strings(value); + + private static IReadOnlyList Int64(long value) => [DiagnosticFieldValue.Int64(value)]; + + private static IReadOnlyList OptionalInt64(long? value) => + value is null ? [] : Int64(value.Value); + + private static IReadOnlyList Timestamp(DateTimeOffset value) => + [DiagnosticFieldValue.Timestamp(value)]; + + private static AttributePayload[] Canonicalize(IDictionary attributes) => + Required(attributes, nameof(attributes)) + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => new AttributePayload(RequiredText(x.Key, "attribute key"), x.Value)) + .ToArray(); + + private static SpanEventPayload ToPayload(TelemetrySpanEvent? item) + { + var value = Required(item, "span event"); + return new(RequiredText(value.Name, nameof(value.Name)), value.Timestamp.UtcTicks, Canonicalize(value.Attributes)); + } + + private static SpanLinkPayload ToPayload(TelemetrySpanLink? item) + { + var value = Required(item, "span link"); + return new( + RequiredText(value.TraceId, nameof(value.TraceId)), + RequiredText(value.SpanId, nameof(value.SpanId)), + Canonicalize(value.Attributes)); + } + + private static TelemetrySpanEvent ToModel(SpanEventPayload? item) + { + var value = Required(item, "span event"); + return new(RequiredText(value.Name, nameof(value.Name)), Utc(value.TimestampUtcTicks), Restore(value.Attributes)); + } + + private static TelemetrySpanLink ToModel(SpanLinkPayload? item) + { + var value = Required(item, "span link"); + return new( + RequiredText(value.TraceId, nameof(value.TraceId)), + RequiredText(value.SpanId, nameof(value.SpanId)), + Restore(value.Attributes)); + } + + private static string[] CanonicalSet( + IEnumerable? values, + string field, + bool requireValue = false) + { + var result = Required(values, field) + .Select(x => RequiredText(x, field)) + .OrderBy(x => x, StringComparer.Ordinal) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (requireValue && result.Length == 0) + throw new RecordPayloadException($"The OpenTelemetry record field '{field}' requires at least one value."); + + return result; + } + + private static Dictionary Restore(AttributePayload[]? attributes) + { + if (attributes is null) + throw new RecordPayloadException("The OpenTelemetry record attributes are missing."); + + var result = new Dictionary(StringComparer.Ordinal); + foreach (var attribute in attributes) + { + var value = Required(attribute, "attribute"); + if (value.Key is null || !result.TryAdd(value.Key, value.Value)) + throw new RecordPayloadException("The OpenTelemetry record contains an invalid or duplicate attribute key."); + } + return result; + } + + private static T Required(T? value, string field) where T : class => + value ?? throw new RecordPayloadException($"The OpenTelemetry record field '{field}' is required."); + + private static string RequiredText(string? value, string field) => + !string.IsNullOrWhiteSpace(value) + ? value + : throw new RecordPayloadException($"The OpenTelemetry record field '{field}' requires a non-empty value."); + + private static TEnum EnumValue(int value, string field) where TEnum : struct, Enum => + Enum.IsDefined(typeof(TEnum), value) + ? (TEnum)Enum.ToObject(typeof(TEnum), value) + : throw new RecordPayloadException($"The OpenTelemetry record field '{field}' has an unsupported value."); + + private static DateTimeOffset Utc(long ticks) => new(ticks, TimeSpan.Zero); + + private static long? ToBits(double? value) => value.HasValue ? BitConverter.DoubleToInt64Bits(value.Value) : null; + + private static double? FromBits(long? bits) => bits.HasValue ? BitConverter.Int64BitsToDouble(bits.Value) : null; + + private sealed record RecordEnvelope(int SchemaVersion, string Kind, TPayload Value); + + private sealed record AttributePayload(string Key, string? Value); + + private sealed record TracePayload( + string TraceId, + string? RootSpanId, + string? Name, + long StartTimeUtcTicks, + long EndTimeUtcTicks, + long DurationTicks, + int Status, + string[] ResourceIds, + string[] WorkflowInstanceIds, + int SpanCount); + + private sealed record SpanPayload( + string Id, + string TraceId, + string SpanId, + string? ParentSpanId, + string ResourceId, + string Name, + string Kind, + long StartTimeUtcTicks, + long EndTimeUtcTicks, + int Status, + string? StatusDescription, + AttributePayload[] Attributes, + SpanEventPayload[] Events, + SpanLinkPayload[] Links); + + private sealed record SpanEventPayload(string Name, long TimestampUtcTicks, AttributePayload[] Attributes); + + private sealed record SpanLinkPayload(string TraceId, string SpanId, AttributePayload[] Attributes); + + private sealed record MetricPointPayload( + string Id, + string InstrumentId, + string InstrumentName, + string ResourceId, + long TimestampUtcTicks, + long? ValueBits, + long? SumBits, + long? Count, + AttributePayload[] Attributes, + string? TraceId, + string? SpanId); + + private sealed record LogPayload( + string Id, + string ResourceId, + long TimestampUtcTicks, + string SeverityText, + int? SeverityNumber, + string Body, + string? TraceId, + string? SpanId, + AttributePayload[] Attributes); +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/OpenTelemetryRecordStreamDefinitions.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/OpenTelemetryRecordStreamDefinitions.cs new file mode 100644 index 000000000..ada98a289 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/OpenTelemetryRecordStreamDefinitions.cs @@ -0,0 +1,133 @@ +using Groundwork.DiagnosticRecords; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; + +/// Creates the bounded Groundwork stream contracts for immutable OpenTelemetry signals. +public static class OpenTelemetryRecordStreamDefinitions +{ + private const int MaxIdentifierBytes = 512; + private const int MaxNameBytes = 4_096; + private const int MaxBodyBytes = 65_536; + private const int MaxMultiValues = 256; + + /// Creates the trace-summary stream definition for a host-selected stream identity. + public static DiagnosticRecordStreamDefinition CreateTraces(string streamId) => Definition( + streamId, + "elsa_open_telemetry_traces", + [ + String(RecordFields.TraceId, required: true, latestPerKey: true), + String(RecordFields.ResourceId, required: true, multiple: true), + String(RecordFields.WorkflowInstanceId, multiple: true), + Int64(RecordFields.Status, required: true), + Timestamp(RecordFields.StartTime, required: true, orderable: true), + Timestamp(RecordFields.EndTime, required: true), + String(RecordFields.Name, maxStringBytes: MaxNameBytes) + ]); + + /// Creates the span stream definition for a host-selected stream identity. + public static DiagnosticRecordStreamDefinition CreateSpans(string streamId) => Definition( + streamId, + "elsa_open_telemetry_spans", + [ + String(RecordFields.TraceId, required: true), + String(RecordFields.SpanId, required: true), + String(RecordFields.ParentSpanId), + String(RecordFields.ResourceId, required: true), + Int64(RecordFields.Status, required: true), + Timestamp(RecordFields.StartTime, required: true, orderable: true), + Timestamp(RecordFields.EndTime, required: true), + String(RecordFields.Name, required: true, maxStringBytes: MaxNameBytes) + ]); + + /// Creates the metric-point stream definition for a host-selected stream identity. + public static DiagnosticRecordStreamDefinition CreateMetricPoints(string streamId) => Definition( + streamId, + "elsa_open_telemetry_metric_points", + [ + String(RecordFields.InstrumentId, required: true), + String(RecordFields.InstrumentName, required: true, maxStringBytes: MaxNameBytes), + String(RecordFields.ResourceId, required: true), + Timestamp(RecordFields.Timestamp, required: true, orderable: true), + String(RecordFields.TraceId), + String(RecordFields.SpanId) + ]); + + /// Creates the telemetry-log stream definition for a host-selected stream identity. + public static DiagnosticRecordStreamDefinition CreateLogs(string streamId) => Definition( + streamId, + "elsa_open_telemetry_logs", + [ + String(RecordFields.ResourceId, required: true), + Timestamp(RecordFields.Timestamp, required: true, orderable: true), + String(RecordFields.SeverityText, required: true, maxStringBytes: MaxNameBytes), + Int64(RecordFields.SeverityNumber), + String(RecordFields.TraceId), + String(RecordFields.SpanId), + String(RecordFields.Body, required: true, maxStringBytes: MaxBodyBytes) + ]); + + private static DiagnosticRecordStreamDefinition Definition( + string streamId, + string storageName, + IReadOnlyList fields) + { + ArgumentException.ThrowIfNullOrWhiteSpace(streamId); + + var definition = new DiagnosticRecordStreamDefinition( + new(streamId), + CanonicalRecordSerializer.SchemaVersion, + storageName, + fields, + new( + MaxBatchRecords: 1_000, + MaxPayloadBytes: 1_048_576, + MaxRecordIdBytes: 256, + MaxFieldsPerRecord: fields.Count, + MaxQueryLimit: 5_000, + MaxPredicateNodes: 32, + MaxPredicateValues: 256, + MaxJsonDepth: 64), + MaxOperationClockSkew: TimeSpan.FromMinutes(5), + AppendIdempotencyWindow: TimeSpan.FromHours(1), + TrimIdempotencyWindow: TimeSpan.FromHours(1)); + + DiagnosticRecordStreamDefinitionValidator.ValidateAndThrow(definition); + return definition; + } + + private static DiagnosticFieldDefinition String( + string name, + bool required = false, + bool orderable = false, + bool latestPerKey = false, + bool multiple = false, + int maxStringBytes = MaxIdentifierBytes) => new( + name, + DiagnosticFieldType.String, + multiple ? DiagnosticFieldCardinality.Multiple : DiagnosticFieldCardinality.Scalar, + Set(DiagnosticPredicateOperator.Equal, DiagnosticPredicateOperator.In, DiagnosticPredicateOperator.Contains), + IsRequired: required, + IsOrderable: orderable, + SupportsLatestPerKey: latestPerKey, + CasePolicy: DiagnosticStringCasePolicy.Ordinal, + MaxValues: multiple ? MaxMultiValues : 1, + MaxStringBytes: maxStringBytes); + + private static DiagnosticFieldDefinition Int64(string name, bool required = false) => new( + name, + DiagnosticFieldType.Int64, + DiagnosticFieldCardinality.Scalar, + Set(DiagnosticPredicateOperator.Equal, DiagnosticPredicateOperator.In, DiagnosticPredicateOperator.RangeInclusive), + IsRequired: required); + + private static DiagnosticFieldDefinition Timestamp(string name, bool required = false, bool orderable = false) => new( + name, + DiagnosticFieldType.Timestamp, + DiagnosticFieldCardinality.Scalar, + Set(DiagnosticPredicateOperator.Equal, DiagnosticPredicateOperator.In, DiagnosticPredicateOperator.RangeInclusive), + IsRequired: required, + IsOrderable: orderable); + + private static IReadOnlySet Set(params DiagnosticPredicateOperator[] values) => + values.ToHashSet(); +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordFields.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordFields.cs new file mode 100644 index 000000000..b2b584e4a --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordFields.cs @@ -0,0 +1,21 @@ +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; + +/// Canonical field names projected from immutable OpenTelemetry records. +public static class RecordFields +{ + public const string TraceId = "traceId"; + public const string SpanId = "spanId"; + public const string ParentSpanId = "parentSpanId"; + public const string ResourceId = "resourceId"; + public const string WorkflowInstanceId = "workflowInstanceId"; + public const string Status = "status"; + public const string StartTime = "startTime"; + public const string EndTime = "endTime"; + public const string Name = "name"; + public const string InstrumentId = "instrumentId"; + public const string InstrumentName = "instrumentName"; + public const string Timestamp = "timestamp"; + public const string SeverityText = "severityText"; + public const string SeverityNumber = "severityNumber"; + public const string Body = "body"; +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordPayloadException.cs b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordPayloadException.cs new file mode 100644 index 000000000..893e7d778 --- /dev/null +++ b/src/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Records/RecordPayloadException.cs @@ -0,0 +1,13 @@ +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; + +/// Raised when an immutable OpenTelemetry record cannot be mapped to or from its durable payload. +public sealed class RecordPayloadException : Exception +{ + public RecordPayloadException(string message) : base(message) + { + } + + public RecordPayloadException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs b/src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs index e0225c840..cab0c48fb 100644 --- a/src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs +++ b/src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs @@ -3,11 +3,15 @@ using Elsa.Diagnostics.OpenTelemetry.Core.Contracts; using Elsa.Diagnostics.OpenTelemetry.Core.Models; using Elsa.Diagnostics.OpenTelemetry.Core.Options; +using Elsa.Diagnostics.Persistence.Observability; using Microsoft.Extensions.Options; namespace Elsa.Diagnostics.OpenTelemetry.Providers.InMemory; -public class InMemoryOpenTelemetryLiveFeed(IOptions options, IOpenTelemetrySourceRegistry sourceRegistry) : IOpenTelemetryLiveFeed +public sealed class InMemoryOpenTelemetryLiveFeed( + IOptions options, + IOpenTelemetrySourceRegistry sourceRegistry, + DiagnosticsSubscriberDeliveryLossBridge? deliveryLossBridge = null) : IOpenTelemetryLiveFeed { private readonly OpenTelemetryDiagnosticsOptions _options = options.Value; private readonly object _subscribersLock = new(); @@ -29,7 +33,11 @@ public IAsyncEnumerable SubscribeAsync(OpenTelemetryTra { // Register eagerly (not inside the async iterator) so telemetry published between obtaining the // enumerable and the first MoveNextAsync is still delivered — matching the structured-logs feed. - var subscriber = new OpenTelemetrySubscriber(filter, sourceRegistry, _options.SubscriberChannelCapacity); + var subscriber = new OpenTelemetrySubscriber( + filter, + sourceRegistry, + _options.SubscriberChannelCapacity, + deliveryLossBridge); lock (_subscribersLock) _subscribers.Add(subscriber); @@ -54,7 +62,11 @@ private async IAsyncEnumerable Enumerate(OpenTelemetryS } } - private sealed class OpenTelemetrySubscriber(OpenTelemetryTraceFilter filter, IOpenTelemetrySourceRegistry sourceRegistry, int channelCapacity) + private sealed class OpenTelemetrySubscriber( + OpenTelemetryTraceFilter filter, + IOpenTelemetrySourceRegistry sourceRegistry, + int channelCapacity, + DiagnosticsSubscriberDeliveryLossBridge? deliveryLossBridge) { private readonly object _lock = new(); private readonly int _capacity = Math.Max(1, channelCapacity); @@ -126,8 +138,18 @@ private void TrackDrop(OpenTelemetryStreamItem item) if (signalType == null) return; + var isReaggregatedSummary = item.DroppedItems is not null; _droppedSinceLastSummary.TryGetValue(signalType.Value, out var count); _droppedSinceLastSummary[signalType.Value] = count + (item.DroppedItems?.Count ?? 1); + // Raw items are accounted at their first loss. A summary may itself be evicted and requeued to + // preserve the in-band signal, but its underlying losses have already been observed. + if (!isReaggregatedSummary) + { + deliveryLossBridge?.RecordOpenTelemetry(new( + signalType.Value, + 1, + "SubscriberQueueFull")); + } } private void TryWriteDroppedSummary() @@ -139,7 +161,8 @@ private void TryWriteDroppedSummary() return; var dropped = _droppedSinceLastSummary.OrderBy(x => x.Key).First(); - if (Channel.Writer.TryWrite(new OpenTelemetryStreamItem { DroppedItems = new(dropped.Key, dropped.Value, "SubscriberQueueFull") })) + var summary = new OpenTelemetryDroppedItemSummary(dropped.Key, dropped.Value, "SubscriberQueueFull"); + if (Channel.Writer.TryWrite(new OpenTelemetryStreamItem { DroppedItems = summary })) { _pendingItemCount++; // Clear the emitted signal as soon as it is queued so a write before the client reads the diff --git a/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrain.cs b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrain.cs new file mode 100644 index 000000000..f4a28e9e8 --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrain.cs @@ -0,0 +1,424 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Elsa.Diagnostics.Persistence.Observability; + +namespace Elsa.Diagnostics.Persistence.Draining; + +/// +/// Composes one concrete target with Elsa-owned bounded queue, retry, acknowledgement, retention, and +/// shutdown policy. It contains no provider types and never emits captured diagnostic payloads. +/// +public sealed class DiagnosticsDrain : IDisposable, IAsyncDisposable where TItem : notnull +{ + private readonly IDiagnosticsDrainTarget _target; + private readonly DiagnosticsDrainOptions _options; + private readonly IDiagnosticsPersistenceObserver _observer; + private readonly Channel _channel; + private readonly CancellationTokenSource _shutdown = new(); + private readonly ConcurrentDictionary _accepted = new(); + private readonly object _lifecycleGate = new(); + private readonly object _shutdownGate = new(); + private Task? _drainLoop; + private Task? _stopTask; + private Task? _asyncDisposeTask; + private Task? _shutdownCancellationTask; + private long _retentionUnits; + private int _accepting = 1; + private int _forcedTermination; + private int _shutdownDisposed; + private int _state = (int)DiagnosticsDrainState.Created; + + public DiagnosticsDrain( + IDiagnosticsDrainTarget target, + DiagnosticsDrainOptions options, + IDiagnosticsPersistenceObserver? observer = null) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(options); + options.Validate(); + _target = target; + _options = options; + _observer = observer ?? NullDiagnosticsPersistenceObserver.Instance; + _channel = Channel.CreateBounded(new BoundedChannelOptions(options.QueueCapacity) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + SingleWriter = false + }, OnItemShed); + Observe(observer => observer.RecordState(DiagnosticsDrainState.Created)); + } + + public DiagnosticsDrainState State => (DiagnosticsDrainState)Volatile.Read(ref _state); + + /// Starts the single drain loop. Calling this more than once is harmless. + public void Start() + { + lock (_lifecycleGate) + { + if (_drainLoop is not null) + { + if (_stopTask is null && Volatile.Read(ref _forcedTermination) == 0) + return; + throw new InvalidOperationException("The diagnostics drain has entered its terminal lifecycle and cannot be restarted."); + } + if (Volatile.Read(ref _accepting) == 0 || _stopTask is not null) + throw new InvalidOperationException("The diagnostics drain is closing and cannot be started."); + StartCore(); + } + } + + /// + /// Accepts without waiting for provider I/O. The returned acknowledgement completes exactly once with + /// the authoritative commit result or an explicit loss failure. + /// + public bool TryEnqueue(TItem item, out Task acknowledgement) + { + ArgumentNullException.ThrowIfNull(item); + var pending = new PendingItem(item); + acknowledgement = pending.Completion.Task; + _accepted.TryAdd(pending, 0); + if (_channel.Writer.TryWrite(pending)) + return true; + + Fail(pending, DiagnosticsPersistenceLossReason.WriteAfterClosure, "The diagnostics drain is not accepting captures."); + return false; + } + + public ValueTask EnqueueAsync(TItem item, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + TryEnqueue(item, out var acknowledgement); + return new(acknowledgement.WaitAsync(cancellationToken)); + } + + /// + /// Closes producers, drains within the configured window, applies final retention, and returns an + /// observable stopped or timed-out outcome. Caller cancellation only cancels that caller's wait; the + /// shared stop continues so accepted acknowledgements are never abandoned. + /// + public async Task StopAsync(CancellationToken cancellationToken = default) + { + Task stopTask; + lock (_lifecycleGate) + { + if (_stopTask is null) + { + if (Volatile.Read(ref _forcedTermination) != 0) + _stopTask = Task.FromResult(new DiagnosticsDrainStopResult(DiagnosticsDrainState.TimedOut, Drained: false)); + else + { + _drainLoop ??= StartCore(); + Volatile.Write(ref _accepting, 0); + SetState(DiagnosticsDrainState.Closing); + _channel.Writer.TryComplete(); + SetState(DiagnosticsDrainState.Draining); + _stopTask = StopCoreAsync(_drainLoop); + } + } + + stopTask = _stopTask; + } + + return await stopTask.WaitAsync(cancellationToken); + } + + private Task StartCore() + { + SetState(DiagnosticsDrainState.Running); + return _drainLoop = Task.Run(() => RunAsync(_shutdown.Token)); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var reader = _channel.Reader; + var batch = new List(_options.BatchSize); + try + { + while (await reader.WaitToReadAsync(cancellationToken)) + { + batch.Clear(); + while (batch.Count < _options.BatchSize && reader.TryRead(out var item)) + batch.Add(item); + if (batch.Count == 0) + continue; + + var committedUnits = await CommitWithRetryAsync(batch.ToArray(), cancellationToken); + await ApplyPeriodicRetentionAsync(committedUnits, cancellationToken); + } + + await ApplyRetentionWithRetryAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The bounded stop path settles every accepted acknowledgement below. + } + finally + { + if (!_accepted.IsEmpty) + FailAll(DiagnosticsPersistenceLossReason.ShutdownTimeout, "The diagnostics capture was canceled before commit."); + } + } + + private async Task CommitWithRetryAsync( + IReadOnlyList pending, + CancellationToken cancellationToken) + { + var batch = new DiagnosticsDrainBatch(DiagnosticsDrainBatchId.New(), pending.Select(item => item.Item).ToArray()); + Exception? lastFailure = null; + for (var attempt = 1; attempt <= _options.MaxAttempts; attempt++) + { + try + { + var commit = await _target.CommitAsync(batch, cancellationToken); + if (commit.Results.Count != pending.Count) + throw new InvalidOperationException("The diagnostics target returned an incomplete commit result."); + ArgumentOutOfRangeException.ThrowIfNegative(commit.RetentionUnits); + for (var index = 0; index < pending.Count; index++) + Complete(pending[index], commit.Results[index]); + return commit.RetentionUnits; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + lastFailure = exception; + if (attempt == _options.MaxAttempts) + break; + Observe(observer => observer.RecordRetry(DiagnosticsPersistenceOperation.Commit, attempt, _options.MaxAttempts)); + await Task.Delay(RetryDelay(attempt), cancellationToken); + } + } + + Observe(observer => observer.RecordOperationFailure(DiagnosticsPersistenceOperation.Commit)); + foreach (var item in pending) + Fail(item, DiagnosticsPersistenceLossReason.RetryExhausted, "The diagnostics capture could not be committed after bounded retries.", lastFailure); + return 0; + } + + private async Task ApplyPeriodicRetentionAsync(int committedUnits, CancellationToken cancellationToken) + { + _retentionUnits += committedUnits; + if (_retentionUnits >= _options.RetentionInterval) + await ApplyRetentionWithRetryAsync(cancellationToken); + } + + private async Task ApplyRetentionWithRetryAsync(CancellationToken cancellationToken) + { + for (var attempt = 1; attempt <= _options.MaxAttempts; attempt++) + { + try + { + var deleted = await _target.ApplyRetentionAsync(cancellationToken); + ArgumentOutOfRangeException.ThrowIfNegative(deleted); + _retentionUnits = 0; + if (deleted > 0) + Observe(observer => observer.RecordLoss(DiagnosticsPersistenceLossReason.DurableRetentionDeletion, deleted)); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch when (attempt < _options.MaxAttempts) + { + Observe(observer => observer.RecordRetry(DiagnosticsPersistenceOperation.Retention, attempt, _options.MaxAttempts)); + await Task.Delay(RetryDelay(attempt), cancellationToken); + } + catch + { + Observe(observer => observer.RecordOperationFailure(DiagnosticsPersistenceOperation.Retention)); + return; + } + } + } + + private async Task StopCoreAsync(Task drainLoop) + { + try + { + await drainLoop.WaitAsync(_options.ShutdownTimeout); + lock (_lifecycleGate) + { + if (Volatile.Read(ref _forcedTermination) != 0) + return new(DiagnosticsDrainState.TimedOut, Drained: false); + SetState(DiagnosticsDrainState.Stopped); + } + DisposeShutdownSource(); + return new(DiagnosticsDrainState.Stopped, Drained: true); + } + catch (TimeoutException) + { + return TransitionToTimedOut( + drainLoop, + "The diagnostics drain exceeded its shutdown window."); + } + catch + { + return TransitionToTimedOut( + drainLoop, + "The diagnostics drain stopped before every acknowledgement completed."); + } + } + + private DiagnosticsDrainStopResult TransitionToTimedOut(Task drainLoop, string message) + { + lock (_lifecycleGate) + { + if (Volatile.Read(ref _forcedTermination) != 0) + return new(DiagnosticsDrainState.TimedOut, Drained: false); + Volatile.Write(ref _forcedTermination, 1); + Volatile.Write(ref _accepting, 0); + _channel.Writer.TryComplete(); + SetState(DiagnosticsDrainState.TimedOut); + } + + var cancellation = RequestShutdownCancellation(); + FailAll(DiagnosticsPersistenceLossReason.ShutdownTimeout, message); + ScheduleShutdownDisposal(drainLoop, cancellation); + return new(DiagnosticsDrainState.TimedOut, Drained: false); + } + + private TimeSpan RetryDelay(int completedAttempt) + { + var multiplier = Math.Pow(2, completedAttempt - 1); + var delay = _options.BaseRetryDelay * multiplier; + return delay <= _options.MaxRetryDelay ? delay : _options.MaxRetryDelay; + } + + private void OnItemShed(PendingItem pending) => + Fail(pending, DiagnosticsPersistenceLossReason.QueueOverflow, "The diagnostics capture was shed before commit because the queue was full."); + + private void Complete(PendingItem pending, TResult result) + { + if (!pending.Completion.TrySetResult(result)) + return; + _accepted.TryRemove(pending, out _); + } + + private void Fail( + PendingItem pending, + DiagnosticsPersistenceLossReason reason, + string message, + Exception? innerException = null) + { + if (!pending.Completion.TrySetException(new DiagnosticsDrainException(reason, message, innerException))) + return; + _accepted.TryRemove(pending, out _); + Observe(observer => observer.RecordLoss(reason, 1)); + } + + private void FailAll(DiagnosticsPersistenceLossReason reason, string message) + { + foreach (var pending in _accepted.Keys) + Fail(pending, reason, message); + } + + private void SetState(DiagnosticsDrainState state) + { + Volatile.Write(ref _state, (int)state); + Observe(observer => observer.RecordState(state)); + } + + private void Observe(Action report) + { + try + { + report(_observer); + } + catch + { + // Observability is deliberately outside the persistence correctness boundary. + } + } + + public void Dispose() + { + Task? drainLoop; + lock (_lifecycleGate) + { + if (State == DiagnosticsDrainState.Stopped) + { + ScheduleShutdownDisposal(_drainLoop); + return; + } + if (Volatile.Read(ref _forcedTermination) != 0) + return; + + Volatile.Write(ref _forcedTermination, 1); + Volatile.Write(ref _accepting, 0); + _channel.Writer.TryComplete(); + SetState(DiagnosticsDrainState.TimedOut); + _stopTask ??= Task.FromResult(new DiagnosticsDrainStopResult(DiagnosticsDrainState.TimedOut, Drained: false)); + drainLoop = _drainLoop; + } + + var cancellation = RequestShutdownCancellation(); + FailAll(DiagnosticsPersistenceLossReason.ShutdownTimeout, "The diagnostics drain was synchronously disposed before commit."); + ScheduleShutdownDisposal(drainLoop, cancellation); + } + + public ValueTask DisposeAsync() + { + lock (_lifecycleGate) + return new(_asyncDisposeTask ??= DisposeCoreAsync()); + } + + private async Task DisposeCoreAsync() => await StopAsync(); + + private Task RequestShutdownCancellation() + { + lock (_shutdownGate) + { + if (_shutdownCancellationTask is not null) + return _shutdownCancellationTask; + + try + { + return _shutdownCancellationTask = _shutdown.CancelAsync(); + } + catch (Exception exception) + { + return _shutdownCancellationTask = Task.FromException(exception); + } + } + } + + private void ScheduleShutdownDisposal(Task? drainLoop, Task? cancellation = null) + { + var cleanup = Task.WhenAll( + drainLoop ?? Task.CompletedTask, + cancellation ?? Task.CompletedTask); + if (cleanup.IsCompleted) + { + _ = cleanup.Exception; + DisposeShutdownSource(); + return; + } + + _ = cleanup.ContinueWith( + static (completed, state) => + { + _ = completed.Exception; + ((DiagnosticsDrain)state!).DisposeShutdownSource(); + }, + this, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + private void DisposeShutdownSource() + { + if (Interlocked.Exchange(ref _shutdownDisposed, 1) == 0) + _shutdown.Dispose(); + } + + private sealed class PendingItem(TItem item) + { + public TItem Item { get; } = item; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainContracts.cs b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainContracts.cs new file mode 100644 index 000000000..f90a6c92d --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainContracts.cs @@ -0,0 +1,51 @@ +using Elsa.Diagnostics.Persistence.Observability; + +namespace Elsa.Diagnostics.Persistence.Draining; + +/// A stable identity reused for every attempt to commit one drain batch. +public readonly record struct DiagnosticsDrainBatchId(Guid Value) +{ + public static DiagnosticsDrainBatchId New() => new(Guid.NewGuid()); + public override string ToString() => Value.ToString("N"); +} + +/// The bounded unit supplied to a concrete diagnostics persistence adapter. +public sealed record DiagnosticsDrainBatch( + DiagnosticsDrainBatchId Id, + IReadOnlyList Items) where TItem : notnull; + +/// An authoritative commit outcome in the same order as the accepted batch. +public sealed record DiagnosticsDrainCommit( + IReadOnlyList Results, + int RetentionUnits); + +/// +/// Replacement contract: exactly one target is composed per drain. Implementations keep provider +/// operations and operation-id binding in their concrete persistence project. +/// +public interface IDiagnosticsDrainTarget where TItem : notnull +{ + ValueTask> CommitAsync( + DiagnosticsDrainBatch batch, + CancellationToken cancellationToken = default); + + ValueTask ApplyRetentionAsync(CancellationToken cancellationToken = default); +} + +public sealed record DiagnosticsDrainStopResult( + DiagnosticsDrainState State, + bool Drained); + +/// A caller-visible failure for an accepted or rejected diagnostics capture. +public sealed class DiagnosticsDrainException : Exception +{ + public DiagnosticsDrainException( + DiagnosticsPersistenceLossReason reason, + string message, + Exception? innerException = null) : base(message, innerException) + { + Reason = reason; + } + + public DiagnosticsPersistenceLossReason Reason { get; } +} diff --git a/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainOptions.cs b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainOptions.cs new file mode 100644 index 000000000..603cfcb59 --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainOptions.cs @@ -0,0 +1,24 @@ +namespace Elsa.Diagnostics.Persistence.Draining; + +/// Bounds one diagnostics capture drain's queue, batches, retries, retention, and shutdown. +public sealed class DiagnosticsDrainOptions +{ + public int BatchSize { get; init; } = 100; + public int QueueCapacity { get; init; } = 1_000; + public int RetentionInterval { get; init; } = 5_000; + public int MaxAttempts { get; init; } = 9; + public TimeSpan BaseRetryDelay { get; init; } = TimeSpan.FromMilliseconds(50); + public TimeSpan MaxRetryDelay { get; init; } = TimeSpan.FromSeconds(5); + public TimeSpan ShutdownTimeout { get; init; } = TimeSpan.FromSeconds(30); + + internal void Validate() + { + ArgumentOutOfRangeException.ThrowIfLessThan(BatchSize, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(QueueCapacity, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(RetentionInterval, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxAttempts, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(BaseRetryDelay, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxRetryDelay, BaseRetryDelay); + ArgumentOutOfRangeException.ThrowIfLessThan(ShutdownTimeout, TimeSpan.Zero); + } +} diff --git a/src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md b/src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md new file mode 100644 index 000000000..a5867458f --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md @@ -0,0 +1,92 @@ +# Extension points — Diagnostics persistence lifecycle + +This is the owning catalog for the provider-neutral diagnostics persistence lifecycle helpers. The +library owns bounded draining, durable acknowledgement, retry, retention, shutdown, and loss +classification. It contains no Groundwork or EF Core types. + +## Overridable contracts + +`Elsa.Diagnostics.Persistence` is a narrow **provider-neutral helper boundary**, not a third +diagnostics domain and not a substitute for the Structured Logs or OpenTelemetry `.Core` projects. +The two domain cores continue to own their store contracts and live-feed loss signals. This helper +may reference those core models only to bridge their existing loss signals into one lifecycle +classification; neither core references this helper. + +Both interfaces below are **Adapter / Bridge** seams with **Replacement** semantics. They are not +additive contributors: exactly one implementation is meaningful for one composed drain. A caller +that needs fan-out must place it behind one implementation instead of resolving an enumerable. + +### `IDiagnosticsDrainTarget` *(Feature contract — `Elsa.Diagnostics.Persistence`)* + +- **Kind:** Adapter / Bridge; single Replacement. +- **Role:** bridges the Elsa-owned drain policy to one concrete provider adapter's idempotent commit + and retention operations. +- **Conflict behavior:** one target is required by the `DiagnosticsDrain` constructor. Host DI must + select one concrete store/provider implementation; two explicit selections are a configuration + conflict and must be rejected. +- **Dependency rule:** implementations live in concrete persistence projects. Groundwork types never + cross this contract. + +### `IDiagnosticsPersistenceObserver` *(Feature contract — `Elsa.Diagnostics.Persistence`)* + +- **Kind:** Adapter / Bridge; single Replacement. +- **Role:** receives low-cardinality pull counters for lifecycle, retry, failure, and classified loss. +- **Default:** omission selects the internal no-op observer; `DiagnosticsPersistenceCounters` is the + first-party pull-only implementation. +- **Conflict behavior:** one observer may be supplied to a drain. Multiple sinks require one explicit + aggregate bridge; contribution-style `IEnumerable` resolution is not supported. +- **Data rule:** the contract accepts no diagnostic payload, identifier, tenant, or free-form label. + +The adapter extension surface is intentionally limited to the contracts and models required for +concrete adapter assemblies to compose the shared drain. Provider packages stay in the concrete +adapter projects. A separate `.Core` project would add another public package and dependency layer +without creating an independent domain contract, so this narrow helper-boundary exception is +deliberate. + +`DiagnosticsPersistenceObserverRegistrationValidator` is the constitution-mandated +first-party implementation required by §2.23.3; it is not an adapter extension contract. Its public +constructor accepts only `IServiceCollection`, and its public validation method owns observer +conflict detection and the actionable result. An internal Options adapter delegates .NET +`ValidateOnStart` into that implementation without creating another public implementation or +extension seam. + +## Implementable contributor interfaces + +None. Diagnostics drain targets and observers are single-implementation Replacement contracts, not +fan-in surfaces. Multiple observer sinks must be composed behind one aggregate implementation. + +## Events + +None. The helper publishes no domain events. Domain-owned live feeds retain their existing in-band +loss signals. + +## Production bridges and registration + +`DiagnosticsSubscriberDeliveryLossBridge` consumes the existing `DroppedEntriesSignal` and +`OpenTelemetryDroppedItemSummary` models and classifies both as `SubscriberDelivery`. It observes +domain fan-out loss; it does not move fan-out into persistence. Structured Logs signals are +cumulative, so the live feed uses a per-subscription delta recorder; OpenTelemetry summaries are +already incremental. OpenTelemetry loss is observed when each raw item is first dropped; evicted +and requeued summaries preserve the in-band total without counting the underlying items again. + +`AddDefaultDiagnosticsStore` installs a fallback only when no store has +been selected. `ReplaceDiagnosticsStore` makes one explicit selection, +removes a tracked default, and rejects a second explicit provider at registration with a diagnostic +naming the contract and both implementations. This is the required default-vs-explicit Replacement +contract behavior; silent last-write-wins is forbidden. + +`AddDiagnosticsPersistenceObservability` installs its default observer through the same tracked +Replacement path. Explicit observer selection is order-independent; multiple direct or tracked +explicit observers are rejected with a configuration diagnostic. + +Store registrations are scoped by default in accordance with §2.5.1. A caller may explicitly select +another `ServiceLifetime` when the implementation's dependency graph and state ownership justify it. +The live-feed observer, counters, and bridge are explicitly singleton because their totals span +subscriptions and they hold neither payloads nor scoped dependencies. + +## Cross-references + +- Repo-wide index: [`../../../../EXTENSION_POINTS.md`](../../../../EXTENSION_POINTS.md). +- Structured Logs owner catalog: [`../StructuredLogs/EXTENSION_POINTS.md`](../StructuredLogs/EXTENSION_POINTS.md). +- OpenTelemetry owner catalog: [`../OpenTelemetry/EXTENSION_POINTS.md`](../OpenTelemetry/EXTENSION_POINTS.md). +- Constitutional basis: §2.6.2, §2.7, §2.22.1, and §2.23.2. diff --git a/src/Elsa/Diagnostics/Persistence/Elsa.Diagnostics.Persistence.csproj b/src/Elsa/Diagnostics/Persistence/Elsa.Diagnostics.Persistence.csproj new file mode 100644 index 000000000..6da3b94eb --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Elsa.Diagnostics.Persistence.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + 0.0.1-local + + + + + + + + + + + + + diff --git a/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceObserverRegistrationValidator.cs b/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceObserverRegistrationValidator.cs new file mode 100644 index 000000000..3636b9161 --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceObserverRegistrationValidator.cs @@ -0,0 +1,58 @@ +using Elsa.Diagnostics.Persistence.Observability; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Elsa.Diagnostics.Persistence.Extensions; + +/// +/// Validates that diagnostics persistence composition selects at most one observer implementation. +/// +public sealed class DiagnosticsPersistenceObserverRegistrationValidator(IServiceCollection services) +{ + /// Returns a failed result that names every conflicting registration, or success. + public ValidateOptionsResult Validate() + { + var observers = services + .Where(descriptor => descriptor.ServiceType == typeof(IDiagnosticsPersistenceObserver)) + .ToArray(); + if (observers.Length <= 1) + return ValidateOptionsResult.Success; + + var selection = DiagnosticsPersistenceRegistration.FindObserverSelection(services); + var implementations = observers + .Select(descriptor => DescribeObserverDescriptor(descriptor, selection)) + .ToArray(); + var message = + $"Diagnostics replacement contract '{typeof(IDiagnosticsPersistenceObserver).FullName}' has conflicting " + + $"registrations: {string.Join(", ", implementations)}. Select one observer explicitly through " + + $"'{nameof(DiagnosticsPersistenceRegistration.ReplaceDiagnosticsStore)}'."; + return ValidateOptionsResult.Fail(message); + } + + private static string DescribeObserverDescriptor( + ServiceDescriptor descriptor, + (ServiceDescriptor ContractDescriptor, Type ImplementationType)? selection) + { + if (selection is { } selected && ReferenceEquals(descriptor, selected.ContractDescriptor)) + return selected.ImplementationType.ToString(); + + if (descriptor.ImplementationType is { } implementationType) + return implementationType.ToString(); + + if (descriptor.ImplementationInstance is { } implementationInstance) + return implementationInstance.GetType().ToString(); + + return $"factory registration for '{typeof(IDiagnosticsPersistenceObserver).FullName}'"; + } +} + +internal sealed class DiagnosticsPersistenceObserverRegistrationOptions; + +internal sealed class DiagnosticsPersistenceObserverRegistrationOptionsAdapter( + DiagnosticsPersistenceObserverRegistrationValidator validator) + : IValidateOptions +{ + public ValidateOptionsResult Validate( + string? name, + DiagnosticsPersistenceObserverRegistrationOptions options) => validator.Validate(); +} diff --git a/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceRegistration.cs b/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceRegistration.cs new file mode 100644 index 000000000..eed54bd38 --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Extensions/DiagnosticsPersistenceRegistration.cs @@ -0,0 +1,149 @@ +using Elsa.Diagnostics.Persistence.Observability; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; + +namespace Elsa.Diagnostics.Persistence.Extensions; + +/// Provides one explicit replacement path for a diagnostics store contract. +public static class DiagnosticsPersistenceRegistration +{ + /// + /// Registers the fallback implementation only when no store has been selected. An explicit replacement + /// registered before or after this call always owns the contract. Stores are scoped by default; select + /// another lifetime explicitly only when the implementation's state and dependency graph justify it. + /// + public static IServiceCollection AddDefaultDiagnosticsStore( + this IServiceCollection services, + ServiceLifetime lifetime = ServiceLifetime.Scoped) + where TContract : class + where TImplementation : class, TContract + { + ArgumentNullException.ThrowIfNull(services); + ValidateLifetime(lifetime); + if (FindSelection(services) is not null || services.Any(x => x.ServiceType == typeof(TContract))) + return services; + + services.TryAdd(ServiceDescriptor.Describe(typeof(TImplementation), typeof(TImplementation), lifetime)); + var contractDescriptor = ServiceDescriptor.Describe( + typeof(TContract), + provider => provider.GetRequiredService(), + lifetime); + services.TryAdd(contractDescriptor); + services.AddSingleton(new DiagnosticsStoreSelection( + typeof(TContract), + typeof(TImplementation), + contractDescriptor, + IsExplicit: false)); + return services; + } + + /// + /// Selects the one explicit implementation for a replacement contract. A second explicit selection is a + /// configuration conflict and is rejected immediately instead of silently becoming last-write-wins. + /// Stores are scoped by default; any other lifetime is an explicit host decision. + /// + public static IServiceCollection ReplaceDiagnosticsStore( + this IServiceCollection services, + ServiceLifetime lifetime = ServiceLifetime.Scoped) + where TContract : class + where TImplementation : class, TContract + { + ArgumentNullException.ThrowIfNull(services); + ValidateLifetime(lifetime); + var existing = FindSelection(services); + if (existing is { IsExplicit: true }) + { + throw new InvalidOperationException( + $"Diagnostics replacement contract '{typeof(TContract).FullName}' already selects explicit implementation " + + $"'{existing.ImplementationType.FullName}' and cannot also select '{typeof(TImplementation).FullName}'."); + } + + services.RemoveAll(); + if (existing is not null) + { + services.RemoveAll(existing.ImplementationType); + services.Remove(existing.Descriptor); + } + services.RemoveAll(); + services.Add(ServiceDescriptor.Describe(typeof(TImplementation), typeof(TImplementation), lifetime)); + var contractDescriptor = ServiceDescriptor.Describe( + typeof(TContract), + provider => provider.GetRequiredService(), + lifetime); + services.Add(contractDescriptor); + services.AddSingleton(new DiagnosticsStoreSelection( + typeof(TContract), + typeof(TImplementation), + contractDescriptor, + IsExplicit: true)); + return services; + } + + /// + /// Registers the singleton pull-only observer used by singleton live feeds. This lifetime is explicit: + /// subscriber-loss totals must span all subscriptions and contain no scoped dependencies or payloads. + /// + public static IServiceCollection AddDiagnosticsPersistenceObservability(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + RejectObserverConflicts(services); + services.AddDefaultDiagnosticsStore( + ServiceLifetime.Singleton); + services.TryAddSingleton(); + AddObserverRegistrationValidation(services); + return services; + } + + private static DiagnosticsStoreSelection? FindSelection(IServiceCollection services) => + services + .Where(descriptor => descriptor.ServiceType == typeof(DiagnosticsStoreSelection)) + .Select(descriptor => (Descriptor: descriptor, Selection: descriptor.ImplementationInstance as DiagnosticsStoreSelection)) + .Where(item => item.Selection?.ContractType == typeof(TContract)) + .Select(item => item.Selection! with { Descriptor = item.Descriptor }) + .SingleOrDefault(); + + internal static (ServiceDescriptor ContractDescriptor, Type ImplementationType)? FindObserverSelection( + IServiceCollection services) + { + var selection = FindSelection(services); + return selection is null + ? null + : (selection.ContractDescriptor, selection.ImplementationType); + } + + private static void ValidateLifetime(ServiceLifetime lifetime) + { + if (!Enum.IsDefined(lifetime)) + throw new ArgumentOutOfRangeException(nameof(lifetime), lifetime, "Unsupported diagnostics store lifetime."); + } + + private static void RejectObserverConflicts(IServiceCollection services) + { + var result = new DiagnosticsPersistenceObserverRegistrationValidator(services).Validate(); + if (result.Failed) + throw new InvalidOperationException(result.Failures.Single()); + } + + private static void AddObserverRegistrationValidation(IServiceCollection services) + { + if (services.Any(descriptor => + descriptor.ServiceType == typeof(DiagnosticsPersistenceObserverRegistrationValidator))) + return; + + services.AddSingleton(new DiagnosticsPersistenceObserverRegistrationValidator(services)); + services.AddSingleton< + IValidateOptions, + DiagnosticsPersistenceObserverRegistrationOptionsAdapter>(); + services.AddOptions().ValidateOnStart(); + } + + private sealed record DiagnosticsStoreSelection( + Type ContractType, + Type ImplementationType, + ServiceDescriptor ContractDescriptor, + bool IsExplicit) + { + public ServiceDescriptor Descriptor { get; init; } = null!; + } +} diff --git a/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsPersistenceObservability.cs b/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsPersistenceObservability.cs new file mode 100644 index 000000000..e764a32da --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsPersistenceObservability.cs @@ -0,0 +1,132 @@ +namespace Elsa.Diagnostics.Persistence.Observability; + +public enum DiagnosticsDrainState +{ + Created, + Running, + Closing, + Draining, + Stopped, + TimedOut +} + +public enum DiagnosticsPersistenceOperation +{ + Commit, + Retention +} + +public enum DiagnosticsPersistenceLossReason +{ + QueueOverflow, + RetryExhausted, + ShutdownTimeout, + WriteAfterClosure, + DurableRetentionDeletion, + SubscriberDelivery +} + +/// +/// Replacement contract: exactly one low-cardinality pull observer is composed per drain. It deliberately +/// accepts no diagnostic payloads, identifiers, tenant values, or free-form labels, preventing the drain +/// from recursively emitting captured signals. +/// +public interface IDiagnosticsPersistenceObserver +{ + void RecordState(DiagnosticsDrainState state); + void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts); + void RecordOperationFailure(DiagnosticsPersistenceOperation operation); + void RecordLoss(DiagnosticsPersistenceLossReason reason, long count); +} + +public sealed record DiagnosticsPersistenceSnapshot( + DiagnosticsDrainState State, + long CommitRetries, + long RetentionRetries, + long CommitFailures, + long RetentionFailures, + IReadOnlyDictionary Losses); + +/// Thread-safe, pull-only lifecycle and loss counters shared by diagnostics adapters. +public sealed class DiagnosticsPersistenceCounters : IDiagnosticsPersistenceObserver +{ + private readonly long[] _losses = new long[Enum.GetValues().Length]; + private long _commitRetries; + private long _retentionRetries; + private long _commitFailures; + private long _retentionFailures; + private int _state = (int)DiagnosticsDrainState.Created; + + public void RecordState(DiagnosticsDrainState state) + { + ValidateDefined(state); + Volatile.Write(ref _state, (int)state); + } + + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) + { + ValidateDefined(operation); + ArgumentOutOfRangeException.ThrowIfLessThan(maxAttempts, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(attempt, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(attempt, maxAttempts); + switch (operation) + { + case DiagnosticsPersistenceOperation.Commit: + Interlocked.Increment(ref _commitRetries); + break; + case DiagnosticsPersistenceOperation.Retention: + Interlocked.Increment(ref _retentionRetries); + break; + } + } + + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) + { + ValidateDefined(operation); + switch (operation) + { + case DiagnosticsPersistenceOperation.Commit: + Interlocked.Increment(ref _commitFailures); + break; + case DiagnosticsPersistenceOperation.Retention: + Interlocked.Increment(ref _retentionFailures); + break; + } + } + + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) + { + ValidateDefined(reason); + ArgumentOutOfRangeException.ThrowIfLessThan(count, 1); + Interlocked.Add(ref _losses[(int)reason], count); + } + + public DiagnosticsPersistenceSnapshot Snapshot() + { + var losses = Enum.GetValues() + .ToDictionary(reason => reason, reason => Interlocked.Read(ref _losses[(int)reason])); + return new( + (DiagnosticsDrainState)Volatile.Read(ref _state), + Interlocked.Read(ref _commitRetries), + Interlocked.Read(ref _retentionRetries), + Interlocked.Read(ref _commitFailures), + Interlocked.Read(ref _retentionFailures), + losses); + } + + private static void ValidateDefined(TEnum value, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(value))] string? parameterName = null) + where TEnum : struct, Enum + { + if (!Enum.IsDefined(value)) + throw new ArgumentOutOfRangeException(parameterName, value, $"Unsupported {typeof(TEnum).Name} value."); + } +} + +internal sealed class NullDiagnosticsPersistenceObserver : IDiagnosticsPersistenceObserver +{ + public static NullDiagnosticsPersistenceObserver Instance { get; } = new(); + public void RecordState(DiagnosticsDrainState state) { } + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) { } + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) { } + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) { } +} diff --git a/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsSubscriberDeliveryLossBridge.cs b/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsSubscriberDeliveryLossBridge.cs new file mode 100644 index 000000000..aba87b5f5 --- /dev/null +++ b/src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsSubscriberDeliveryLossBridge.cs @@ -0,0 +1,73 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.StructuredLogs.Core.Models; + +namespace Elsa.Diagnostics.Persistence.Observability; + +/// +/// Maps the two domain-owned live-feed loss signals into the shared low-cardinality subscriber-delivery +/// classification. It observes signals that already exist; it does not move fan-out into persistence. +/// +public sealed class DiagnosticsSubscriberDeliveryLossBridge +{ + private readonly IDiagnosticsPersistenceObserver _observer; + + public DiagnosticsSubscriberDeliveryLossBridge(IDiagnosticsPersistenceObserver observer) + { + ArgumentNullException.ThrowIfNull(observer); + _observer = observer; + } + + public void RecordOpenTelemetry(OpenTelemetryDroppedItemSummary signal) + { + ArgumentNullException.ThrowIfNull(signal); + ArgumentOutOfRangeException.ThrowIfLessThan(signal.Count, 1); + RecordLoss(signal.Count); + } + + /// + /// Creates one cumulative-signal recorder for one Structured Logs subscription. Per-subscription state + /// prevents repeated cumulative signals from being counted twice and is released with the subscriber. + /// + public Action CreateStructuredLogRecorder() + { + var counter = new CumulativeDropCounter(); + return signal => + { + ArgumentNullException.ThrowIfNull(signal); + ArgumentOutOfRangeException.ThrowIfLessThan(signal.DroppedCount, 1); + RecordLoss(counter.Advance(signal.DroppedCount)); + }; + } + + private void RecordLoss(long count) + { + if (count == 0) + return; + + try + { + _observer.RecordLoss(DiagnosticsPersistenceLossReason.SubscriberDelivery, count); + } + catch + { + // Observability must never break live-feed delivery. + } + } + + private sealed class CumulativeDropCounter + { + private long _count; + + public long Advance(long count) + { + while (true) + { + var previous = Volatile.Read(ref _count); + if (count <= previous) + return 0; + if (Interlocked.CompareExchange(ref _count, count, previous) == previous) + return count - previous; + } + } + } +} diff --git a/src/Elsa/Diagnostics/StructuredLogs/Elsa.Diagnostics.StructuredLogs.csproj b/src/Elsa/Diagnostics/StructuredLogs/Elsa.Diagnostics.StructuredLogs.csproj index b478974f7..ba5badacf 100644 --- a/src/Elsa/Diagnostics/StructuredLogs/Elsa.Diagnostics.StructuredLogs.csproj +++ b/src/Elsa/Diagnostics/StructuredLogs/Elsa.Diagnostics.StructuredLogs.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs b/src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs index 8cec27401..79b2aa2b5 100644 --- a/src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs +++ b/src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs @@ -3,6 +3,7 @@ using Elsa.Diagnostics.StructuredLogs.Core.Contracts; using Elsa.Diagnostics.StructuredLogs.Core.Models; using Elsa.Diagnostics.StructuredLogs.Core.Options; +using Elsa.Diagnostics.Persistence.Observability; using Microsoft.Extensions.Options; namespace Elsa.Diagnostics.StructuredLogs.Live; @@ -16,11 +17,17 @@ namespace Elsa.Diagnostics.StructuredLogs.Live; public sealed class InMemoryStructuredLogLiveFeed : IStructuredLogLiveFeed, IStructuredLogLivePublisher { private readonly int _subscriberQueueCapacity; + private readonly DiagnosticsSubscriberDeliveryLossBridge? _deliveryLossBridge; private readonly object _gate = new(); private readonly List _subscribers = []; - public InMemoryStructuredLogLiveFeed(IOptions options) => + public InMemoryStructuredLogLiveFeed( + IOptions options, + DiagnosticsSubscriberDeliveryLossBridge? deliveryLossBridge = null) + { _subscriberQueueCapacity = Math.Max(1, options.Value.SubscriberQueueCapacity); + _deliveryLossBridge = deliveryLossBridge; + } /// public void Publish(StructuredLogEntry entry) @@ -43,7 +50,10 @@ public IAsyncEnumerable Subscribe( ArgumentNullException.ThrowIfNull(filter); // Register eagerly (not inside the iterator) so entries published before the first MoveNext are delivered. - var subscriber = new Subscriber(filter, _subscriberQueueCapacity); + var subscriber = new Subscriber( + filter, + _subscriberQueueCapacity, + _deliveryLossBridge?.CreateStructuredLogRecorder()); lock (_gate) _subscribers.Add(subscriber); @@ -84,13 +94,18 @@ private sealed class Subscriber { private readonly StructuredLogFilter _filter; private readonly Channel _channel; + private readonly Action? _recordDeliveryLoss; private long _droppedCount; private long _signalledDrops; private long _firstDropTicks; - public Subscriber(StructuredLogFilter filter, int capacity) + public Subscriber( + StructuredLogFilter filter, + int capacity, + Action? recordDeliveryLoss) { _filter = filter; + _recordDeliveryLoss = recordDeliveryLoss; _channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) { FullMode = BoundedChannelFullMode.Wait, @@ -139,6 +154,7 @@ private bool TryTakeDropSignal(out DroppedEntriesSignal signal) _signalledDrops = dropped; var since = new DateTimeOffset(Interlocked.Read(ref _firstDropTicks), TimeSpan.Zero); signal = new DroppedEntriesSignal(dropped, since); + _recordDeliveryLoss?.Invoke(signal); return true; } diff --git a/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkReplayCursorCodec.cs b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkReplayCursorCodec.cs index 9d0332d52..36989a35c 100644 --- a/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkReplayCursorCodec.cs +++ b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkReplayCursorCodec.cs @@ -8,6 +8,7 @@ internal static class GroundworkReplayCursorCodec { private const string Version = "slrc1"; private const string Source = "groundwork"; + private static readonly UTF8Encoding StrictUtf8 = new(false, true); public static StructuredLogReplayCursor Encode( StructuredLogStoreBinding binding, @@ -35,6 +36,7 @@ public static bool TryDecode( StructuredLogStoreBinding expectedBinding, out GroundworkReplayCursorParts parts) { + ArgumentNullException.ThrowIfNull(expectedBinding); parts = default; if (!cursor.IsValid) return false; @@ -47,9 +49,9 @@ public static bool TryDecode( try { - var entrySourceId = Encoding.UTF8.GetString(FromBase64Url(entrySource)); - var recordToken = Encoding.UTF8.GetString(FromBase64Url(record)); - var providerPosition = Encoding.UTF8.GetString(FromBase64Url(provider)); + var entrySourceId = DecodeCanonical(entrySource); + var recordToken = DecodeCanonical(record); + var providerPosition = DecodeCanonical(provider); if (string.IsNullOrWhiteSpace(entrySourceId) || string.IsNullOrWhiteSpace(recordToken) || string.IsNullOrWhiteSpace(providerPosition)) @@ -62,6 +64,10 @@ public static bool TryDecode( { return false; } + catch (DecoderFallbackException) + { + return false; + } } private static string BindingHash(StructuredLogStoreBinding binding) @@ -73,6 +79,14 @@ private static string BindingHash(StructuredLogStoreBinding binding) private static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + private static string DecodeCanonical(string value) + { + var bytes = FromBase64Url(value); + if (!StringComparer.Ordinal.Equals(value, Base64Url(bytes))) + throw new FormatException("The replay cursor contains a noncanonical Base64URL segment."); + return StrictUtf8.GetString(bytes); + } + private static byte[] FromBase64Url(string value) { var base64 = value.Replace('-', '+').Replace('_', '/'); diff --git a/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs index 5407a5e2b..a0d45e938 100644 --- a/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs +++ b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/GroundworkStructuredLogStore.cs @@ -173,49 +173,34 @@ public async Task ReadAfterAsync( ArgumentOutOfRangeException.ThrowIfLessThan(maxCount, 1); var limit = Math.Min(maxCount, _maxRecentQuerySize); - try + DiagnosticRecord? anchor = null; + if (afterCursor is { } cursor) + anchor = await ValidateAnchorAsync(cursor, cancellationToken); + + var query = new DiagnosticRecordQuery( + _scope, + _stream, + limit, + DiagnosticRecordOrder.CursorAscending); + if (anchor is not null) { - DiagnosticRecord? anchor = null; - if (afterCursor is { } cursor) - anchor = await ValidateAnchorAsync(cursor, cancellationToken); - - var query = new DiagnosticRecordQuery( - _scope, - _stream, - limit, - DiagnosticRecordOrder.CursorAscending); - if (anchor is not null) + var statistics = await _store.InspectAsync(new(_scope, _stream), cancellationToken); + if (statistics.LifetimeCommittedCursorHighWater is not { } snapshotHighWater) + throw new StructuredLogReplayCursorUnavailableException(); + query = query with { - var statistics = await _store.InspectAsync(new(_scope, _stream), cancellationToken); - if (statistics.LifetimeCommittedCursorHighWater is not { } snapshotHighWater) - throw new StructuredLogReplayCursorUnavailableException(); - query = query with - { - Continuation = new( - snapshotHighWater, - anchor.Cursor, - DiagnosticRequestFingerprint.ForQuery(query, _definition)) - }; - } - - var page = await _store.QueryAsync(query, cancellationToken); - var scanned = page.Records; - var entries = scanned.Select(ToEntry).Where(filter.Matches).ToArray(); - var next = scanned.Count == 0 ? afterCursor : ToEntry(scanned[^1]).ReplayCursor; - return new(entries, next, page.Continuation is not null); - } - catch (OperationCanceledException) - { - throw; - } - catch (StructuredLogReplayCursorUnavailableException) - { - throw; - } - catch - { - throw new StructuredLogReplayCursorUnavailableException(); + Continuation = new( + snapshotHighWater, + anchor.Cursor, + DiagnosticRequestFingerprint.ForQuery(query, _definition)) + }; } + + var page = await _store.QueryAsync(query, cancellationToken); + var scanned = page.Records; + var entries = scanned.Select(ToEntry).Where(filter.Matches).ToArray(); + var next = scanned.Count == 0 ? afterCursor : ToEntry(scanned[^1]).ReplayCursor; + return new(entries, next, page.Continuation is not null); } public async Task TrimAsync(int keepNewest, CancellationToken cancellationToken = default) diff --git a/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/StructuredLogsGroundworkStorageSchema.cs b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/StructuredLogsGroundworkStorageSchema.cs new file mode 100644 index 000000000..f080daa2f --- /dev/null +++ b/src/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/StructuredLogsGroundworkStorageSchema.cs @@ -0,0 +1,34 @@ +using Elsa.Diagnostics.StructuredLogs.Core.Models; +using Groundwork.DiagnosticRecords; + +namespace Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork; + +/// Groundwork stream, projected-index, and durable ledger requirements for Structured Logs. +public static class StructuredLogsGroundworkStorageSchema +{ + public static IReadOnlySet RequiredIndexFields { get; } = + new HashSet(StringComparer.Ordinal) + { + "sequence", + "level", + "categoryKey", + "sourceKey", + "replayToken" + }; + + public static IReadOnlySet RequiredOperationLedgers { get; } = + new HashSet(StringComparer.Ordinal) + { + "diagnostic-append-idempotency", + "diagnostic-trim-idempotency" + }; + + public static DiagnosticRecordStreamDefinition CreateStream(StructuredLogStoreBinding binding) + { + ArgumentNullException.ThrowIfNull(binding); + ArgumentException.ThrowIfNullOrWhiteSpace(binding.TenantId); + ArgumentException.ThrowIfNullOrWhiteSpace(binding.ScopeId); + ArgumentException.ThrowIfNullOrWhiteSpace(binding.StreamId); + return GroundworkStructuredLogStore.CreateStreamDefinition(binding.StreamId); + } +} diff --git a/src/Elsa/Diagnostics/StructuredLogs/StructuredLogsFeature.cs b/src/Elsa/Diagnostics/StructuredLogs/StructuredLogsFeature.cs index 55ff3e9e9..9047113cc 100644 --- a/src/Elsa/Diagnostics/StructuredLogs/StructuredLogsFeature.cs +++ b/src/Elsa/Diagnostics/StructuredLogs/StructuredLogsFeature.cs @@ -7,6 +7,7 @@ using Elsa.Diagnostics.StructuredLogs.Live; using Elsa.Diagnostics.StructuredLogs.Sources; using Elsa.Diagnostics.StructuredLogs.Storage; +using Elsa.Diagnostics.Persistence.Extensions; using Elsa.Platform.PackageManifest.Generator.Hints; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -43,6 +44,7 @@ public class StructuredLogsFeature : FastEndpointsFeatureBase public override void ConfigureServices(IServiceCollection services) { base.ConfigureServices(services); + services.AddDiagnosticsPersistenceObservability(); var minimumLevel = Enum.TryParse(MinimumLevel, ignoreCase: true, out var parsed) ? parsed diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CanonicalRecordSerializerTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CanonicalRecordSerializerTests.cs new file mode 100644 index 000000000..9291f6e51 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CanonicalRecordSerializerTests.cs @@ -0,0 +1,264 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; +using Groundwork.DiagnosticRecords; +using System.Text.Json.Nodes; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class CanonicalRecordSerializerTests +{ + private static readonly DateTimeOffset Timestamp = + new(2026, 7, 14, 10, 11, 12, 345, TimeSpan.FromHours(2)); + + private readonly CanonicalRecordSerializer _serializer = new(); + + [Fact] + public void Every_immutable_signal_round_trips_without_losing_domain_fields() + { + var trace = Trace(); + var span = Span(Attributes(("z", "last"), ("a", "first"))); + var point = Point(); + var log = Log(); + + var actualTrace = _serializer.ToTrace(Committed(_serializer.ToRecord("trace-record-1", trace))); + var actualSpan = _serializer.ToSpan(Committed(_serializer.ToRecord("span-record-1", span))); + var actualPoint = _serializer.ToMetricPoint(Committed(_serializer.ToRecord("point-record-1", point))); + var actualLog = _serializer.ToLog(Committed(_serializer.ToRecord("log-record-1", log))); + + Assert.Equal(trace with + { + ResourceIds = actualTrace.ResourceIds, + WorkflowInstanceIds = actualTrace.WorkflowInstanceIds + }, actualTrace); + Assert.Equal(span with + { + Attributes = actualSpan.Attributes, + Events = actualSpan.Events, + Links = actualSpan.Links + }, actualSpan); + AssertAttributes(span.Attributes, actualSpan.Attributes); + AssertAttributes(Assert.Single(span.Events).Attributes, Assert.Single(actualSpan.Events).Attributes); + AssertAttributes(Assert.Single(span.Links).Attributes, Assert.Single(actualSpan.Links).Attributes); + Assert.Equal(BitConverter.DoubleToInt64Bits(point.Value!.Value), BitConverter.DoubleToInt64Bits(actualPoint.Value!.Value)); + Assert.Equal(point with { Attributes = actualPoint.Attributes }, actualPoint); + AssertAttributes(point.Attributes, actualPoint.Attributes); + Assert.Equal(log with { Attributes = actualLog.Attributes }, actualLog); + AssertAttributes(log.Attributes, actualLog.Attributes); + } + + [Fact] + public void Equivalent_attribute_insertion_orders_produce_one_payload_and_append_fingerprint() + { + var first = _serializer.ToRecord("span-record-1", Span(Attributes(("b", "2"), ("a", "1")))); + var second = _serializer.ToRecord("span-record-1", Span(Attributes(("a", "1"), ("b", "2")))); + var scope = new DiagnosticStorageScope("tenant", "scope"); + var stream = new DiagnosticStreamId("spans"); + + Assert.Equal(first.Payload, second.Payload); + Assert.Equal(first.Fields, second.Fields); + Assert.Equal( + DiagnosticRequestFingerprint.ForAppend(scope, stream, [first]), + DiagnosticRequestFingerprint.ForAppend(scope, stream, [second])); + } + + [Fact] + public void Trace_set_fields_are_canonical_when_input_enumeration_order_changes() + { + var first = Trace() with + { + ResourceIds = ["resource-2", "resource-1"], + WorkflowInstanceIds = ["workflow-2", "workflow-1"] + }; + var second = Trace() with + { + ResourceIds = ["resource-1", "resource-2"], + WorkflowInstanceIds = ["workflow-1", "workflow-2"] + }; + + var firstRecord = _serializer.ToRecord("trace-record-1", first); + var secondRecord = _serializer.ToRecord("trace-record-1", second); + + Assert.Equal(firstRecord.Payload, secondRecord.Payload); + Assert.Equal(firstRecord.Fields, secondRecord.Fields); + } + + [Fact] + public void Span_event_and_link_order_is_preserved_as_meaningful_payload_order() + { + var span = Span(Attributes()) with + { + Events = + [ + new("second-by-name", Timestamp.AddMilliseconds(2), Attributes()), + new("first-by-name", Timestamp.AddMilliseconds(1), Attributes()) + ], + Links = + [ + new("trace-z", "span-z", Attributes()), + new("trace-a", "span-a", Attributes()) + ] + }; + + var restored = _serializer.ToSpan(Committed(_serializer.ToRecord("span-record", span))); + + Assert.Equal(["second-by-name", "first-by-name"], restored.Events.Select(x => x.Name)); + Assert.Equal(["trace-z", "trace-a"], restored.Links.Select(x => x.TraceId)); + } + + [Fact] + public void Record_fields_cover_bounded_query_identity_order_and_filter_values() + { + var trace = _serializer.ToRecord("trace-record-1", Trace()); + var span = _serializer.ToRecord("span-record-1", Span(Attributes())); + var point = _serializer.ToRecord("point-record-1", Point()); + var log = _serializer.ToRecord("log-record-1", Log()); + + Assert.Equal(7, trace.Fields!.Count); + Assert.Equal(8, span.Fields!.Count); + Assert.Equal(6, point.Fields!.Count); + Assert.Equal(7, log.Fields!.Count); + Assert.Equal("trace-1", Assert.Single(trace.Fields[RecordFields.TraceId]).CanonicalValue); + Assert.Equal(Timestamp.ToUniversalTime().ToString("O"), Assert.Single(point.Fields[RecordFields.Timestamp]).CanonicalValue); + Assert.Equal("Information", Assert.Single(log.Fields[RecordFields.SeverityText]).CanonicalValue); + } + + [Fact] + public void Optional_query_fields_are_absent_instead_of_becoming_empty_filter_values() + { + var point = Point() with { TraceId = null, SpanId = null }; + var log = Log() with { SeverityNumber = null, TraceId = null, SpanId = null }; + + var pointFields = _serializer.ToRecord("point-record-1", point).Fields!; + var logFields = _serializer.ToRecord("log-record-1", log).Fields!; + + Assert.DoesNotContain(RecordFields.TraceId, pointFields.Keys); + Assert.DoesNotContain(RecordFields.SpanId, pointFields.Keys); + Assert.DoesNotContain(RecordFields.SeverityNumber, logFields.Keys); + Assert.DoesNotContain(RecordFields.TraceId, logFields.Keys); + Assert.DoesNotContain(RecordFields.SpanId, logFields.Keys); + } + + [Fact] + public void Wrong_kind_and_malformed_payloads_are_visible_as_record_payload_failures() + { + var logRecord = Committed(_serializer.ToRecord("log-record-1", Log())); + var malformed = logRecord with { Payload = "{" }; + + Assert.Throws(() => _serializer.ToTrace(logRecord)); + var exception = Assert.Throws(() => _serializer.ToLog(malformed)); + Assert.NotNull(exception.InnerException); + + var missingRequiredFields = logRecord with + { + Payload = "{\"schemaVersion\":1,\"kind\":\"log\",\"value\":{\"id\":\"log-1\"}}" + }; + Assert.Throws(() => _serializer.ToLog(missingRequiredFields)); + } + + [Fact] + public void Missing_required_domain_values_and_null_attributes_fail_before_append() + { + Assert.Throws(() => + _serializer.ToRecord("trace-record", Trace() with { TraceId = " " })); + Assert.Throws(() => + _serializer.ToRecord("trace-record", Trace() with { ResourceIds = [] })); + Assert.Throws(() => + _serializer.ToRecord("point-record", Point() with { Attributes = null! })); + } + + [Fact] + public void Null_and_duplicate_persisted_attribute_keys_are_visible_payload_failures() + { + var input = _serializer.ToRecord("point-record", Point()); + var duplicate = WithMutatedAttributes(input, attributes => attributes.Add(attributes[0]!.DeepClone())); + var nullKey = WithMutatedAttributes(input, attributes => attributes[0]!["key"] = null); + + Assert.Throws(() => _serializer.ToMetricPoint(duplicate)); + Assert.Throws(() => _serializer.ToMetricPoint(nullKey)); + } + + private static DiagnosticRecord Committed(DiagnosticRecordInput input) => new( + input.RecordId, + input.OccurredAt, + input.Payload, + new DiagnosticCursor("1"), + input.Fields); + + private static DiagnosticRecord WithMutatedAttributes( + DiagnosticRecordInput input, + Action mutate) + { + var payload = JsonNode.Parse(input.Payload)!; + var attributes = payload["value"]!["attributes"]!.AsArray(); + mutate(attributes); + return Committed(input) with { Payload = payload.ToJsonString() }; + } + + private static TelemetryTrace Trace() => new( + "trace-1", + "root-span", + "orders", + Timestamp, + Timestamp.AddMilliseconds(15), + TimeSpan.FromMilliseconds(15), + SpanStatus.Ok, + ["resource-1", "resource-2"], + ["workflow-1"], + 2); + + private static TelemetrySpan Span(IDictionary attributes) => new( + "span-domain-id", + "trace-1", + "span-1", + "root-span", + "resource-1", + "process-order", + "internal", + Timestamp, + Timestamp.AddMilliseconds(10), + SpanStatus.Ok, + "complete", + attributes, + [new("accepted", Timestamp.AddMilliseconds(1), Attributes(("event-z", null), ("event-a", "1")))], + [new("trace-linked", "span-linked", Attributes(("link-z", "2"), ("link-a", "1")))]); + + private static MetricPoint Point() => new( + "point-1", + "instrument-1", + "request.duration", + "resource-1", + Timestamp, + -0d, + 42.5, + 3, + Attributes(("point-z", null), ("point-a", "1")), + "trace-1", + "span-1"); + + private static OtlpLogRecord Log() => new( + "log-1", + "resource-1", + Timestamp, + "Information", + 9, + "order accepted", + "trace-1", + "span-1", + Attributes(("log-z", null), ("log-a", "1"))); + + private static Dictionary Attributes(params (string Key, string? Value)[] values) + { + var result = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in values) + result.Add(key, value); + return result; + } + + private static void AssertAttributes( + IDictionary expected, + IDictionary actual) => + Assert.Equal( + expected.OrderBy(x => x.Key, StringComparer.Ordinal), + actual.OrderBy(x => x.Key, StringComparer.Ordinal)); +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CatalogDocumentSerializerTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CatalogDocumentSerializerTests.cs new file mode 100644 index 000000000..6bdaa9726 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/CatalogDocumentSerializerTests.cs @@ -0,0 +1,169 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; +using Groundwork.Documents.Store; +using System.Text.Json.Nodes; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class CatalogDocumentSerializerTests +{ + private static readonly DateTimeOffset Timestamp = + new(2026, 7, 14, 13, 14, 15, TimeSpan.FromHours(3)); + + private readonly CatalogDocumentSerializer _serializer = new(); + + [Fact] + public void Resource_document_is_canonical_keyed_and_restart_stable() + { + var first = Resource(Attributes(("z", "last"), ("a", "first"))); + var equivalent = Resource(Attributes(("a", "first"), ("z", "last"))); + + var request = _serializer.ToSaveRequest(first, expectedRevision: 4); + var equivalentRequest = _serializer.ToSaveRequest(equivalent, expectedRevision: 4); + var restored = _serializer.ToResource(Envelope(request, revision: 5)); + var afterRestart = _serializer.ToSaveRequest(restored.Resource, expectedRevision: restored.Revision); + + Assert.Equal(CatalogDocuments.ResourceKind, request.DocumentKind); + Assert.Equal(first.Id, request.Id); + Assert.Equal(CatalogDocuments.SchemaVersion, request.SchemaVersion); + Assert.Equal(request.ContentJson, equivalentRequest.ContentJson); + Assert.Equal(request.ContentJson, afterRestart.ContentJson); + Assert.Equal(5, restored.Revision); + Assert.Equal(first with { Attributes = restored.Resource.Attributes }, restored.Resource); + AssertAttributes(first.Attributes, restored.Resource.Attributes); + } + + [Fact] + public void Instrument_document_preserves_explicit_retry_time_tie_breaker_and_revision() + { + var instrument = Instrument(Attributes(("z", null), ("a", "first"))); + var request = _serializer.ToSaveRequest(instrument, Timestamp, expectedRevision: 2); + var restored = _serializer.ToInstrument(Envelope(request, revision: 3)); + var retried = _serializer.ToSaveRequest(restored.Instrument, restored.LastSeen, expectedRevision: 2); + + Assert.Equal(CatalogDocuments.InstrumentKind, request.DocumentKind); + Assert.Equal(request.ContentJson, retried.ContentJson); + Assert.Equal(Timestamp, restored.LastSeen); + Assert.Equal(instrument.Id, restored.RetentionTieBreaker); + Assert.Equal(3, restored.Revision); + Assert.Equal(instrument with { Attributes = restored.Instrument.Attributes }, restored.Instrument); + AssertAttributes(instrument.Attributes, restored.Instrument.Attributes); + } + + [Fact] + public void Catalog_payload_exposes_top_level_fields_required_for_bounded_retention_and_queries() + { + var resource = _serializer.ToSaveRequest(Resource(Attributes())); + var instrument = _serializer.ToSaveRequest(Instrument(Attributes()), Timestamp); + + using var resourceJson = System.Text.Json.JsonDocument.Parse(resource.ContentJson); + using var instrumentJson = System.Text.Json.JsonDocument.Parse(instrument.ContentJson); + + Assert.Equal("api", resourceJson.RootElement.GetProperty("serviceName").GetString()); + Assert.Equal(Resource(Attributes()).LastSeen.UtcTicks, resourceJson.RootElement.GetProperty("lastSeenUtcTicks").GetInt64()); + Assert.Equal("request.duration", instrumentJson.RootElement.GetProperty("name").GetString()); + Assert.Equal("instrument-1", instrumentJson.RootElement.GetProperty("retentionTieBreaker").GetString()); + } + + [Fact] + public void Wrong_kind_schema_identity_and_malformed_json_fail_without_fallback() + { + var request = _serializer.ToSaveRequest(Resource(Attributes())); + var envelope = Envelope(request, revision: 1); + + Assert.Throws(() => _serializer.ToResource(envelope with { DocumentKind = "foreign" })); + Assert.Throws(() => _serializer.ToResource(envelope with { SchemaVersion = "2" })); + Assert.Throws(() => _serializer.ToResource(envelope with { Id = "foreign" })); + Assert.Throws(() => _serializer.ToResource(envelope with { Version = -1 })); + var exception = Assert.Throws(() => + _serializer.ToResource(envelope with { ContentJson = "{" })); + Assert.NotNull(exception.InnerException); + + Assert.Throws(() => _serializer.ToResource(envelope with + { + ContentJson = "{\"id\":\"resource-1\",\"serviceName\":\"api\"}" + })); + } + + [Fact] + public void Negative_expected_revision_is_rejected_before_a_document_request_is_created() + { + Assert.Throws(() => + _serializer.ToSaveRequest(Resource(Attributes()), expectedRevision: -1)); + Assert.Throws(() => + _serializer.ToSaveRequest(Instrument(Attributes()), Timestamp, expectedRevision: -1)); + } + + [Fact] + public void Missing_required_domain_values_and_null_attributes_fail_before_save() + { + Assert.Throws(() => + _serializer.ToSaveRequest(Resource(Attributes()) with { ServiceName = " " })); + Assert.Throws(() => + _serializer.ToSaveRequest(Instrument(Attributes()) with { Attributes = null! }, Timestamp)); + } + + [Fact] + public void Null_and_duplicate_persisted_attribute_keys_are_visible_payload_failures() + { + var request = _serializer.ToSaveRequest(Resource(Attributes(("key", "value")))); + var duplicate = WithMutatedAttributes(request, attributes => attributes.Add(attributes[0]!.DeepClone())); + var nullKey = WithMutatedAttributes(request, attributes => attributes[0]!["key"] = null); + + Assert.Throws(() => _serializer.ToResource(duplicate)); + Assert.Throws(() => _serializer.ToResource(nullKey)); + } + + private static DocumentEnvelope Envelope(SaveDocumentRequest request, long revision) => new( + request.DocumentKind, + request.Id, + request.SchemaVersion, + revision, + request.ContentJson, + Timestamp, + Timestamp); + + private static DocumentEnvelope WithMutatedAttributes( + SaveDocumentRequest request, + Action mutate) + { + var content = JsonNode.Parse(request.ContentJson)!; + var attributes = content["attributes"]!.AsArray(); + mutate(attributes); + return Envelope(request, revision: 1) with { ContentJson = content.ToJsonString() }; + } + + private static TelemetryResource Resource(IDictionary attributes) => new( + "resource-1", + "api", + "api-1", + "dotnet", + attributes, + Timestamp, + TelemetryResourceStatus.Active); + + private static MetricInstrument Instrument(IDictionary attributes) => new( + "instrument-1", + "resource-1", + "request.duration", + "ms", + "request duration", + MetricKind.Gauge, + attributes); + + private static Dictionary Attributes(params (string Key, string? Value)[] values) + { + var result = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in values) + result.Add(key, value); + return result; + } + + private static void AssertAttributes( + IDictionary expected, + IDictionary actual) => + Assert.Equal( + expected.OrderBy(x => x.Key, StringComparer.Ordinal), + actual.OrderBy(x => x.Key, StringComparer.Ordinal)); +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj new file mode 100644 index 000000000..dd1592675 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + + diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryRestartTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryRestartTests.cs new file mode 100644 index 000000000..1a2e452b2 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryRestartTests.cs @@ -0,0 +1,174 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Contracts; +using Elsa.Diagnostics.OpenTelemetry.Core.Exceptions; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class GroundworkOpenTelemetryRestartTests : OpenTelemetryRestartContractTests, IAsyncLifetime +{ + private readonly OpenTelemetryGroundworkSqliteFixture _fixture = new(); + + protected override async ValueTask CreateStoreAsync() => + await _fixture.CreateStoreAsync(); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() => await _fixture.DisposeAsync(); +} + +public abstract class OpenTelemetryRestartContractTests +{ + [Fact] + public async Task Exact_immutable_counts_and_unfiltered_logs_survive_store_restart() + { + var expected = RestartScenario.Create(); + var beforeRestart = await CreateStoreAsync(); + + try + { + await beforeRestart.WriteAsync(expected.Batch); + } + finally + { + await DisposeStoreAsync(beforeRestart); + } + + var restarted = await CreateStoreAsync(); + + try + { + var diagnostics = await restarted.GetDiagnosticsAsync(); + Assert.Equal((0, 1, 1, 0, 1, 1), + (diagnostics.ResourceCount, diagnostics.TraceCount, diagnostics.SpanCount, + diagnostics.MetricInstrumentCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + + var logs = await restarted.QueryLogsAsync(new OpenTelemetryLogFilter { Take = 10 }); + var log = Assert.Single(logs.Items); + Assert.Equal(expected.Log.Id, log.Id); + Assert.Equal(expected.Log.Body, log.Body); + Assert.Equal(expected.Log.Timestamp, log.Timestamp); + Assert.Equal(expected.Log.SeverityText, log.SeverityText); + AssertAttributes(expected.Log.Attributes, log.Attributes); + + await Assert.ThrowsAsync(() => + restarted.QueryTracesAsync(new OpenTelemetryTraceFilter()).AsTask()); + await Assert.ThrowsAsync(() => + restarted.GetTraceAsync(expected.Trace.TraceId).AsTask()); + await Assert.ThrowsAsync(() => + restarted.QueryMetricsAsync(new OpenTelemetryMetricFilter()).AsTask()); + } + finally + { + await DisposeStoreAsync(restarted); + } + } + + protected abstract ValueTask CreateStoreAsync(); + + private static async ValueTask DisposeStoreAsync(IOpenTelemetryStore store) + { + if (store is IAsyncDisposable asyncDisposable) + await asyncDisposable.DisposeAsync(); + else if (store is IDisposable disposable) + disposable.Dispose(); + } + + private static void AssertAttributes( + IDictionary expected, + IDictionary actual) => + Assert.Equal( + expected.OrderBy(x => x.Key, StringComparer.Ordinal), + actual.OrderBy(x => x.Key, StringComparer.Ordinal)); + + private sealed record RestartScenario( + TelemetryResource Resource, + TelemetryTrace Trace, + TelemetrySpan Span, + MetricInstrument Instrument, + MetricPoint Point, + OtlpLogRecord Log) + { + private static DateTimeOffset Timestamp { get; } = + new(2026, 7, 13, 10, 0, 0, TimeSpan.Zero); + + public OpenTelemetryBatch Batch => + new([], [Trace], [Span], [], [Point], [Log]); + + public static RestartScenario Create() + { + var resource = new TelemetryResource( + "resource-api", + "api", + "api-instance", + "dotnet", + new Dictionary { ["deployment.environment"] = "test" }, + Timestamp, + TelemetryResourceStatus.Active); + var trace = new TelemetryTrace( + "trace-1", + "span-root", + "trace-orders", + Timestamp, + Timestamp.AddMilliseconds(25), + TimeSpan.FromMilliseconds(25), + SpanStatus.Ok, + [resource.Id], + ["workflow-1"], + 2); + var span = new TelemetrySpan( + "span-record-1", + trace.TraceId, + "span-1", + trace.RootSpanId, + resource.Id, + "process-order", + "internal", + Timestamp, + Timestamp.AddMilliseconds(10), + SpanStatus.Ok, + null, + new Dictionary { ["http.method"] = "GET" }, + [new TelemetrySpanEvent( + "order-accepted", + Timestamp.AddMilliseconds(1), + new Dictionary { ["event.attr"] = "value" })], + [new TelemetrySpanLink( + "linked-trace", + "linked-span", + new Dictionary { ["link.attr"] = "value" })]); + var instrument = new MetricInstrument( + "instrument-1", + resource.Id, + "request.duration", + "ms", + "duration", + MetricKind.Gauge, + new Dictionary { ["instrument.attr"] = "value" }); + var point = new MetricPoint( + "point-1", + instrument.Id, + instrument.Name, + resource.Id, + Timestamp, + 42, + null, + null, + new Dictionary { ["point.attr"] = "value" }, + trace.TraceId, + span.SpanId); + var log = new OtlpLogRecord( + "log-1", + resource.Id, + Timestamp, + "Information", + null, + "hello after restart", + trace.TraceId, + span.SpanId, + new Dictionary { ["log.attr"] = "value" }); + + return new(resource, trace, span, instrument, point, log); + } + } +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryStoreTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryStoreTests.cs new file mode 100644 index 000000000..f79033022 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/GroundworkOpenTelemetryStoreTests.cs @@ -0,0 +1,489 @@ +using System.Text.Json; +using Elsa.Diagnostics.OpenTelemetry.Core.Exceptions; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.OpenTelemetry.Core.Options; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; +using Elsa.Diagnostics.Persistence.Draining; +using Groundwork.DiagnosticRecords; +using Groundwork.Documents.Store; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class GroundworkOpenTelemetryStoreTests : IAsyncLifetime +{ + private readonly OpenTelemetryGroundworkSqliteFixture _fixture = new(); + + [Fact] + public void Tenant_scope_and_source_are_all_part_of_non_aliasable_stream_identity() + { + var bindings = new[] + { + GroundworkOpenTelemetryBinding.Create("tenant-a", "shell-a", "collector-a"), + GroundworkOpenTelemetryBinding.Create("tenant-b", "shell-a", "collector-a"), + GroundworkOpenTelemetryBinding.Create("tenant-a", "shell-b", "collector-a"), + GroundworkOpenTelemetryBinding.Create("tenant-a", "shell-a", "collector-b") + }; + + Assert.Empty(typeof(GroundworkOpenTelemetryBinding).GetConstructors()); + var streams = bindings.SelectMany(binding => + new[] { binding.TraceStreamId, binding.SpanStreamId, binding.MetricPointStreamId, binding.LogStreamId }).ToArray(); + Assert.Equal(streams.Length, streams.Distinct(StringComparer.Ordinal).Count()); + } + + [Fact] + public async Task Document_session_must_match_the_explicit_tenant_scope_and_source_binding() + { + var providers = await _fixture.CreateProvidersAsync(); + var wrongSource = GroundworkOpenTelemetryBinding.Create("tenant-a", "shell-a", "collector-b"); + + Assert.Throws(() => new GroundworkOpenTelemetryStore( + providers, + Options.Create(new OpenTelemetryDiagnosticsOptions()), + wrongSource)); + } + + [Fact] + public async Task Partial_multi_stream_retry_after_restart_reuses_batch_identity_and_does_not_duplicate_records() + { + var providers = await _fixture.CreateProvidersAsync(); + var traces = new ObservingRecordStore(providers.Traces); + var points = new ObservingRecordStore(providers.MetricPoints, failFirstAppend: true); + var store = _fixture.CreateStore(providers with + { + Traces = traces, + MetricPoints = points + }); + var batchId = DiagnosticsDrainBatchId.New(); + var batch = CreateBatch(includeCatalogs: false); + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batchId, batch).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.ProviderFailure, failure.Reason); + Assert.Equal("write", failure.Operation); + Assert.Equal(batchId.ToString(), failure.Context["batchId"]); + Assert.IsType(failure.InnerException); + var partial = await store.GetDiagnosticsAsync(); + Assert.Equal(1, partial.TraceCount); + Assert.Equal(1, partial.SpanCount); + Assert.Equal(0, partial.MetricPointCount); + Assert.Equal(0, partial.LogRecordCount); + + var restartedProviders = await _fixture.CreateProvidersAsync(); + var restartedTraces = new ObservingRecordStore(restartedProviders.Traces); + var restartedPoints = new ObservingRecordStore(restartedProviders.MetricPoints); + var restarted = _fixture.CreateStore(restartedProviders with + { + Traces = restartedTraces, + MetricPoints = restartedPoints + }); + await restarted.WriteAsync(batchId, batch); + + var completed = await restarted.GetDiagnosticsAsync(); + Assert.Equal((1, 1, 1, 1), + (completed.TraceCount, completed.SpanCount, completed.MetricPointCount, completed.LogRecordCount)); + Assert.Single(traces.Requests); + Assert.Empty(restartedTraces.Requests); + Assert.Equal([DiagnosticAppendStatus.Committed], traces.Outcomes); + Assert.Empty(restartedTraces.Outcomes); + Assert.Single(points.Requests); + Assert.Single(restartedPoints.Requests); + Assert.Equal(points.Requests[0].OperationId, restartedPoints.Requests[0].OperationId); + Assert.Equal(points.Requests[0].RequestFingerprint, restartedPoints.Requests[0].RequestFingerprint); + Assert.Empty(points.Outcomes); + Assert.Equal([DiagnosticAppendStatus.Committed], restartedPoints.Outcomes); + + Assert.True((await CaptureOperationsAsync(restartedProviders.Documents)).Single().Version > 1); + } + + [Fact] + public async Task Identical_independent_captures_do_not_collapse() + { + var store = await _fixture.CreateStoreAsync(); + var batch = CreateBatch(includeCatalogs: false); + + await store.WriteAsync(batch); + await store.WriteAsync(batch); + + var diagnostics = await store.GetDiagnosticsAsync(); + Assert.Equal((2, 2, 2, 2), + (diagnostics.TraceCount, diagnostics.SpanCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + } + + [Fact] + public async Task Reusing_batch_identity_for_different_canonical_input_fails_without_mutation() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batchId = DiagnosticsDrainBatchId.New(); + var first = CreateBatch(includeCatalogs: false); + await store.WriteAsync(batchId, first); + var changed = first with + { + Logs = first.Logs.Select(x => x with { Body = $"{x.Body}-changed" }).ToArray() + }; + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batchId, changed).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.ConflictingOperation, failure.Reason); + Assert.Equal(batchId.ToString(), failure.Context["batchId"]); + Assert.IsType(failure.InnerException); + + var diagnostics = await store.GetDiagnosticsAsync(); + Assert.Equal((1, 1, 1, 1), + (diagnostics.TraceCount, diagnostics.SpanCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + } + + [Fact] + public async Task Expired_pending_stream_attempt_fails_explicitly_after_restart_without_duplicate_or_late_progress() + { + var clock = new MutableTimeProvider(TimeProvider.System.GetUtcNow()); + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers with + { + MetricPoints = new ObservingRecordStore(providers.MetricPoints, failFirstAppend: true) + }, clock); + var batchId = DiagnosticsDrainBatchId.New(); + var batch = CreateBatch(includeCatalogs: false); + var appendFailure = await Assert.ThrowsAsync(() => + store.WriteAsync(batchId, batch).AsTask()); + Assert.IsType(appendFailure.InnerException); + clock.Advance(TimeSpan.FromHours(1) + TimeSpan.FromMinutes(5) + TimeSpan.FromTicks(1)); + + var restartedProviders = await _fixture.CreateProvidersAsync(); + var restarted = _fixture.CreateStore(restartedProviders, clock); + var failure = await Assert.ThrowsAsync(() => + restarted.WriteAsync(batchId, batch).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.ExpiredOperation, failure.Reason); + Assert.Equal(batchId.ToString(), failure.Context["batchId"]); + Assert.IsType(failure.InnerException); + + var diagnostics = await restarted.GetDiagnosticsAsync(); + Assert.Equal((1, 1, 0, 0), + (diagnostics.TraceCount, diagnostics.SpanCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + } + + [Fact] + public async Task Catalog_identity_writes_are_rejected_before_mutation_until_portable_case_equivalence_is_available() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(CreateBatch()).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.CapabilityUnavailable, failure.Reason); + Assert.Equal("portable-case-equivalence", failure.Context["capability"]); + + await AssertNoMutationAsync(store, providers.Documents); + } + + [Fact] + public async Task Affected_case_insensitive_reads_are_rejected_before_provider_io() + { + var store = await _fixture.CreateStoreAsync(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAsync(() => store.QueryResourcesAsync( + new() { ServiceName = "API" }, cancellation.Token).AsTask()); + await Assert.ThrowsAsync(() => store.QueryTracesAsync( + new(), cancellation.Token).AsTask()); + await Assert.ThrowsAsync(() => store.GetTraceAsync( + "TRACE-1", cancellation.Token).AsTask()); + await Assert.ThrowsAsync(() => store.QueryMetricsAsync( + new(), cancellation.Token).AsTask()); + await Assert.ThrowsAsync(() => store.QueryLogsAsync( + new() { TraceId = "TRACE-1" }, cancellation.Token).AsTask()); + } + + [Fact] + public async Task Oversized_normalized_stream_batch_is_rejected_before_ledger_or_record_mutation() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batch = CreateBatch(includeCatalogs: false); + var template = Assert.Single(batch.Logs); + batch = batch with + { + Traces = [], + Spans = [], + MetricPoints = [], + Logs = Enumerable.Range(0, 1_001) + .Select(index => template with { Id = $"log-{index:D4}" }) + .ToArray() + }; + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batch).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.InvalidRecord, failure.Reason); + Assert.IsType(failure.InnerException); + + await AssertNoMutationAsync(store, providers.Documents); + } + + [Fact] + public async Task Provider_invalid_normalized_record_is_rejected_before_ledger_or_record_mutation() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batch = CreateBatch(includeCatalogs: false); + batch = batch with + { + Traces = [], + Spans = [], + MetricPoints = [], + Logs = batch.Logs.Select(x => x with { SeverityText = new string('s', 4_097) }).ToArray() + }; + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batch).AsTask()); + Assert.Equal(OpenTelemetryPersistenceFailureReason.InvalidRecord, failure.Reason); + Assert.IsType(failure.InnerException); + + await AssertNoMutationAsync(store, providers.Documents); + } + + [Fact] + public async Task Canonical_payload_failures_are_translated_but_caller_argument_failures_remain_argument_exceptions() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batch = CreateBatch(includeCatalogs: false); + var log = Assert.Single(batch.Logs); + var invalidPayload = batch with + { + Logs = [log with { Attributes = null! }] + }; + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(invalidPayload).AsTask()); + Assert.IsType(failure.InnerException); + + var conflictingInput = batch with + { + Logs = [log, log with { Body = "different" }] + }; + await Assert.ThrowsAsync(() => store.WriteAsync(conflictingInput).AsTask()); + + await AssertNoMutationAsync(store, providers.Documents); + } + + [Fact] + public async Task Concurrent_same_batch_writers_converge_through_capture_ledger_cas() + { + var first = _fixture.CreateStore(await _fixture.CreateProvidersAsync()); + var second = _fixture.CreateStore(await _fixture.CreateProvidersAsync()); + var batchId = DiagnosticsDrainBatchId.New(); + var batch = CreateBatch(includeCatalogs: false); + + await Task.WhenAll( + first.WriteAsync(batchId, batch).AsTask(), + second.WriteAsync(batchId, batch).AsTask()); + + var diagnostics = await first.GetDiagnosticsAsync(); + Assert.Equal((1, 1, 1, 1), + (diagnostics.TraceCount, diagnostics.SpanCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + } + + [Fact] + public async Task Cancellation_after_one_committed_stream_records_partial_progress_and_retry_resumes() + { + using var cancellation = new CancellationTokenSource(); + var providers = await _fixture.CreateProvidersAsync(); + var traces = new ObservingRecordStore(providers.Traces, afterSuccessfulAppend: cancellation.Cancel); + var store = _fixture.CreateStore(providers with { Traces = traces }); + var batchId = DiagnosticsDrainBatchId.New(); + var batch = CreateBatch(includeCatalogs: false); + + await Assert.ThrowsAnyAsync(() => + store.WriteAsync(batchId, batch, cancellation.Token).AsTask()); + + var restarted = _fixture.CreateStore(await _fixture.CreateProvidersAsync()); + await restarted.WriteAsync(batchId, batch); + var diagnostics = await restarted.GetDiagnosticsAsync(); + Assert.Equal((1, 1, 1, 1), + (diagnostics.TraceCount, diagnostics.SpanCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + } + + [Fact] + public async Task Malformed_capture_ledger_is_reported_as_provider_neutral_corrupt_data() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batchId = DiagnosticsDrainBatchId.New(); + var result = await providers.Documents.SaveAsync(new( + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind, + batchId.ToString(), + OpenTelemetryGroundworkStorageSchema.SchemaVersion, + "[]", + 0)); + Assert.Equal(DocumentStoreWriteStatus.Saved, result.Status); + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batchId, CreateBatch(includeCatalogs: false)).AsTask()); + + Assert.Equal(OpenTelemetryPersistenceFailureReason.CorruptData, failure.Reason); + Assert.Equal(batchId.ToString(), failure.Context["batchId"]); + Assert.IsType(failure.InnerException); + } + + [Fact] + public async Task Null_capture_stream_attempt_is_reported_as_provider_neutral_corrupt_data() + { + var providers = await _fixture.CreateProvidersAsync(); + var store = _fixture.CreateStore(providers); + var batchId = DiagnosticsDrainBatchId.New(); + var content = JsonSerializer.Serialize(new + { + ledgerSchemaVersion = 2, + batchId = batchId.ToString(), + fingerprint = "persisted-fingerprint", + createdAt = DateTimeOffset.UtcNow, + tenantId = _fixture.Binding.TenantId, + scopeId = _fixture.Binding.ScopeId, + sourceId = _fixture.Binding.SourceId, + streams = new Dictionary { ["traces"] = null } + }); + var result = await providers.Documents.SaveAsync(new( + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind, + batchId.ToString(), + OpenTelemetryGroundworkStorageSchema.SchemaVersion, + content, + 0)); + Assert.Equal(DocumentStoreWriteStatus.Saved, result.Status); + + var failure = await Assert.ThrowsAsync(() => + store.WriteAsync(batchId, CreateBatch(includeCatalogs: false)).AsTask()); + + Assert.Equal(OpenTelemetryPersistenceFailureReason.CorruptData, failure.Reason); + Assert.Equal(batchId.ToString(), failure.Context["batchId"]); + } + + [Fact] + public async Task Query_and_diagnostics_provider_failures_are_translated_at_the_public_boundary() + { + var providers = await _fixture.CreateProvidersAsync(); + var queryFailure = new IOException("Injected query failure."); + var inspectFailure = new IOException("Injected inspect failure."); + var store = _fixture.CreateStore(providers with + { + Logs = new ObservingRecordStore(providers.Logs, queryFailure: queryFailure), + Traces = new ObservingRecordStore(providers.Traces, inspectFailure: inspectFailure) + }); + + var queryError = await Assert.ThrowsAsync(() => + store.QueryLogsAsync(new()).AsTask()); + Assert.Equal("query-logs", queryError.Operation); + Assert.Same(queryFailure, queryError.InnerException); + + var diagnosticsError = await Assert.ThrowsAsync(() => + store.GetDiagnosticsAsync().AsTask()); + Assert.Equal("get-diagnostics", diagnosticsError.Operation); + Assert.Same(inspectFailure, diagnosticsError.InnerException); + } + +#pragma warning disable GW0004 + private static async Task> CaptureOperationsAsync(IDocumentStore documents) => + (await documents.QueryAsync(new PortableDocumentQuery( + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind))).Documents; +#pragma warning restore GW0004 + + private static async Task AssertNoMutationAsync( + GroundworkOpenTelemetryStore store, + IDocumentStore documents) + { + var diagnostics = await store.GetDiagnosticsAsync(); + Assert.Equal((0, 0, 0, 0, 0, 0), + (diagnostics.ResourceCount, diagnostics.TraceCount, diagnostics.SpanCount, + diagnostics.MetricInstrumentCount, diagnostics.MetricPointCount, diagnostics.LogRecordCount)); + Assert.Empty(await CaptureOperationsAsync(documents)); + } + + private static OpenTelemetryBatch CreateBatch(bool includeCatalogs = true) + { + var timestamp = new DateTimeOffset(2026, 7, 14, 1, 0, 0, TimeSpan.Zero); + var resource = new TelemetryResource( + "resource-1", "api", "api-1", "dotnet", new Dictionary(), timestamp, + TelemetryResourceStatus.Active); + var trace = new TelemetryTrace( + "trace-1", "span-root", "request", timestamp, timestamp.AddMilliseconds(10), + TimeSpan.FromMilliseconds(10), SpanStatus.Ok, [resource.Id], [], 1); + var span = new TelemetrySpan( + "span-record-1", trace.TraceId, "span-1", null, resource.Id, "request", "internal", + timestamp, timestamp.AddMilliseconds(10), SpanStatus.Ok, null, + new Dictionary(), [], []); + var instrument = new MetricInstrument( + "instrument-1", resource.Id, "request.duration", "ms", null, MetricKind.Gauge, + new Dictionary()); + var point = new MetricPoint( + "point-1", instrument.Id, instrument.Name, resource.Id, timestamp, 10, null, null, + new Dictionary(), trace.TraceId, span.SpanId); + var log = new OtlpLogRecord( + "log-1", resource.Id, timestamp, "Information", null, "request completed", trace.TraceId, + span.SpanId, new Dictionary()); + return new( + includeCatalogs ? [resource] : [], + [trace], + [span], + includeCatalogs ? [instrument] : [], + [point], + [log]); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() => await _fixture.DisposeAsync(); + + private sealed class ObservingRecordStore( + IDiagnosticRecordStore inner, + bool failFirstAppend = false, + Action? afterSuccessfulAppend = null, + Exception? queryFailure = null, + Exception? inspectFailure = null) : IDiagnosticRecordStore + { + private int _failNext = failFirstAppend ? 1 : 0; + + public DiagnosticRecordStoreHandlers Handlers => inner.Handlers; + public List Requests { get; } = []; + public List Outcomes { get; } = []; + + public async ValueTask AppendAsync( + DiagnosticRecordBatch batch, + CancellationToken cancellationToken = default) + { + Requests.Add(batch); + if (Interlocked.Exchange(ref _failNext, 0) != 0) + throw new IOException("Injected metric append failure after earlier streams committed."); + + var result = await inner.AppendAsync(batch, cancellationToken); + Outcomes.Add(result.Status); + afterSuccessfulAppend?.Invoke(); + return result; + } + + public ValueTask QueryAsync( + DiagnosticRecordQuery query, + CancellationToken cancellationToken = default) => + queryFailure is null + ? inner.QueryAsync(query, cancellationToken) + : ValueTask.FromException(queryFailure); + + public ValueTask InspectAsync( + DiagnosticStreamInspectionRequest request, + CancellationToken cancellationToken = default) => + inspectFailure is null + ? inner.InspectAsync(request, cancellationToken) + : ValueTask.FromException(inspectFailure); + } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan duration) => _utcNow += duration; + } +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkSqliteFixture.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkSqliteFixture.cs new file mode 100644 index 000000000..9ce6fddd1 --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkSqliteFixture.cs @@ -0,0 +1,55 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Options; +using Groundwork.Core.Capabilities; +using Groundwork.Documents.Scoping; +using Groundwork.Sqlite.DiagnosticRecords; +using Groundwork.Sqlite.Documents; +using Microsoft.Extensions.Options; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +internal sealed class OpenTelemetryGroundworkSqliteFixture : IAsyncDisposable +{ + private readonly string _databasePath = Path.Combine( + Path.GetTempPath(), + $"elsa-open-telemetry-{Guid.NewGuid():N}.db"); + + public GroundworkOpenTelemetryBinding Binding { get; } = + GroundworkOpenTelemetryBinding.Create("tenant-a", "shell-a", "collector-a"); + + public async Task CreateProvidersAsync() + { + var connectionString = $"Data Source={_databasePath}"; + var streams = OpenTelemetryGroundworkStorageSchema.CreateStreams(Binding); + var traces = await SqliteDiagnosticRecordStoreFactory.CreateAsync(connectionString, streams[0]); + var spans = await SqliteDiagnosticRecordStoreFactory.CreateAsync(connectionString, streams[1]); + var points = await SqliteDiagnosticRecordStoreFactory.CreateAsync(connectionString, streams[2]); + var logs = await SqliteDiagnosticRecordStoreFactory.CreateAsync(connectionString, streams[3]); + var documents = await SqliteDocumentStoreFactory.CreateAsync( + connectionString, + OpenTelemetryGroundworkStorageSchema.CreateDocumentManifest(), + new ProviderIdentity("groundwork-sqlite", "1.0.0"), + DocumentStoreAccess.Scoped(Binding.DocumentStorageScope)); + return new(traces, spans, points, logs, documents); + } + + public async ValueTask CreateStoreAsync() + { + var providers = await CreateProvidersAsync(); + return CreateStore(providers); + } + + public GroundworkOpenTelemetryStore CreateStore( + GroundworkOpenTelemetryStores providers, + TimeProvider? timeProvider = null) => new( + providers, + Options.Create(new OpenTelemetryDiagnosticsOptions()), + Binding, + timeProvider); + + public ValueTask DisposeAsync() + { + if (File.Exists(_databasePath)) + File.Delete(_databasePath); + return ValueTask.CompletedTask; + } +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkStorageSchemaTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkStorageSchemaTests.cs new file mode 100644 index 000000000..3c75d2abd --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryGroundworkStorageSchemaTests.cs @@ -0,0 +1,100 @@ +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Catalogs; +using Groundwork.Core.Intents; +using Groundwork.Core.Manifests; +using Groundwork.Core.PhysicalStorage; +using Groundwork.Documents.Scoping; +using Groundwork.SqlServer; +using Groundwork.SqlServer.Documents; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class OpenTelemetryGroundworkStorageSchemaTests +{ + [Fact] + public void Schema_declares_explicit_streams_dedicated_catalog_tables_indexes_and_capture_ledger() + { + var binding = GroundworkOpenTelemetryBinding.Create("tenant", "scope", "source"); + var streams = OpenTelemetryGroundworkStorageSchema.CreateStreams(binding); + var manifest = OpenTelemetryGroundworkStorageSchema.CreateDocumentManifest(); + + Assert.Equal( + new[] { binding.TraceStreamId, binding.SpanStreamId, binding.MetricPointStreamId, binding.LogStreamId }, + streams.Select(x => x.Stream.Value)); + Assert.Equal( + new[] + { + CatalogDocuments.ResourceKind, + CatalogDocuments.InstrumentKind, + OpenTelemetryGroundworkStorageSchema.OperationLedgerKind + }.Order(StringComparer.Ordinal), + manifest.StorageUnits.Select(x => x.Identity.Value).Order(StringComparer.Ordinal)); + Assert.All(manifest.StorageUnits, unit => Assert.Equal(TenancyPolicy.Scoped, unit.Tenancy)); + Assert.All(manifest.StorageUnits, unit => + { + var policy = Assert.IsType(unit.PhysicalStorage!.Policy); + Assert.Equal(PhysicalStorageForm.DedicatedDocumentTable, policy.Definition.Form); + }); + + var resources = manifest.StorageUnits.Single(x => x.Identity.Value == CatalogDocuments.ResourceKind); + Assert.Contains(resources.PhysicalStorage!.LogicalIndexes, + x => x.Identity == OpenTelemetryGroundworkStorageSchema.ByResourceStatusIndex); + Assert.Contains(resources.PhysicalStorage.BoundedQueries, + x => x.Identity == "resources-by-last-seen" && x.ExecutionClass == BoundedQueryExecutionClass.ScaleBearing); + var resourcePolicy = Assert.IsType(resources.PhysicalStorage.Policy); + Assert.Contains(resourcePolicy.Definition.Indexes, + x => x.LogicalName == OpenTelemetryGroundworkStorageSchema.ByRetentionIndex && x.Columns.Count == 3); + Assert.Contains("open-telemetry-capture-operation", OpenTelemetryGroundworkStorageSchema.RequiredOperationLedgers); + } + + [Fact] + public void Schema_resolves_compiles_and_passes_the_sql_server_physical_store_validator_without_io() + { + var binding = GroundworkOpenTelemetryBinding.Create("tenant", "scope", "source"); + var manifest = OpenTelemetryGroundworkStorageSchema.CreateDocumentManifest(); + var resolution = PhysicalStorageResolver.Resolve( + manifest, + PhysicalNamePolicy.Identity, + SqlServerGroundworkCapabilities.PhysicalNames); + + Assert.True(resolution.IsValid, string.Join("; ", resolution.Diagnostics.Select(x => x.Message))); + var compilation = ExecutableStorageRouteCompiler.Compile(resolution.Definitions); + Assert.True(compilation.IsValid, string.Join("; ", compilation.Diagnostics.Select(x => x.Message))); + + var store = new SqlServerPhysicalDocumentStore( + "Server=localhost;Database=groundwork;User ID=sa;Password=not-used;Encrypt=False", + manifest, + compilation.Routes, + DocumentStoreAccess.Scoped(binding.DocumentStorageScope)); + + Assert.Equal(binding.DocumentStorageScope, store.Access.Scope); + } + + [Fact] + public void Every_declared_physical_index_is_within_sql_server_key_width_limits() + { + var manifest = OpenTelemetryGroundworkStorageSchema.CreateDocumentManifest(); + + foreach (var unit in manifest.StorageUnits) + { + var policy = Assert.IsType(unit.PhysicalStorage!.Policy); + var columns = policy.Definition.ProjectedColumns.ToDictionary(x => x.LogicalName, StringComparer.Ordinal); + foreach (var index in policy.Definition.Indexes) + { + var keyBytes = index.Columns.Sum(column => column.ColumnLogicalName switch + { + "storage_scope" => 128 * 2, + _ when columns[column.ColumnLogicalName].Type == PortablePhysicalType.String => + columns[column.ColumnLogicalName].Length!.Value * 2, + _ when columns[column.ColumnLogicalName].Type == PortablePhysicalType.Int32 => 4, + _ when columns[column.ColumnLogicalName].Type == PortablePhysicalType.Int64 => 8, + _ => throw new Xunit.Sdk.XunitException( + $"SQL Server validator does not recognize index column '{column.ColumnLogicalName}'.") + }); + + Assert.True(keyBytes <= 1_700, + $"SQL Server physical index '{index.LogicalName}' requires {keyBytes} key bytes; the provider maximum is 1700."); + } + } + } +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryRecordStreamDefinitionTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryRecordStreamDefinitionTests.cs new file mode 100644 index 000000000..f70c4f32b --- /dev/null +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Groundwork/Tests/OpenTelemetryRecordStreamDefinitionTests.cs @@ -0,0 +1,142 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Records; +using Groundwork.DiagnosticRecords; +using Xunit; + +namespace Elsa.Diagnostics.OpenTelemetry.Persistence.Groundwork.Tests; + +public sealed class OpenTelemetryRecordStreamDefinitionTests +{ + private static readonly DateTimeOffset Timestamp = + new(2026, 7, 14, 10, 11, 12, TimeSpan.Zero); + + [Fact] + public void Every_immutable_signal_has_a_valid_bounded_distinct_stream_definition() + { + var definitions = Definitions(); + + Assert.Equal(4, definitions.Select(x => x.LogicalStorageName).Distinct(StringComparer.Ordinal).Count()); + foreach (var definition in definitions) + { + DiagnosticRecordStreamDefinitionValidator.ValidateAndThrow(definition); + Assert.Equal(CanonicalRecordSerializer.SchemaVersion, definition.SchemaVersion); + Assert.Equal(definition.Fields.Count, definition.Limits.MaxFieldsPerRecord); + Assert.InRange(definition.Limits.MaxBatchRecords, 1, 1_000); + Assert.InRange(definition.Limits.MaxQueryLimit, 1, 5_000); + Assert.All(definition.Fields, field => Assert.NotEmpty(field.SupportedPredicates)); + Assert.All( + definition.Fields.Where(x => x.Type == DiagnosticFieldType.String), + field => Assert.Equal(DiagnosticStringCasePolicy.Ordinal, field.CasePolicy)); + } + } + + [Fact] + public void Definitions_declare_the_portable_ordinal_range_order_membership_and_latest_query_contract() + { + var trace = OpenTelemetryRecordStreamDefinitions.CreateTraces("traces"); + var span = OpenTelemetryRecordStreamDefinitions.CreateSpans("spans"); + var point = OpenTelemetryRecordStreamDefinitions.CreateMetricPoints("points"); + var log = OpenTelemetryRecordStreamDefinitions.CreateLogs("logs"); + + var traceId = Field(trace, RecordFields.TraceId); + Assert.True(traceId.SupportsLatestPerKey); + Assert.Equal(DiagnosticStringCasePolicy.Ordinal, traceId.CasePolicy); + Assert.Contains(DiagnosticPredicateOperator.Contains, traceId.SupportedPredicates); + + var resourceId = Field(trace, RecordFields.ResourceId); + Assert.Equal(DiagnosticFieldCardinality.Multiple, resourceId.Cardinality); + Assert.True(resourceId.IsRequired); + Assert.Equal(256, resourceId.MaxValues); + + AssertOrderableRange(Field(trace, RecordFields.StartTime)); + AssertOrderableRange(Field(span, RecordFields.StartTime)); + AssertOrderableRange(Field(point, RecordFields.Timestamp)); + AssertOrderableRange(Field(log, RecordFields.Timestamp)); + Assert.Contains(DiagnosticPredicateOperator.Contains, Field(log, RecordFields.Body).SupportedPredicates); + } + + [Fact] + public void Unicode_query_fields_remain_valid_raw_ordinal_values() + { + var definition = OpenTelemetryRecordStreamDefinitions.CreateTraces("traces"); + var record = new CanonicalRecordSerializer().ToRecord( + "trace-record-unicode", + new TelemetryTrace( + "trace-één", + "span-hoofd", + "订单 verwerken", + Timestamp, + Timestamp.AddSeconds(1), + TimeSpan.FromSeconds(1), + SpanStatus.Ok, + ["resource-tōkyō"], + ["workflow-δοκιμή"], + 1)); + + DiagnosticRecordRequestValidator.Validate( + DiagnosticRecordBatch.Create( + new("tenant", "scope"), + definition.Stream, + new(Timestamp, "append-unicode"), + [record]), + definition); + } + + [Fact] + public void Canonical_record_fields_are_fully_declared_by_the_corresponding_stream() + { + var serializer = new CanonicalRecordSerializer(); + var attributes = new Dictionary { ["key"] = "value" }; + + var records = new[] + { + (serializer.ToRecord("trace-record", new TelemetryTrace( + "trace-1", "root", "orders", Timestamp, Timestamp.AddSeconds(1), TimeSpan.FromSeconds(1), + SpanStatus.Ok, ["resource-1"], ["workflow-1"], 1)), + OpenTelemetryRecordStreamDefinitions.CreateTraces("traces")), + (serializer.ToRecord("span-record", new TelemetrySpan( + "span-record", "trace-1", "span-1", "root", "resource-1", "orders", "internal", + Timestamp, Timestamp.AddSeconds(1), SpanStatus.Ok, null, attributes, [], [])), + OpenTelemetryRecordStreamDefinitions.CreateSpans("spans")), + (serializer.ToRecord("point-record", new MetricPoint( + "point-1", "instrument-1", "request.duration", "resource-1", Timestamp, 1, null, null, + attributes, "trace-1", "span-1")), + OpenTelemetryRecordStreamDefinitions.CreateMetricPoints("points")), + (serializer.ToRecord("log-record", new OtlpLogRecord( + "log-1", "resource-1", Timestamp, "Information", 9, "accepted", "trace-1", "span-1", + attributes)), + OpenTelemetryRecordStreamDefinitions.CreateLogs("logs")) + }; + + foreach (var (record, definition) in records) + { + Assert.Equal( + definition.Fields.Select(x => x.Name).OrderBy(x => x, StringComparer.Ordinal), + record.Fields!.Keys.OrderBy(x => x, StringComparer.Ordinal)); + DiagnosticRecordRequestValidator.Validate( + DiagnosticRecordBatch.Create( + new("tenant", "scope"), + definition.Stream, + new(Timestamp, $"append-{definition.Stream.Value}"), + [record]), + definition); + } + } + + private static DiagnosticRecordStreamDefinition[] Definitions() => + [ + OpenTelemetryRecordStreamDefinitions.CreateTraces("traces"), + OpenTelemetryRecordStreamDefinitions.CreateSpans("spans"), + OpenTelemetryRecordStreamDefinitions.CreateMetricPoints("points"), + OpenTelemetryRecordStreamDefinitions.CreateLogs("logs") + ]; + + private static DiagnosticFieldDefinition Field(DiagnosticRecordStreamDefinition definition, string name) => + Assert.Single(definition.Fields, x => StringComparer.Ordinal.Equals(x.Name, name)); + + private static void AssertOrderableRange(DiagnosticFieldDefinition field) + { + Assert.True(field.IsOrderable); + Assert.Contains(DiagnosticPredicateOperator.RangeInclusive, field.SupportedPredicates); + } +} diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/Elsa.Diagnostics.OpenTelemetry.Tests.csproj b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/Elsa.Diagnostics.OpenTelemetry.Tests.csproj index 051dd1742..1b0373e2d 100644 --- a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/Elsa.Diagnostics.OpenTelemetry.Tests.csproj +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/Elsa.Diagnostics.OpenTelemetry.Tests.csproj @@ -5,6 +5,7 @@ enable enable false + true @@ -21,6 +22,7 @@ + diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/InMemoryOpenTelemetryLiveFeedTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/InMemoryOpenTelemetryLiveFeedTests.cs index 399c3cbce..dc6ee4b50 100644 --- a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/InMemoryOpenTelemetryLiveFeedTests.cs +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/InMemoryOpenTelemetryLiveFeedTests.cs @@ -10,6 +10,9 @@ public class InMemoryOpenTelemetryLiveFeedTests { private static readonly DateTimeOffset Now = new(2026, 5, 26, 10, 0, 0, TimeSpan.Zero); + [Fact] + public void Live_feed_is_sealed() => Assert.True(typeof(InMemoryOpenTelemetryLiveFeed).IsSealed); + [Fact] public async Task SubscribeAsync_DeliversPublishedTraceMatchingFilter() { diff --git a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/OpenTelemetryFeatureTests.cs b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/OpenTelemetryFeatureTests.cs index 4f801cfb4..2d86289a0 100644 --- a/tests/Elsa/Diagnostics/OpenTelemetry/Tests/OpenTelemetryFeatureTests.cs +++ b/tests/Elsa/Diagnostics/OpenTelemetry/Tests/OpenTelemetryFeatureTests.cs @@ -1,6 +1,8 @@ using Elsa.Diagnostics.OpenTelemetry.Core.Contracts; using Elsa.Diagnostics.OpenTelemetry.Core.Options; +using Elsa.Diagnostics.OpenTelemetry.Core.Models; using Elsa.Diagnostics.OpenTelemetry.Endpoints; +using Elsa.Diagnostics.Persistence.Observability; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -8,6 +10,7 @@ namespace Elsa.Diagnostics.OpenTelemetry.Tests; public sealed class OpenTelemetryFeatureTests { + private static readonly DateTimeOffset Now = new(2026, 7, 13, 10, 0, 0, TimeSpan.Zero); [Fact] public void RegistersResolvableDiagnosticsServices() { @@ -50,4 +53,71 @@ public void AppliesConfiguredOptions() Assert.Equal("secret", options.ApiKey); Assert.False(options.AllowUnauthenticatedLoopback); } + + [Fact] + public async Task Production_composition_counts_each_subscriber_drop_summary_once() + { + var services = new ServiceCollection(); + new OpenTelemetryFeature { SubscriberChannelCapacity = 1 }.ConfigureServices(services); + using var provider = services.BuildServiceProvider(); + var feed = provider.GetRequiredService(); + var counters = provider.GetRequiredService(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await using var subscriber = feed.SubscribeAsync(new OpenTelemetryTraceFilter(), cancellation.Token) + .GetAsyncEnumerator(cancellation.Token); + var traces = Enumerable.Range(1, 3) + .Select(CreateTrace) + .ToArray(); + + await feed.PublishAsync(new OpenTelemetryBatch([], traces, [], [], [], []), cancellation.Token); + Assert.True(await subscriber.MoveNextAsync()); + Assert.True(await subscriber.MoveNextAsync()); + + var signal = Assert.IsType(subscriber.Current.DroppedItems); + Assert.Equal(signal.Count, + counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + [Fact] + public async Task Evicted_and_requeued_drop_summary_counts_each_underlying_item_once() + { + var services = new ServiceCollection(); + new OpenTelemetryFeature { SubscriberChannelCapacity = 1 }.ConfigureServices(services); + using var provider = services.BuildServiceProvider(); + var feed = provider.GetRequiredService(); + var counters = provider.GetRequiredService(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await using var subscriber = feed.SubscribeAsync(new OpenTelemetryTraceFilter(), cancellation.Token) + .GetAsyncEnumerator(cancellation.Token); + + await feed.PublishAsync( + new OpenTelemetryBatch([], Enumerable.Range(1, 3).Select(CreateTrace).ToArray(), [], [], [], []), + cancellation.Token); + Assert.True(await subscriber.MoveNextAsync()); + Assert.Equal("trace-3", subscriber.Current.Trace?.TraceId); + Assert.Equal(2, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + + await feed.PublishAsync( + new OpenTelemetryBatch([], [CreateTrace(4)], [], [], [], []), + cancellation.Token); + Assert.True(await subscriber.MoveNextAsync()); + Assert.Equal("trace-4", subscriber.Current.Trace?.TraceId); + Assert.True(await subscriber.MoveNextAsync()); + + var summary = Assert.IsType(subscriber.Current.DroppedItems); + Assert.Equal(2, summary.Count); + Assert.Equal(2, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + private static TelemetryTrace CreateTrace(int index) => new( + $"trace-{index}", + "root", + "name", + Now, + Now.AddMilliseconds(5), + TimeSpan.FromMilliseconds(5), + SpanStatus.Ok, + ["resource-1"], + [], + 1); } diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDocumentationTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDocumentationTests.cs new file mode 100644 index 000000000..0cf8708c2 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDocumentationTests.cs @@ -0,0 +1,90 @@ +using System.Runtime.CompilerServices; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDocumentationTests +{ + [Fact] + public void Owning_extension_point_catalog_declares_public_seam_semantics_and_helper_boundary() + { + var catalogPath = RepositoryPath("src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md"); + + Assert.True(File.Exists(catalogPath), $"Missing owning catalog: {catalogPath}"); + var catalog = File.ReadAllText(catalogPath); + Assert.Contains("IDiagnosticsDrainTarget", catalog, StringComparison.Ordinal); + Assert.Contains("IDiagnosticsPersistenceObserver", catalog, StringComparison.Ordinal); + Assert.Contains("Bridge", catalog, StringComparison.Ordinal); + Assert.Contains("Replacement", catalog, StringComparison.Ordinal); + Assert.Contains("conflict", catalog, StringComparison.OrdinalIgnoreCase); + Assert.Contains("helper boundary", catalog, StringComparison.OrdinalIgnoreCase); + Assert.Contains("DiagnosticsPersistenceObserverRegistrationValidator", catalog, StringComparison.Ordinal); + Assert.Contains("constitution-mandated", catalog, StringComparison.OrdinalIgnoreCase); + Assert.Contains("first-party implementation", catalog, StringComparison.OrdinalIgnoreCase); + Assert.Contains("internal Options adapter", catalog, StringComparison.Ordinal); + Assert.Contains("not an adapter extension contract", catalog, StringComparison.OrdinalIgnoreCase); + Assert.Contains("## Overridable contracts", catalog, StringComparison.Ordinal); + Assert.Contains("## Implementable contributor interfaces", catalog, StringComparison.Ordinal); + Assert.Contains("## Events", catalog, StringComparison.Ordinal); + Assert.True( + catalog.Contains("*(Core —", StringComparison.Ordinal) || + catalog.Contains("*(Feature contract —", StringComparison.Ordinal)); + } + + [Fact] + public void Root_extension_point_index_links_the_diagnostics_persistence_owner() + { + var rootIndex = File.ReadAllText(RepositoryPath("EXTENSION_POINTS.md")); + + Assert.Contains( + "src/Elsa/Diagnostics/Persistence/EXTENSION_POINTS.md", + rootIndex, + StringComparison.Ordinal); + } + + [Fact] + public void Replacement_contract_kind_is_declared_on_the_contracts_themselves() + { + var drainTarget = File.ReadAllText(RepositoryPath( + "src/Elsa/Diagnostics/Persistence/Draining/DiagnosticsDrainContracts.cs")); + var observer = File.ReadAllText(RepositoryPath( + "src/Elsa/Diagnostics/Persistence/Observability/DiagnosticsPersistenceObservability.cs")); + + Assert.Contains("Replacement contract", drainTarget, StringComparison.Ordinal); + Assert.Contains("Replacement contract", observer, StringComparison.Ordinal); + } + + [Fact] + public void Ef_oracle_inventory_names_exact_source_and_test_evidence_for_every_required_area() + { + var inventory = File.ReadAllText(RepositoryPath("specs/091-groundwork-diagnostics-persistence/oracle-inventory.md")); + var requiredEvidence = new[] + { + "src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Storage/EfCoreStructuredLogStore.cs", + "tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStoreTests.cs", + "tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStorePruneRetryTests.cs", + "src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/EFCoreStructuredLogsPersistenceFeatureBase.cs", + "tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/SqliteStructuredLogsPersistenceFeatureTests.cs", + "src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Sqlite/Migrations/20260618004216_Initial.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Storage/EfCoreOpenTelemetryStore.cs", + "tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/EfCoreOpenTelemetryStoreTests.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/EFCoreOpenTelemetryPersistenceFeatureBase.cs", + "tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/SqliteOpenTelemetryPersistenceFeatureTests.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Sqlite/Migrations/20260623005000_Initial.cs", + "src/Elsa/Persistence/EFCore/Storage/ChannelDrainingStoreBase.cs", + "tests/Elsa/Diagnostics/StructuredLogs/Persistence/Tests/EfCoreStructuredLogStoreResilienceTests.cs", + "tests/Elsa/Diagnostics/OpenTelemetry/Persistence/Tests/EfCoreOpenTelemetryStoreResilienceTests.cs", + "src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Tasks/StartStructuredLogDrainingStartupTask.cs", + "src/Elsa/Diagnostics/StructuredLogs/Persistence/EFCore/Tasks/StopStructuredLogDrainingShellTerminator.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Tasks/StartOpenTelemetryDrainingStartupTask.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Persistence/EFCore/Tasks/StopOpenTelemetryDrainingShellTerminator.cs", + "src/Elsa/Diagnostics/StructuredLogs/Live/InMemoryStructuredLogLiveFeed.cs", + "src/Elsa/Diagnostics/OpenTelemetry/Providers/InMemory/InMemoryOpenTelemetryLiveFeed.cs" + }; + + Assert.All(requiredEvidence, path => Assert.Contains($"`{path}`", inventory, StringComparison.Ordinal)); + } + + private static string RepositoryPath(string relativePath, [CallerFilePath] string sourceFile = "") => + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(sourceFile)!, "../../../../..", relativePath)); +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLifecycleTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLifecycleTests.cs new file mode 100644 index 000000000..0655155e6 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLifecycleTests.cs @@ -0,0 +1,272 @@ +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDrainLifecycleTests : DiagnosticsDrainTestBase +{ + [Fact] + public async Task Accepted_item_completes_only_after_authoritative_commit() + { + var target = new DiagnosticsFailureTarget { CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously) }; + var drain = Fixture.Create(target); + drain.Start(); + + Assert.True(drain.TryEnqueue(42, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(acknowledgement.IsCompleted); + + target.CommitRelease.SetResult(); + Assert.Equal(42, await acknowledgement); + Assert.Equal(DiagnosticsDrainState.Stopped, (await drain.StopAsync()).State); + } + + [Fact] + public async Task Acknowledgement_loss_retries_the_same_batch_identity_and_returns_original_result() + { + var target = new DiagnosticsFailureTarget { LoseFirstAcknowledgement = true }; + var drain = Fixture.Create(target); + drain.Start(); + var acknowledgement = drain.EnqueueAsync(7).AsTask(); + + await drain.StopAsync(); + + Assert.Equal(7, await acknowledgement); + Assert.Equal(2, target.Attempts); + Assert.Single(target.AttemptedBatchIds.Distinct()); + Assert.Single(target.State.PersistedItems); + } + + [Fact] + public async Task Retry_delay_is_capped_while_the_same_batch_continues_to_retry() + { + var target = new DiagnosticsFailureTarget + { + Failure = (_, attempt) => attempt < 3 + ? new DiagnosticsOperationalException("provider unavailable") + : null + }; + var drain = Fixture.Create( + target, + maxAttempts: 3, + baseRetryDelay: TimeSpan.FromMilliseconds(1), + maxRetryDelay: TimeSpan.FromMilliseconds(1)); + var acknowledgement = drain.EnqueueAsync(8).AsTask(); + + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(8, await acknowledgement); + Assert.Equal(3, target.Attempts); + Assert.Single(target.AttemptedBatchIds.Distinct()); + } + + [Fact] + public async Task Caller_cancellation_does_not_abandon_an_accepted_acknowledgement() + { + var target = new DiagnosticsFailureTarget { CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously) }; + var drain = Fixture.Create(target); + drain.Start(); + using var cancellation = new CancellationTokenSource(); + var callerWait = drain.EnqueueAsync(9, cancellation.Token).AsTask(); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => callerWait); + + target.CommitRelease.SetResult(); + await drain.StopAsync(); + Assert.Equal(new[] { 9 }, target.State.PersistedItems); + } + + [Fact] + public async Task Writes_after_closure_are_rejected_with_an_explicit_loss_reason() + { + var counters = new DiagnosticsPersistenceCounters(); + var drain = Fixture.Create(new DiagnosticsFailureTarget(), counters); + drain.Start(); + await drain.StopAsync(); + + Assert.False(drain.TryEnqueue(1, out var acknowledgement)); + var exception = await Assert.ThrowsAsync(() => acknowledgement); + Assert.Equal(DiagnosticsPersistenceLossReason.WriteAfterClosure, exception.Reason); + Assert.Equal(1, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.WriteAfterClosure]); + } + + [Fact] + public async Task Async_disposal_gracefully_drains_and_is_idempotent() + { + var target = new DiagnosticsFailureTarget(); + var drain = Fixture.Create(target); + var acknowledgement = drain.EnqueueAsync(11).AsTask(); + + await drain.DisposeAsync(); + await drain.DisposeAsync(); + + Assert.Equal(11, await acknowledgement); + Assert.Equal(DiagnosticsDrainState.Stopped, drain.State); + } + + [Fact] + public async Task Synchronous_disposal_settles_queued_acknowledgements_and_is_idempotent() + { + var counters = new DiagnosticsPersistenceCounters(); + var drain = Fixture.Create(new DiagnosticsFailureTarget(), counters); + Assert.True(drain.TryEnqueue(12, out var acknowledgement)); + + drain.Dispose(); + drain.Dispose(); + + var exception = await Assert.ThrowsAsync(() => acknowledgement); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, exception.Reason); + Assert.Equal(DiagnosticsDrainState.TimedOut, drain.State); + Assert.Equal(1, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.ShutdownTimeout]); + } + + [Fact] + public async Task Graceful_stop_followed_by_synchronous_disposal_preserves_stopped() + { + var drain = Fixture.Create(new DiagnosticsFailureTarget()); + drain.Start(); + var result = await drain.StopAsync(); + + drain.Dispose(); + + Assert.True(result.Drained); + Assert.Equal(DiagnosticsDrainState.Stopped, drain.State); + } + + [Fact] + public async Task Synchronous_then_asynchronous_disposal_is_safe_and_preserves_timeout() + { + var drain = Fixture.Create(new DiagnosticsFailureTarget()); + Assert.True(drain.TryEnqueue(13, out var acknowledgement)); + + drain.Dispose(); + await drain.DisposeAsync(); + + Assert.Equal(DiagnosticsDrainState.TimedOut, drain.State); + var exception = await Assert.ThrowsAsync(() => acknowledgement); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, exception.Reason); + } + + [Fact] + public async Task Concurrent_stop_and_synchronous_disposal_cannot_overwrite_timeout_with_stopped() + { + var target = new DiagnosticsFailureTarget + { + CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously), + IgnoreCancellation = true + }; + var drain = Fixture.Create(target); + drain.Start(); + Assert.True(drain.TryEnqueue(14, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var stop = drain.StopAsync(); + Assert.False(drain.TryEnqueue(15, out var rejected)); + + drain.Dispose(); + target.CommitRelease.SetResult(); + var result = await stop; + + Assert.False(result.Drained); + Assert.Equal(DiagnosticsDrainState.TimedOut, result.State); + Assert.Equal(DiagnosticsDrainState.TimedOut, drain.State); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + Assert.Equal(DiagnosticsPersistenceLossReason.WriteAfterClosure, + (await Assert.ThrowsAsync(() => rejected)).Reason); + await drain.DisposeAsync(); + } + + [Fact] + public async Task Repeated_start_is_idempotent_while_running_but_late_start_is_rejected() + { + var drain = Fixture.Create(new DiagnosticsFailureTarget()); + drain.Start(); + drain.Start(); + await drain.StopAsync(); + + Assert.Throws(drain.Start); + } + + [Fact] + public async Task Cancelled_stop_wait_can_reenter_the_same_terminal_transition() + { + var target = new DiagnosticsFailureTarget { CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously) }; + var drain = Fixture.Create(target); + drain.Start(); + Assert.True(drain.TryEnqueue(16, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + using var cancellation = new CancellationTokenSource(); + var firstWait = drain.StopAsync(cancellation.Token); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => firstWait); + target.CommitRelease.SetResult(); + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(16, await acknowledgement); + Assert.Equal(DiagnosticsDrainState.Stopped, result.State); + } + + [Fact] + public async Task Concurrent_synchronous_and_asynchronous_disposal_share_one_timeout_outcome() + { + var target = new DiagnosticsFailureTarget + { + CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously), + IgnoreCancellation = true + }; + var drain = Fixture.Create(target); + drain.Start(); + Assert.True(drain.TryEnqueue(17, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + var asyncDisposals = Enumerable.Range(0, 4) + .Select(_ => Task.Run(async () => await drain.DisposeAsync())) + .ToArray(); + var syncDisposals = Enumerable.Range(0, 4) + .Select(_ => Task.Run(drain.Dispose)) + .ToArray(); + await Task.WhenAll(syncDisposals); + target.CommitRelease.SetResult(); + await Task.WhenAll(asyncDisposals); + var stop = await drain.StopAsync(); + + Assert.False(stop.Drained); + Assert.Equal(DiagnosticsDrainState.TimedOut, stop.State); + Assert.Equal(DiagnosticsDrainState.TimedOut, drain.State); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + } + + [Fact] + public async Task Observer_failure_cannot_break_persistence_or_terminal_state() + { + var drain = Fixture.Create(new DiagnosticsFailureTarget(), new ThrowingObserver()); + drain.Start(); + var acknowledgement = drain.EnqueueAsync(18).AsTask(); + + var stop = await drain.StopAsync(); + + Assert.Equal(18, await acknowledgement); + Assert.True(stop.Drained); + Assert.Equal(DiagnosticsDrainState.Stopped, stop.State); + } + + private sealed class ThrowingObserver : IDiagnosticsPersistenceObserver + { + public void RecordState(DiagnosticsDrainState state) => throw new InvalidOperationException("observer unavailable"); + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) => + throw new InvalidOperationException("observer unavailable"); + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) => + throw new InvalidOperationException("observer unavailable"); + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) => + throw new InvalidOperationException("observer unavailable"); + } + +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLoadTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLoadTests.cs new file mode 100644 index 000000000..2579ab6ad --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainLoadTests.cs @@ -0,0 +1,73 @@ +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDrainLoadTests : DiagnosticsDrainTestBase +{ + [Fact] + public async Task Concurrent_producers_complete_every_accepted_acknowledgement() + { + var target = new DiagnosticsFailureTarget(); + var drain = Fixture.Create(target, queueCapacity: 512, batchSize: 25); + drain.Start(); + + var acknowledgements = Enumerable.Range(1, 200) + .AsParallel() + .Select(item => + { + Assert.True(drain.TryEnqueue(item, out var acknowledgement)); + return acknowledgement; + }) + .ToArray(); + + await drain.StopAsync(); + var results = await Task.WhenAll(acknowledgements); + Assert.Equal(Enumerable.Range(1, 200).Order(), results.Order()); + Assert.Equal(200, target.State.PersistedItems.Count); + } + + [Fact] + public async Task Overflow_sheds_the_oldest_item_and_settles_its_acknowledgement() + { + var counters = new DiagnosticsPersistenceCounters(); + var drain = Fixture.Create(new DiagnosticsFailureTarget(), counters, queueCapacity: 2); + Assert.True(drain.TryEnqueue(1, out var first)); + Assert.True(drain.TryEnqueue(2, out var second)); + Assert.True(drain.TryEnqueue(3, out var third)); + + var exception = await Assert.ThrowsAsync(() => first); + Assert.Equal(DiagnosticsPersistenceLossReason.QueueOverflow, exception.Reason); + drain.Start(); + await drain.StopAsync(); + Assert.Equal(new[] { 2, 3 }, await Task.WhenAll(second, third)); + Assert.Equal(1, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.QueueOverflow]); + } + + [Fact] + public async Task Retry_exhaustion_fails_one_batch_and_the_later_batch_recovers() + { + var counters = new DiagnosticsPersistenceCounters(); + var target = new DiagnosticsFailureTarget + { + Failure = (batch, _) => batch.Items[0] == 1 ? new DiagnosticsOperationalException("persistently unavailable") : null + }; + var drain = Fixture.Create(target, counters, batchSize: 1); + drain.Start(); + + var failed = drain.EnqueueAsync(1).AsTask(); + var exception = await Assert.ThrowsAsync(() => failed); + Assert.Equal(DiagnosticsPersistenceLossReason.RetryExhausted, exception.Reason); + var recovered = drain.EnqueueAsync(2).AsTask(); + + await drain.StopAsync(); + Assert.Equal(2, await recovered); + var snapshot = counters.Snapshot(); + Assert.Equal(2, snapshot.CommitRetries); + Assert.Equal(1, snapshot.CommitFailures); + Assert.Equal(1, snapshot.Losses[DiagnosticsPersistenceLossReason.RetryExhausted]); + } + +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainShutdownTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainShutdownTests.cs new file mode 100644 index 000000000..655392df7 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainShutdownTests.cs @@ -0,0 +1,186 @@ +using System.Diagnostics; +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDrainShutdownTests : DiagnosticsDrainTestBase +{ + [Fact] + public async Task Graceful_stop_drains_tail_applies_final_retention_and_completes_all_acks() + { + var counters = new DiagnosticsPersistenceCounters(); + var target = new DiagnosticsFailureTarget { RetentionDeleted = 3 }; + var drain = Fixture.Create(target, counters); + drain.Start(); + var acknowledgements = Enumerable.Range(1, 8).Select(x => drain.EnqueueAsync(x).AsTask()).ToArray(); + + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(DiagnosticsDrainState.Stopped, result.State); + Assert.Equal(Enumerable.Range(1, 8), await Task.WhenAll(acknowledgements)); + Assert.True(target.RetentionCalls >= 1); + Assert.Equal(3, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.DurableRetentionDeletion]); + } + + [Fact] + public async Task Timeout_is_bounded_and_completes_the_inflight_acknowledgement() + { + var target = new DiagnosticsFailureTarget + { + CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously), + IgnoreCancellation = true + }; + var drain = Fixture.Create(target, new DiagnosticsPersistenceCounters(), shutdownTimeout: TimeSpan.FromMilliseconds(50)); + drain.Start(); + Assert.True(drain.TryEnqueue(1, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var stopwatch = Stopwatch.StartNew(); + + var result = await drain.StopAsync(); + + stopwatch.Stop(); + Assert.False(result.Drained); + Assert.Equal(DiagnosticsDrainState.TimedOut, result.State); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1)); + var exception = await Assert.ThrowsAsync(() => acknowledgement); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, exception.Reason); + target.CommitRelease.SetResult(); + await drain.DisposeAsync(); + } + + [Fact] + public async Task Shutdown_timeout_is_not_extended_by_a_blocking_cancellation_callback() + { + using var callbackEntered = new ManualResetEventSlim(); + using var callbackRelease = new ManualResetEventSlim(); + var target = new DiagnosticsFailureTarget + { + CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously), + IgnoreCancellation = true, + CancellationCallback = () => + { + callbackEntered.Set(); + callbackRelease.Wait(); + } + }; + var drain = Fixture.Create(target, shutdownTimeout: TimeSpan.FromMilliseconds(50)); + drain.Start(); + Assert.True(drain.TryEnqueue(1, out var acknowledgement)); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var stop = drain.StopAsync(); + + try + { + Assert.True(callbackEntered.Wait(TimeSpan.FromSeconds(2))); + Assert.Same(stop, await Task.WhenAny(stop, Task.Delay(TimeSpan.FromMilliseconds(500)))); + var result = await stop; + + Assert.Equal(DiagnosticsDrainState.TimedOut, result.State); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + } + finally + { + callbackRelease.Set(); + target.CommitRelease.TrySetResult(); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + } + } + + [Fact] + public async Task Unexpected_drain_loop_fault_uses_the_same_terminal_fallback_and_settles_acks() + { + var target = new DiagnosticsFailureTarget + { + Failure = (_, _) => new DiagnosticsOperationalException("provider unavailable") + }; + var drain = Fixture.Create( + target, + maxAttempts: 2, + baseRetryDelay: TimeSpan.MaxValue, + maxRetryDelay: TimeSpan.MaxValue); + Assert.True(drain.TryEnqueue(1, out var acknowledgement)); + + var result = await drain.StopAsync(); + + Assert.False(result.Drained); + Assert.Equal(DiagnosticsDrainState.TimedOut, result.State); + Assert.Equal(DiagnosticsPersistenceLossReason.ShutdownTimeout, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + } + + [Fact] + public async Task Timeout_and_overflow_leave_no_accepted_acknowledgement_incomplete() + { + var target = new DiagnosticsFailureTarget + { + CommitRelease = new(TaskCreationOptions.RunContinuationsAsynchronously), + IgnoreCancellation = true + }; + var drain = Fixture.Create(target, new DiagnosticsPersistenceCounters(), queueCapacity: 2, batchSize: 1, + shutdownTimeout: TimeSpan.FromMilliseconds(50)); + drain.Start(); + var acknowledgements = new List>(); + Assert.True(drain.TryEnqueue(1, out var first)); + acknowledgements.Add(first); + await target.CommitEntered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + foreach (var item in Enumerable.Range(2, 4)) + { + Assert.True(drain.TryEnqueue(item, out var acknowledgement)); + acknowledgements.Add(acknowledgement); + } + + await drain.StopAsync(); + + Assert.All(acknowledgements, acknowledgement => Assert.True(acknowledgement.IsCompleted)); + target.CommitRelease.SetResult(); + await drain.DisposeAsync(); + } + + [Fact] + public async Task Transient_final_retention_failure_retries_and_recovers() + { + var counters = new DiagnosticsPersistenceCounters(); + var target = new DiagnosticsFailureTarget + { + RetentionDeleted = 2, + RetentionFailure = call => call < 3 ? new DiagnosticsOperationalException("retention unavailable") : null + }; + var drain = Fixture.Create(target, counters); + var acknowledgement = drain.EnqueueAsync(1).AsTask(); + + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(1, await acknowledgement); + Assert.Equal(3, target.RetentionCalls); + Assert.Equal(2, counters.Snapshot().RetentionRetries); + Assert.Equal(2, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.DurableRetentionDeletion]); + } + + [Fact] + public async Task Exhausted_final_retention_failure_is_classified_without_invalidating_commits() + { + var counters = new DiagnosticsPersistenceCounters(); + var target = new DiagnosticsFailureTarget + { + RetentionFailure = _ => new DiagnosticsOperationalException("retention unavailable") + }; + var drain = Fixture.Create(target, counters); + var acknowledgement = drain.EnqueueAsync(1).AsTask(); + + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(1, await acknowledgement); + Assert.Equal(3, target.RetentionCalls); + var snapshot = counters.Snapshot(); + Assert.Equal(2, snapshot.RetentionRetries); + Assert.Equal(1, snapshot.RetentionFailures); + } + +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainValidationTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainValidationTests.cs new file mode 100644 index 000000000..373250016 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDrainValidationTests.cs @@ -0,0 +1,111 @@ +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDrainValidationTests : DiagnosticsDrainTestBase +{ + [Fact] + public async Task Incomplete_commit_results_exhaust_retries_and_fail_every_acknowledgement() + { + var target = new ScriptedTarget + { + Commit = batch => new([batch.Items[0]], batch.Items.Count) + }; + var drain = CreateDrain(target); + var first = drain.EnqueueAsync(1).AsTask(); + var second = drain.EnqueueAsync(2).AsTask(); + + await drain.StopAsync(); + + Assert.Equal(DiagnosticsPersistenceLossReason.RetryExhausted, + (await Assert.ThrowsAsync(() => first)).Reason); + Assert.Equal(DiagnosticsPersistenceLossReason.RetryExhausted, + (await Assert.ThrowsAsync(() => second)).Reason); + Assert.Equal(2, target.CommitCalls); + } + + [Fact] + public async Task Null_commit_results_are_invalid_and_fail_with_retry_exhaustion() + { + var target = new ScriptedTarget { Commit = _ => new(null!, 0) }; + var drain = CreateDrain(target); + var acknowledgement = drain.EnqueueAsync(1).AsTask(); + + await drain.StopAsync(); + + Assert.Equal(DiagnosticsPersistenceLossReason.RetryExhausted, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + Assert.Equal(2, target.CommitCalls); + } + + [Fact] + public async Task Negative_commit_retention_units_are_invalid_and_fail_with_retry_exhaustion() + { + var target = new ScriptedTarget { Commit = batch => new(batch.Items.ToArray(), -1) }; + var drain = CreateDrain(target); + var acknowledgement = drain.EnqueueAsync(1).AsTask(); + + await drain.StopAsync(); + + Assert.Equal(DiagnosticsPersistenceLossReason.RetryExhausted, + (await Assert.ThrowsAsync(() => acknowledgement)).Reason); + Assert.Equal(2, target.CommitCalls); + } + + [Fact] + public async Task Negative_retention_result_exhausts_retries_and_is_classified() + { + var counters = new DiagnosticsPersistenceCounters(); + var target = new ScriptedTarget { Retention = () => -1 }; + var drain = CreateDrain(target, counters); + var acknowledgement = drain.EnqueueAsync(1).AsTask(); + + var result = await drain.StopAsync(); + + Assert.True(result.Drained); + Assert.Equal(1, await acknowledgement); + Assert.Equal(2, target.RetentionCalls); + Assert.Equal(1, counters.Snapshot().RetentionRetries); + Assert.Equal(1, counters.Snapshot().RetentionFailures); + } + + private DiagnosticsDrain CreateDrain( + ScriptedTarget target, + IDiagnosticsPersistenceObserver? observer = null) => + Fixture.Create( + target, + observer, + maxAttempts: 2, + baseRetryDelay: TimeSpan.Zero, + maxRetryDelay: TimeSpan.Zero); + + private sealed class ScriptedTarget : IDiagnosticsDrainTarget + { + private int _commitCalls; + private int _retentionCalls; + + public Func, DiagnosticsDrainCommit> Commit { get; init; } = + batch => new(batch.Items.ToArray(), batch.Items.Count); + + public Func Retention { get; init; } = () => 0; + public int CommitCalls => Volatile.Read(ref _commitCalls); + public int RetentionCalls => Volatile.Read(ref _retentionCalls); + + public ValueTask> CommitAsync( + DiagnosticsDrainBatch batch, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _commitCalls); + return ValueTask.FromResult(Commit(batch)); + } + + public ValueTask ApplyRetentionAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _retentionCalls); + return ValueTask.FromResult(Retention()); + } + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDurableOperationConformanceTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDurableOperationConformanceTests.cs new file mode 100644 index 000000000..6284fff09 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsDurableOperationConformanceTests.cs @@ -0,0 +1,210 @@ +using Groundwork.DiagnosticRecords; +using Groundwork.Sqlite.DiagnosticRecords; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsDurableOperationConformanceTests : IAsyncLifetime +{ + private static readonly DiagnosticStorageScope Scope = new("tenant-a", "diagnostics-a"); + private static readonly DiagnosticStreamId Stream = new("diagnostics-conformance"); + private static readonly DiagnosticRecordStreamDefinition Definition = new( + Stream, + SchemaVersion: 1, + LogicalStorageName: "elsa_diagnostics_conformance", + Fields: + [ + new( + "sequence", + DiagnosticFieldType.Int64, + DiagnosticFieldCardinality.Scalar, + new HashSet { DiagnosticPredicateOperator.Equal }, + IsRequired: true) + ], + Limits: new( + MaxBatchRecords: 2, + MaxPayloadBytes: 64, + MaxRecordIdBytes: 64, + MaxFieldsPerRecord: 1, + MaxQueryLimit: 10, + MaxPredicateNodes: 4), + MaxOperationClockSkew: TimeSpan.FromMinutes(5), + AppendIdempotencyWindow: TimeSpan.FromHours(1), + TrimIdempotencyWindow: TimeSpan.FromHours(1), + LogicalHighWaterField: "sequence"); + + private readonly string _databasePath = Path.Combine( + Path.GetTempPath(), + $"elsa-diagnostics-operations-{Guid.NewGuid():N}.db"); + + [Fact] + public async Task Append_replay_is_idempotent_and_conflicting_identity_reuse_does_not_mutate() + { + var store = await CreateStoreAsync(); + var request = Batch("idempotent", Record("record-1", 1)); + + var committed = await store.AppendAsync(request); + var replayed = await store.AppendAsync(request); + var conflicting = DiagnosticRecordBatch.Create( + Scope, + Stream, + request.OperationId, + [Record("record-2", 2)]); + await Assert.ThrowsAsync(async () => + await store.AppendAsync(conflicting)); + + var retained = await store.QueryAsync(new(Scope, Stream, 10)); + Assert.Equal(DiagnosticAppendStatus.Committed, committed.Status); + Assert.Equal(DiagnosticAppendStatus.Replayed, replayed.Status); + Assert.Equal(committed.Records.Select(x => x.Cursor), replayed.Records.Select(x => x.Cursor)); + Assert.Equal("record-1", Assert.Single(retained.Records).RecordId); + } + + [Fact] + public async Task Acknowledgement_loss_retry_returns_the_original_outcome_without_duplication() + { + var provider = await CreateStoreAsync(); + var store = new AcknowledgementLosingStore(provider); + var request = Batch("acknowledgement-loss", Record("record-1", 1)); + + await Assert.ThrowsAsync(async () => + await store.AppendAsync(request)); + var replayed = await store.AppendAsync(request); + var retained = await provider.QueryAsync(new(Scope, Stream, 10)); + + Assert.Equal(2, store.AppendCalls); + Assert.Equal(DiagnosticAppendStatus.Replayed, replayed.Status); + Assert.Equal("record-1", Assert.Single(retained.Records).RecordId); + } + + [Fact] + public async Task Cancellation_before_or_during_provider_work_does_not_mutate() + { + var provider = await CreateStoreAsync(); + using (var cancelled = new CancellationTokenSource()) + { + cancelled.Cancel(); + await Assert.ThrowsAnyAsync(async () => + await provider.AppendAsync(Batch("cancelled-before", Record("record-before", 1)), cancelled.Token)); + } + + var blocking = new CancellationBoundaryStore(provider); + using var during = new CancellationTokenSource(); + var append = blocking.AppendAsync( + Batch("cancelled-during", Record("record-during", 2)), + during.Token).AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(10)); + await during.CancelAsync(); + await Assert.ThrowsAnyAsync(() => append); + + var inspection = await provider.InspectAsync(new(Scope, Stream)); + Assert.Equal(0, inspection.RetainedCount.Value); + } + + [Fact] + public async Task Concurrent_writers_commit_distinct_records_in_one_durable_order() + { + var first = await CreateStoreAsync(); + var second = await CreateStoreAsync(); + + var results = await Task.WhenAll( + first.AppendAsync(Batch("writer-a", Record("record-a", 1))).AsTask(), + second.AppendAsync(Batch("writer-b", Record("record-b", 2))).AsTask()); + var retained = await first.QueryAsync(new(Scope, Stream, 10)); + + Assert.Equal(2, results.SelectMany(x => x.Records).Select(x => x.Cursor).Distinct().Count()); + Assert.Equal(["record-a", "record-b"], retained.Records.Select(x => x.RecordId).Order()); + Assert.Equal(["1", "2"], retained.Records.Select(x => x.Cursor.Value)); + } + + [Fact] + public async Task Malformed_payload_and_oversized_batch_are_rejected_without_mutation() + { + var store = await CreateStoreAsync(); + var malformed = Batch("malformed", Record("record-malformed", 1) with { Payload = "not-json" }); + var oversized = Batch( + "oversized", + Record("record-1", 1), + Record("record-2", 2), + Record("record-3", 3)); + + var malformedError = await Assert.ThrowsAsync(async () => + await store.AppendAsync(malformed)); + var oversizedError = await Assert.ThrowsAsync(async () => + await store.AppendAsync(oversized)); + var inspection = await store.InspectAsync(new(Scope, Stream)); + + Assert.Contains(malformedError.Errors, x => x.Code == "append.payload.invalid_json"); + Assert.Contains(oversizedError.Errors, x => x.Code == "append.batch.too_large"); + Assert.Equal(0, inspection.RetainedCount.Value); + } + + private Task CreateStoreAsync() => + SqliteDiagnosticRecordStoreFactory.CreateAsync( + $"Data Source={_databasePath}", + Definition); + + private static DiagnosticRecordBatch Batch( + string operationNonce, + params DiagnosticRecordInput[] records) => + DiagnosticRecordBatch.Create( + Scope, + Stream, + new(DateTimeOffset.UtcNow, operationNonce), + records); + + private static DiagnosticRecordInput Record(string id, long sequence) => new( + id, + DateTimeOffset.UnixEpoch.AddSeconds(sequence), + $"{{\"sequence\":{sequence}}}", + new Dictionary>(StringComparer.Ordinal) + { + ["sequence"] = [DiagnosticFieldValue.Int64(sequence)] + }); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + if (File.Exists(_databasePath)) + File.Delete(_databasePath); + return Task.CompletedTask; + } + + private sealed class AcknowledgementLosingStore(IDiagnosticRecordStore inner) : IDiagnosticRecordStore + { + private int _loseNext = 1; + public int AppendCalls { get; private set; } + public DiagnosticRecordStoreHandlers Handlers => inner.Handlers; + + public async ValueTask AppendAsync( + DiagnosticRecordBatch batch, + CancellationToken cancellationToken = default) + { + AppendCalls++; + var result = await inner.AppendAsync(batch, cancellationToken); + if (Interlocked.Exchange(ref _loseNext, 0) == 1) + throw new DiagnosticAcknowledgementLostException( + DiagnosticOperationKind.Append, + batch.Stream, + batch.OperationId); + return result; + } + } + + private sealed class CancellationBoundaryStore(IDiagnosticRecordStore inner) : IDiagnosticRecordStore + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task Entered => _entered.Task; + public DiagnosticRecordStoreHandlers Handlers => inner.Handlers; + + public async ValueTask AppendAsync( + DiagnosticRecordBatch batch, + CancellationToken cancellationToken = default) + { + _entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return await inner.AppendAsync(batch, cancellationToken); + } + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsFoundationFixtureTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsFoundationFixtureTests.cs new file mode 100644 index 000000000..3b2c621b8 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsFoundationFixtureTests.cs @@ -0,0 +1,53 @@ +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsFoundationFixtureTests +{ + public static TheoryData Providers => new() + { + { DiagnosticsProviderKind.Sqlite, DiagnosticsProviderCapability.Relational }, + { DiagnosticsProviderKind.SqlServer, DiagnosticsProviderCapability.Relational }, + { DiagnosticsProviderKind.PostgreSql, DiagnosticsProviderCapability.Relational }, + { DiagnosticsProviderKind.MongoDb, DiagnosticsProviderCapability.Document } + }; + + [Theory] + [MemberData(nameof(Providers))] + public void Provider_assertions_apply_the_same_readiness_contract( + DiagnosticsProviderKind provider, + DiagnosticsProviderCapability storageCapability) + { + var capabilities = new HashSet + { + storageCapability, + DiagnosticsProviderCapability.Transactions, + DiagnosticsProviderCapability.BoundedQueries, + DiagnosticsProviderCapability.ExecutionPlans + }; + var lease = new DiagnosticsProviderLease(provider, "provider-connection", "isolated-database", capabilities); + + DiagnosticsProviderAssertions.IsReady( + lease, + storageCapability, + DiagnosticsProviderCapability.Transactions, + DiagnosticsProviderCapability.BoundedQueries, + DiagnosticsProviderCapability.ExecutionPlans); + } + + [Fact] + public async Task Restart_double_replays_a_committed_batch_without_duplicate_mutation() + { + var target = new DiagnosticsFailureTarget { LoseFirstAcknowledgement = true }; + var batch = new DiagnosticsDrainBatch(DiagnosticsDrainBatchId.New(), [1, 2]); + await Assert.ThrowsAsync(() => target.CommitAsync(batch).AsTask()); + + var restarted = target.Restart(); + var replay = await restarted.CommitAsync(batch); + + Assert.Equal(new[] { 1, 2 }, replay.Results); + Assert.Equal(new[] { 1, 2 }, target.State.PersistedItems); + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceArchitectureTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceArchitectureTests.cs new file mode 100644 index 000000000..a2c1ea62d --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceArchitectureTests.cs @@ -0,0 +1,308 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Contracts; +using Elsa.Diagnostics.Persistence.Extensions; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.StructuredLogs.Core.Contracts; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsPersistenceArchitectureTests +{ + [Fact] + public void Diagnostics_core_and_shared_lifecycle_have_no_Groundwork_references() + { + var assemblies = new[] + { + typeof(IStructuredLogStore).Assembly, + typeof(IOpenTelemetryStore).Assembly, + typeof(DiagnosticsPersistenceRegistration).Assembly + }; + + foreach (var assembly in assemblies) + Assert.DoesNotContain(assembly.GetReferencedAssemblies(), reference => reference.Name?.StartsWith("Groundwork", StringComparison.Ordinal) == true); + } + + [Fact] + public void Replacement_helper_leaves_exactly_one_store_and_one_shared_instance() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + + services.ReplaceDiagnosticsStore(); + + Assert.Single(services, descriptor => descriptor.ServiceType == typeof(ITestStore)); + using var provider = services.BuildServiceProvider(); + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + } + + [Fact] + public void Two_explicit_store_replacements_are_rejected_instead_of_last_write_wins() + { + var services = new ServiceCollection(); + services.ReplaceDiagnosticsStore(); + + var exception = Assert.Throws(() => + services.ReplaceDiagnosticsStore()); + + Assert.Contains(typeof(ITestStore).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(FirstStore).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondStore).FullName!, exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Default_and_explicit_registration_have_order_independent_selection_semantics() + { + var defaultThenExplicit = new ServiceCollection(); + defaultThenExplicit.AddDefaultDiagnosticsStore(); + defaultThenExplicit.ReplaceDiagnosticsStore(); + using var firstProvider = defaultThenExplicit.BuildServiceProvider(); + + var explicitThenDefault = new ServiceCollection(); + explicitThenDefault.ReplaceDiagnosticsStore(); + explicitThenDefault.AddDefaultDiagnosticsStore(); + using var secondProvider = explicitThenDefault.BuildServiceProvider(); + + Assert.IsType(firstProvider.GetRequiredService()); + Assert.IsType(secondProvider.GetRequiredService()); + Assert.Single(defaultThenExplicit, descriptor => descriptor.ServiceType == typeof(ITestStore)); + Assert.Single(explicitThenDefault, descriptor => descriptor.ServiceType == typeof(ITestStore)); + } + + [Fact] + public void Repeated_default_registration_and_preexisting_contract_remain_non_overriding() + { + var repeatedDefault = new ServiceCollection(); + repeatedDefault.AddDefaultDiagnosticsStore(); + repeatedDefault.AddDefaultDiagnosticsStore(); + using var firstProvider = repeatedDefault.BuildServiceProvider(); + + var preexisting = new ServiceCollection(); + preexisting.AddSingleton(); + preexisting.AddDefaultDiagnosticsStore(); + using var secondProvider = preexisting.BuildServiceProvider(); + + Assert.IsType(firstProvider.GetRequiredService()); + Assert.IsType(secondProvider.GetRequiredService()); + Assert.Single(repeatedDefault, descriptor => descriptor.ServiceType == typeof(ITestStore)); + Assert.Single(preexisting, descriptor => descriptor.ServiceType == typeof(ITestStore)); + } + + [Fact] + public void Diagnostics_stores_default_to_scoped_and_share_one_instance_within_each_scope() + { + var services = new ServiceCollection(); + services.AddDefaultDiagnosticsStore(); + + Assert.Equal(ServiceLifetime.Scoped, + Assert.Single(services, descriptor => descriptor.ServiceType == typeof(ITestStore)).Lifetime); + Assert.Equal(ServiceLifetime.Scoped, + Assert.Single(services, descriptor => descriptor.ServiceType == typeof(FirstStore)).Lifetime); + + using var provider = services.BuildServiceProvider(new ServiceProviderOptions { ValidateScopes = true }); + using var firstScope = provider.CreateScope(); + using var secondScope = provider.CreateScope(); + Assert.Same( + firstScope.ServiceProvider.GetRequiredService(), + firstScope.ServiceProvider.GetRequiredService()); + Assert.NotSame( + firstScope.ServiceProvider.GetRequiredService(), + secondScope.ServiceProvider.GetRequiredService()); + } + + [Theory] + [InlineData(nameof(DiagnosticsPersistenceRegistration.AddDefaultDiagnosticsStore))] + [InlineData(nameof(DiagnosticsPersistenceRegistration.ReplaceDiagnosticsStore))] + public void Store_registration_allows_an_explicit_documented_lifetime(string methodName) + { + var method = typeof(DiagnosticsPersistenceRegistration) + .GetMethods() + .Single(candidate => candidate.Name == methodName); + + Assert.Contains(method.GetParameters(), parameter => parameter.ParameterType == typeof(ServiceLifetime)); + } + + [Fact] + public void Explicit_singleton_store_lifetime_is_preserved_for_default_and_replacement_registrations() + { + var defaultServices = new ServiceCollection(); + defaultServices.AddDefaultDiagnosticsStore(ServiceLifetime.Singleton); + var replacementServices = new ServiceCollection(); + replacementServices.ReplaceDiagnosticsStore(ServiceLifetime.Singleton); + + Assert.All( + defaultServices.Where(descriptor => + descriptor.ServiceType == typeof(ITestStore) || descriptor.ServiceType == typeof(FirstStore)), + descriptor => Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime)); + Assert.All( + replacementServices.Where(descriptor => + descriptor.ServiceType == typeof(ITestStore) || descriptor.ServiceType == typeof(ReplacementStore)), + descriptor => Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime)); + } + + [Fact] + public void Invalid_store_lifetime_is_rejected_by_default_and_replacement_registration() + { + var invalid = (ServiceLifetime)int.MaxValue; + + Assert.Throws(() => + new ServiceCollection().AddDefaultDiagnosticsStore(invalid)); + Assert.Throws(() => + new ServiceCollection().ReplaceDiagnosticsStore(invalid)); + } + + [Fact] + public void Observer_replacement_is_order_independent_and_rejects_a_second_explicit_selection() + { + var defaultThenExplicit = new ServiceCollection(); + defaultThenExplicit.AddDiagnosticsPersistenceObservability(); + defaultThenExplicit.ReplaceDiagnosticsStore(ServiceLifetime.Singleton); + Assert.Throws(() => + defaultThenExplicit.ReplaceDiagnosticsStore(ServiceLifetime.Singleton)); + + var explicitThenDefault = new ServiceCollection(); + explicitThenDefault.ReplaceDiagnosticsStore(ServiceLifetime.Singleton); + explicitThenDefault.AddDiagnosticsPersistenceObservability(); + Assert.Throws(() => + explicitThenDefault.ReplaceDiagnosticsStore(ServiceLifetime.Singleton)); + + using var firstProvider = defaultThenExplicit.BuildServiceProvider(); + using var secondProvider = explicitThenDefault.BuildServiceProvider(); + Assert.IsType(firstProvider.GetRequiredService()); + Assert.IsType(secondProvider.GetRequiredService()); + } + + [Fact] + public void Observability_composition_rejects_untracked_explicit_observer_conflicts_before_or_after_default() + { + var beforeDefault = new ServiceCollection(); + beforeDefault.AddSingleton(); + beforeDefault.AddSingleton(); + Assert.Throws(beforeDefault.AddDiagnosticsPersistenceObservability); + + var afterDefault = new ServiceCollection(); + afterDefault.AddDiagnosticsPersistenceObservability(); + afterDefault.AddSingleton(); + afterDefault.AddSingleton(); + Assert.Throws(afterDefault.AddDiagnosticsPersistenceObservability); + } + + [Fact] + public void Startup_validation_rejects_observer_conflicts_registered_after_composition() + { + var services = new ServiceCollection(); + services.AddDiagnosticsPersistenceObservability(); + services.AddSingleton(); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + var exception = Assert.Throws( + provider.GetRequiredService().Validate); + + Assert.Contains(typeof(DiagnosticsPersistenceCounters).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(FirstObserver).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(nameof(DiagnosticsPersistenceRegistration.ReplaceDiagnosticsStore), exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Startup_validation_rejects_two_direct_observer_types_with_actionable_descriptions() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddDiagnosticsPersistenceObservability(); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + var exception = Assert.Throws( + provider.GetRequiredService().Validate); + + Assert.Contains(typeof(FirstObserver).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(nameof(DiagnosticsPersistenceRegistration.ReplaceDiagnosticsStore), exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Startup_validation_describes_factory_and_type_observer_conflicts() + { + var services = new ServiceCollection(); + services.AddSingleton(_ => new FirstObserver()); + services.AddDiagnosticsPersistenceObservability(); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + var exception = Assert.Throws( + provider.GetRequiredService().Validate); + + Assert.Contains("factory registration", exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Startup_validation_accepts_an_instance_observer_and_duplicate_composition_registers_one_validator() + { + var observer = new FirstObserver(); + var services = new ServiceCollection(); + services.AddSingleton(observer); + services.AddDiagnosticsPersistenceObservability(); + services.AddDiagnosticsPersistenceObservability(); + + Assert.Single(services, descriptor => + descriptor.ServiceType == typeof(DiagnosticsPersistenceObserverRegistrationValidator)); + Assert.Single(services, descriptor => + descriptor.ImplementationType?.Name == "DiagnosticsPersistenceObserverRegistrationOptionsAdapter"); + using var provider = services.BuildServiceProvider(); + provider.GetRequiredService().Validate(); + Assert.Same(observer, provider.GetRequiredService()); + } + + [Fact] + public void Observer_registration_validation_exports_only_the_constitution_mandated_implementation() + { + var assembly = typeof(DiagnosticsPersistenceObserverRegistrationValidator).Assembly; + var exportedRegistrationTypes = assembly.ExportedTypes + .Where(type => type.Name.StartsWith("DiagnosticsPersistenceObserverRegistration", StringComparison.Ordinal)) + .ToArray(); + + Assert.Equal([typeof(DiagnosticsPersistenceObserverRegistrationValidator)], exportedRegistrationTypes); + Assert.True(typeof(DiagnosticsPersistenceObserverRegistrationValidator).IsSealed); + var constructor = Assert.Single(typeof(DiagnosticsPersistenceObserverRegistrationValidator).GetConstructors()); + Assert.Equal([typeof(IServiceCollection)], constructor.GetParameters().Select(parameter => parameter.ParameterType)); + Assert.Contains(assembly.GetTypes(), type => + type.Name == "DiagnosticsPersistenceObserverRegistrationOptionsAdapter" && !type.IsPublic); + } + + [Fact] + public void Startup_validation_describes_instance_and_type_observer_conflicts() + { + var observer = new FirstObserver(); + var services = new ServiceCollection(); + services.AddSingleton(observer); + services.AddDiagnosticsPersistenceObservability(); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + var exception = Assert.Throws( + provider.GetRequiredService().Validate); + + Assert.Contains(observer.GetType().FullName!, exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, exception.Message, StringComparison.Ordinal); + } + + private interface ITestStore; + private sealed class FirstStore : ITestStore; + private sealed class SecondStore : ITestStore; + private sealed class ReplacementStore : ITestStore; + private sealed class FirstObserver : TestObserver; + private sealed class SecondObserver : TestObserver; + + private abstract class TestObserver : IDiagnosticsPersistenceObserver + { + public void RecordState(DiagnosticsDrainState state) { } + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) { } + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) { } + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) { } + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObservabilityTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObservabilityTests.cs new file mode 100644 index 000000000..f462bd29d --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObservabilityTests.cs @@ -0,0 +1,156 @@ +using Elsa.Diagnostics.OpenTelemetry.Core.Models; +using Elsa.Diagnostics.Persistence.Observability; +using Elsa.Diagnostics.StructuredLogs.Core.Models; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsPersistenceObservabilityTests +{ + [Fact] + public void Observer_contract_cannot_receive_payloads_or_high_cardinality_labels() + { + var parameterTypes = typeof(IDiagnosticsPersistenceObserver) + .GetMethods() + .SelectMany(method => method.GetParameters()) + .Select(parameter => parameter.ParameterType) + .ToArray(); + + Assert.DoesNotContain(typeof(string), parameterTypes); + Assert.DoesNotContain(typeof(object), parameterTypes); + Assert.All(parameterTypes, type => Assert.False(type.Namespace?.StartsWith("Elsa.Diagnostics.StructuredLogs", StringComparison.Ordinal) == true)); + Assert.All(parameterTypes, type => Assert.False(type.Namespace?.StartsWith("Elsa.Diagnostics.OpenTelemetry", StringComparison.Ordinal) == true)); + } + + [Fact] + public void Pull_counters_do_not_reference_recursive_logging_tracing_or_Groundwork_infrastructure() + { + var references = typeof(DiagnosticsPersistenceCounters).Assembly.GetReferencedAssemblies(); + Assert.DoesNotContain(references, x => x.Name == "Microsoft.Extensions.Logging.Abstractions"); + Assert.DoesNotContain(references, x => x.Name == "System.Diagnostics.DiagnosticSource"); + Assert.DoesNotContain(references, x => x.Name?.StartsWith("Groundwork", StringComparison.Ordinal) == true); + } + + [Fact] + public void Every_loss_reason_is_counted_independently() + { + var counters = new DiagnosticsPersistenceCounters(); + foreach (var reason in Enum.GetValues()) + counters.RecordLoss(reason, (int)reason + 1); + + var snapshot = counters.Snapshot(); + foreach (var reason in Enum.GetValues()) + Assert.Equal((int)reason + 1, snapshot.Losses[reason]); + } + + [Fact] + public void Commit_and_retention_retries_and_failures_remain_distinct() + { + var counters = new DiagnosticsPersistenceCounters(); + counters.RecordRetry(DiagnosticsPersistenceOperation.Commit, 1, 2); + counters.RecordRetry(DiagnosticsPersistenceOperation.Retention, 1, 2); + counters.RecordOperationFailure(DiagnosticsPersistenceOperation.Commit); + counters.RecordOperationFailure(DiagnosticsPersistenceOperation.Retention); + + var snapshot = counters.Snapshot(); + Assert.Equal(1, snapshot.CommitRetries); + Assert.Equal(1, snapshot.RetentionRetries); + Assert.Equal(1, snapshot.CommitFailures); + Assert.Equal(1, snapshot.RetentionFailures); + } + + [Fact] + public void Existing_live_feed_signals_are_classified_through_the_production_bridge() + { + var counters = new DiagnosticsPersistenceCounters(); + var bridge = new DiagnosticsSubscriberDeliveryLossBridge(counters); + var structuredLogs = bridge.CreateStructuredLogRecorder(); + + structuredLogs(new DroppedEntriesSignal(4, DateTimeOffset.UnixEpoch)); + bridge.RecordOpenTelemetry(new OpenTelemetryDroppedItemSummary(OpenTelemetrySignalType.Trace, 3, "SubscriberQueueFull")); + + Assert.Equal(7, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + [Fact] + public void Cumulative_structured_log_signals_record_only_deltas_even_when_repeated_or_concurrent() + { + var counters = new DiagnosticsPersistenceCounters(); + var bridge = new DiagnosticsSubscriberDeliveryLossBridge(counters); + var since = DateTimeOffset.UnixEpoch; + var structuredLogs = bridge.CreateStructuredLogRecorder(); + + Parallel.For(1, 101, count => structuredLogs(new DroppedEntriesSignal(count, since))); + structuredLogs(new DroppedEntriesSignal(100, since)); + + Assert.Equal(100, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + [Fact] + public void A_new_structured_log_subscription_starts_a_new_cumulative_epoch() + { + var counters = new DiagnosticsPersistenceCounters(); + var bridge = new DiagnosticsSubscriberDeliveryLossBridge(counters); + var firstSubscription = bridge.CreateStructuredLogRecorder(); + var restartedSubscription = bridge.CreateStructuredLogRecorder(); + + firstSubscription(new DroppedEntriesSignal(4, DateTimeOffset.UnixEpoch)); + restartedSubscription(new DroppedEntriesSignal(2, DateTimeOffset.UnixEpoch.AddSeconds(1))); + + Assert.Equal(6, counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + [Fact] + public void Subscriber_delivery_bridge_validates_dependencies_signals_and_counts_without_truncation() + { + Assert.Throws(() => new DiagnosticsSubscriberDeliveryLossBridge(null!)); + var counters = new DiagnosticsPersistenceCounters(); + var bridge = new DiagnosticsSubscriberDeliveryLossBridge(counters); + var structuredLogs = bridge.CreateStructuredLogRecorder(); + + Assert.Throws(() => structuredLogs(null!)); + Assert.Throws(() => bridge.RecordOpenTelemetry(null!)); + Assert.Throws(() => + structuredLogs(new DroppedEntriesSignal(0, DateTimeOffset.UnixEpoch))); + bridge.RecordOpenTelemetry(new OpenTelemetryDroppedItemSummary(OpenTelemetrySignalType.Log, 3_000_000_000, "SubscriberQueueFull")); + + Assert.Equal(3_000_000_000, + counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } + + [Fact] + public void Subscriber_delivery_observer_failure_cannot_break_live_feed_accounting_paths() + { + var bridge = new DiagnosticsSubscriberDeliveryLossBridge(new ThrowingObserver()); + var structuredLogs = bridge.CreateStructuredLogRecorder(); + + structuredLogs(new DroppedEntriesSignal(2, DateTimeOffset.UnixEpoch)); + bridge.RecordOpenTelemetry(new OpenTelemetryDroppedItemSummary( + OpenTelemetrySignalType.Trace, + 3, + "SubscriberQueueFull")); + } + + [Fact] + public void Observer_rejects_invalid_state_operation_reason_count_and_attempt_values() + { + var counters = new DiagnosticsPersistenceCounters(); + + Assert.Throws(() => counters.RecordState((DiagnosticsDrainState)int.MaxValue)); + Assert.Throws(() => counters.RecordRetry((DiagnosticsPersistenceOperation)int.MaxValue, 1, 2)); + Assert.Throws(() => counters.RecordOperationFailure((DiagnosticsPersistenceOperation)int.MaxValue)); + Assert.Throws(() => counters.RecordLoss((DiagnosticsPersistenceLossReason)int.MaxValue, 1)); + Assert.Throws(() => counters.RecordLoss(DiagnosticsPersistenceLossReason.QueueOverflow, 0)); + Assert.Throws(() => counters.RecordRetry(DiagnosticsPersistenceOperation.Commit, 0, 2)); + Assert.Throws(() => counters.RecordRetry(DiagnosticsPersistenceOperation.Commit, 3, 2)); + Assert.Throws(() => counters.RecordRetry(DiagnosticsPersistenceOperation.Commit, 1, 0)); + } + + private sealed class ThrowingObserver : IDiagnosticsPersistenceObserver + { + public void RecordState(DiagnosticsDrainState state) => throw new InvalidOperationException(); + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) => throw new InvalidOperationException(); + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) => throw new InvalidOperationException(); + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) => throw new InvalidOperationException(); + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObserverRegistrationValidatorTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObserverRegistrationValidatorTests.cs new file mode 100644 index 000000000..900525daf --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsPersistenceObserverRegistrationValidatorTests.cs @@ -0,0 +1,82 @@ +using Elsa.Diagnostics.Persistence.Extensions; +using Elsa.Diagnostics.Persistence.Observability; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public sealed class DiagnosticsPersistenceObserverRegistrationValidatorTests +{ + [Fact] + public void Validate_WhenOneObserverIsRegistered_Succeeds() + { + var services = new ServiceCollection(); + services.AddSingleton(); + var validator = new DiagnosticsPersistenceObserverRegistrationValidator(services); + + var result = validator.Validate(); + + Assert.True(result.Succeeded); + } + + [Fact] + public void Validate_WhenObserversConflict_FailsWithActionableDescriptions() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + var validator = new DiagnosticsPersistenceObserverRegistrationValidator(services); + + var result = validator.Validate(); + + Assert.True(result.Failed); + var message = Assert.Single(result.Failures); + Assert.Contains(typeof(FirstObserver).FullName!, message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, message, StringComparison.Ordinal); + Assert.Contains(nameof(DiagnosticsPersistenceRegistration.ReplaceDiagnosticsStore), message, StringComparison.Ordinal); + } + + [Fact] + public void Validate_WhenTrackedDefaultConflictsWithDirectObserver_DescribesSelectedImplementation() + { + var services = new ServiceCollection(); + services.AddDiagnosticsPersistenceObservability(); + services.AddSingleton(); + var validator = new DiagnosticsPersistenceObserverRegistrationValidator(services); + + var result = validator.Validate(); + + Assert.True(result.Failed); + var message = Assert.Single(result.Failures); + Assert.Contains(typeof(DiagnosticsPersistenceCounters).FullName!, message, StringComparison.Ordinal); + Assert.Contains(typeof(SecondObserver).FullName!, message, StringComparison.Ordinal); + } + + [Fact] + public void Validate_WhenInstanceAndFactoryObserversConflict_DescribesBothRegistrationForms() + { + var observer = new FirstObserver(); + var services = new ServiceCollection(); + services.AddSingleton(observer); + services.AddSingleton(_ => new SecondObserver()); + var validator = new DiagnosticsPersistenceObserverRegistrationValidator(services); + + var result = validator.Validate(); + + Assert.True(result.Failed); + var message = Assert.Single(result.Failures); + Assert.Contains(observer.GetType().FullName!, message, StringComparison.Ordinal); + Assert.Contains("factory registration", message, StringComparison.Ordinal); + } + + private sealed class FirstObserver : TestObserver; + private sealed class SecondObserver : TestObserver; + + private abstract class TestObserver : IDiagnosticsPersistenceObserver + { + public void RecordState(DiagnosticsDrainState state) { } + public void RecordRetry(DiagnosticsPersistenceOperation operation, int attempt, int maxAttempts) { } + public void RecordOperationFailure(DiagnosticsPersistenceOperation operation) { } + public void RecordLoss(DiagnosticsPersistenceLossReason reason, long count) { } + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderAssertions.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderAssertions.cs new file mode 100644 index 000000000..eff081b04 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderAssertions.cs @@ -0,0 +1,18 @@ +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +public static class DiagnosticsProviderAssertions +{ + public static void IsReady( + DiagnosticsProviderLease lease, + params DiagnosticsProviderCapability[] requiredCapabilities) + { + Assert.True(lease.IsReady, lease.Failure ?? $"{lease.Provider} is not ready."); + Assert.False(string.IsNullOrWhiteSpace(lease.ConnectionString)); + Assert.False(string.IsNullOrWhiteSpace(lease.DatabaseName)); + foreach (var capability in requiredCapabilities) + Assert.Contains(capability, lease.Capabilities); + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderLifecycleSmokeTests.cs b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderLifecycleSmokeTests.cs new file mode 100644 index 000000000..57827024d --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/DiagnosticsProviderLifecycleSmokeTests.cs @@ -0,0 +1,27 @@ +using Elsa.Diagnostics.Persistence.Tests.Fixtures; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests; + +[Collection(DiagnosticsProviderCollection.Name)] +public sealed class DiagnosticsProviderLifecycleSmokeTests(DiagnosticsProviderFixture fixture) +{ + [Fact] + public async Task Real_fixture_creates_isolated_ready_leases_for_all_four_providers() + { + foreach (var provider in Enum.GetValues()) + { + var lease = await fixture.CreateIsolatedAsync(provider); + var storageCapability = provider == DiagnosticsProviderKind.MongoDb + ? DiagnosticsProviderCapability.Document + : DiagnosticsProviderCapability.Relational; + + DiagnosticsProviderAssertions.IsReady( + lease, + storageCapability, + DiagnosticsProviderCapability.Transactions, + DiagnosticsProviderCapability.BoundedQueries, + DiagnosticsProviderCapability.ExecutionPlans); + } + } +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/Elsa.Diagnostics.Persistence.Tests.csproj b/tests/Elsa/Diagnostics/Persistence/Tests/Elsa.Diagnostics.Persistence.Tests.csproj new file mode 100644 index 000000000..006440691 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/Elsa.Diagnostics.Persistence.Tests.csproj @@ -0,0 +1,34 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsDrainTestFixture.cs b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsDrainTestFixture.cs new file mode 100644 index 000000000..d3e9813c0 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsDrainTestFixture.cs @@ -0,0 +1,49 @@ +using Elsa.Diagnostics.Persistence.Draining; +using Elsa.Diagnostics.Persistence.Observability; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests.Fixtures; + +public sealed class DiagnosticsDrainTestFixture : IAsyncDisposable +{ + private readonly List> _drains = []; + + public DiagnosticsDrain Create( + IDiagnosticsDrainTarget target, + IDiagnosticsPersistenceObserver? observer = null, + int queueCapacity = 16, + int batchSize = 4, + int retentionInterval = 100, + int maxAttempts = 3, + TimeSpan? baseRetryDelay = null, + TimeSpan? maxRetryDelay = null, + TimeSpan? shutdownTimeout = null) + { + var drain = new DiagnosticsDrain(target, new() + { + BatchSize = batchSize, + QueueCapacity = queueCapacity, + RetentionInterval = retentionInterval, + MaxAttempts = maxAttempts, + BaseRetryDelay = baseRetryDelay ?? TimeSpan.FromMilliseconds(1), + MaxRetryDelay = maxRetryDelay ?? TimeSpan.FromMilliseconds(2), + ShutdownTimeout = shutdownTimeout ?? TimeSpan.FromSeconds(2) + }, observer); + _drains.Add(drain); + return drain; + } + + public async ValueTask DisposeAsync() + { + foreach (var drain in Enumerable.Reverse(_drains)) + await drain.DisposeAsync(); + } +} + +public abstract class DiagnosticsDrainTestBase : IAsyncLifetime +{ + protected DiagnosticsDrainTestFixture Fixture { get; } = new(); + + public Task InitializeAsync() => Task.CompletedTask; + public Task DisposeAsync() => Fixture.DisposeAsync().AsTask(); +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsFailureFixtures.cs b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsFailureFixtures.cs new file mode 100644 index 000000000..5e0b094d5 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsFailureFixtures.cs @@ -0,0 +1,76 @@ +using System.Collections.Concurrent; +using Elsa.Diagnostics.Persistence.Draining; + +namespace Elsa.Diagnostics.Persistence.Tests.Fixtures; + +public sealed class DiagnosticsOperationalException(string message) : Exception(message); + +public sealed class DiagnosticsTargetState +{ + internal ConcurrentDictionary> Commits { get; } = new(); + public ConcurrentQueue PersistedItems { get; } = new(); +} + +/// Reusable acknowledgement-loss, cancellation, restart, and operational-failure target. +public sealed class DiagnosticsFailureTarget : IDiagnosticsDrainTarget +{ + private readonly DiagnosticsTargetState _state; + private readonly ConcurrentDictionary _lostAcknowledgements = new(); + private int _attempts; + private int _retentionCalls; + + public DiagnosticsFailureTarget(DiagnosticsTargetState? state = null) => _state = state ?? new(); + + public DiagnosticsTargetState State => _state; + public bool LoseFirstAcknowledgement { get; set; } + public bool IgnoreCancellation { get; set; } + public int RetentionDeleted { get; set; } + public Func, int, Exception?>? Failure { get; set; } + public Func? RetentionFailure { get; set; } + public TaskCompletionSource CommitEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource? CommitRelease { get; set; } + public Action? CancellationCallback { get; set; } + public int Attempts => Volatile.Read(ref _attempts); + public int RetentionCalls => Volatile.Read(ref _retentionCalls); + public ConcurrentQueue AttemptedBatchIds { get; } = new(); + + public async ValueTask> CommitAsync( + DiagnosticsDrainBatch batch, + CancellationToken cancellationToken = default) + { + using var cancellationRegistration = cancellationToken.Register(() => CancellationCallback?.Invoke()); + var attempt = Interlocked.Increment(ref _attempts); + AttemptedBatchIds.Enqueue(batch.Id); + CommitEntered.TrySetResult(); + if (CommitRelease is { } release) + { + if (IgnoreCancellation) + await release.Task; + else + await release.Task.WaitAsync(cancellationToken); + } + + if (Failure?.Invoke(batch, attempt) is { } failure) + throw failure; + + var commit = _state.Commits.GetOrAdd(batch.Id, _ => + { + foreach (var item in batch.Items) + _state.PersistedItems.Enqueue(item); + return new(batch.Items.ToArray(), batch.Items.Count); + }); + if (LoseFirstAcknowledgement && _lostAcknowledgements.TryAdd(batch.Id, 0)) + throw new DiagnosticsOperationalException("The commit succeeded but its acknowledgement was lost."); + return commit; + } + + public ValueTask ApplyRetentionAsync(CancellationToken cancellationToken = default) + { + var call = Interlocked.Increment(ref _retentionCalls); + if (RetentionFailure?.Invoke(call) is { } failure) + throw failure; + return ValueTask.FromResult(RetentionDeleted); + } + + public DiagnosticsFailureTarget Restart() => new(_state); +} diff --git a/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsProviderFixture.cs b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsProviderFixture.cs new file mode 100644 index 000000000..eae055ab0 --- /dev/null +++ b/tests/Elsa/Diagnostics/Persistence/Tests/Fixtures/DiagnosticsProviderFixture.cs @@ -0,0 +1,171 @@ +using System.Collections.Concurrent; +using Microsoft.Data.SqlClient; +using Microsoft.Data.Sqlite; +using MongoDB.Bson; +using MongoDB.Driver; +using Npgsql; +using Testcontainers.MongoDb; +using Testcontainers.MsSql; +using Testcontainers.PostgreSql; +using Xunit; + +namespace Elsa.Diagnostics.Persistence.Tests.Fixtures; + +public enum DiagnosticsProviderKind +{ + Sqlite, + SqlServer, + PostgreSql, + MongoDb +} + +public enum DiagnosticsProviderCapability +{ + Relational, + Document, + Transactions, + BoundedQueries, + ExecutionPlans +} + +public sealed record DiagnosticsProviderLease( + DiagnosticsProviderKind Provider, + string ConnectionString, + string DatabaseName, + IReadOnlySet Capabilities, + bool IsReady = true, + string? Failure = null); + +/// Owns real, isolated lifecycle resources for the shared four-provider diagnostics suite. +public sealed class DiagnosticsProviderFixture : IAsyncLifetime +{ + private readonly MsSqlContainer _sqlServer = new MsSqlBuilder("mcr.microsoft.com/mssql/server:2022-CU18-ubuntu-22.04").Build(); + private readonly PostgreSqlContainer _postgreSql = new PostgreSqlBuilder("postgres:16-alpine") + .WithDatabase("elsa") + .WithUsername("postgres") + .WithPassword("postgres") + .Build(); + private readonly MongoDbContainer _mongoDb = new MongoDbBuilder("mongo:7.0.37") + .WithReplicaSet("rs0") + .Build(); + private readonly ConcurrentBag _sqliteFiles = []; + private bool _sqlServerStarted; + private bool _postgreSqlStarted; + private bool _mongoDbStarted; + + public async Task InitializeAsync() + { + try + { + _sqlServerStarted = true; + await _sqlServer.StartAsync(); + _postgreSqlStarted = true; + await _postgreSql.StartAsync(); + _mongoDbStarted = true; + await _mongoDb.StartAsync(); + } + catch + { + await DisposeContainersAsync(); + throw; + } + } + + public async Task CreateIsolatedAsync( + DiagnosticsProviderKind provider, + CancellationToken cancellationToken = default) + { + var databaseName = $"elsa_diagnostics_{Guid.NewGuid():N}"; + return provider switch + { + DiagnosticsProviderKind.Sqlite => await CreateSqliteAsync(databaseName, cancellationToken), + DiagnosticsProviderKind.SqlServer => await CreateSqlServerAsync(databaseName, cancellationToken), + DiagnosticsProviderKind.PostgreSql => await CreatePostgreSqlAsync(databaseName, cancellationToken), + DiagnosticsProviderKind.MongoDb => await CreateMongoDbAsync(databaseName, cancellationToken), + _ => throw new ArgumentOutOfRangeException(nameof(provider), provider, null) + }; + } + + private async Task CreateSqliteAsync(string databaseName, CancellationToken cancellationToken) + { + var path = Path.Combine(Path.GetTempPath(), $"{databaseName}.db"); + var connectionString = new SqliteConnectionStringBuilder { DataSource = path }.ConnectionString; + await using var connection = new SqliteConnection(connectionString); + await connection.OpenAsync(cancellationToken); + _sqliteFiles.Add(path); + return Lease(DiagnosticsProviderKind.Sqlite, connectionString, databaseName, relational: true); + } + + private async Task CreateSqlServerAsync(string databaseName, CancellationToken cancellationToken) + { + await using var connection = new SqlConnection(_sqlServer.GetConnectionString()); + await connection.OpenAsync(cancellationToken); + await using var command = connection.CreateCommand(); + command.CommandText = $"CREATE DATABASE [{databaseName}]"; + await command.ExecuteNonQueryAsync(cancellationToken); + var connectionString = new SqlConnectionStringBuilder(_sqlServer.GetConnectionString()) { InitialCatalog = databaseName }.ConnectionString; + return Lease(DiagnosticsProviderKind.SqlServer, connectionString, databaseName, relational: true); + } + + private async Task CreatePostgreSqlAsync(string databaseName, CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(_postgreSql.GetConnectionString()); + await connection.OpenAsync(cancellationToken); + await using var command = connection.CreateCommand(); + command.CommandText = $"CREATE DATABASE \"{databaseName}\""; + await command.ExecuteNonQueryAsync(cancellationToken); + var connectionString = new NpgsqlConnectionStringBuilder(_postgreSql.GetConnectionString()) { Database = databaseName }.ConnectionString; + return Lease(DiagnosticsProviderKind.PostgreSql, connectionString, databaseName, relational: true); + } + + private async Task CreateMongoDbAsync(string databaseName, CancellationToken cancellationToken) + { + var client = new MongoClient(_mongoDb.GetConnectionString()); + await client.GetDatabase("admin").RunCommandAsync(new BsonDocument("ping", 1), cancellationToken: cancellationToken); + return Lease(DiagnosticsProviderKind.MongoDb, _mongoDb.GetConnectionString(), databaseName, relational: false); + } + + private static DiagnosticsProviderLease Lease( + DiagnosticsProviderKind provider, + string connectionString, + string databaseName, + bool relational) + { + var capabilities = new HashSet + { + DiagnosticsProviderCapability.Transactions, + DiagnosticsProviderCapability.BoundedQueries, + DiagnosticsProviderCapability.ExecutionPlans, + relational ? DiagnosticsProviderCapability.Relational : DiagnosticsProviderCapability.Document + }; + return new(provider, connectionString, databaseName, capabilities); + } + + public async Task DisposeAsync() + { + await DisposeContainersAsync(); + foreach (var path in _sqliteFiles) + File.Delete(path); + } + + private async Task DisposeContainersAsync() + { + var disposals = new List(); + if (_sqlServerStarted) + disposals.Add(_sqlServer.DisposeAsync().AsTask()); + if (_postgreSqlStarted) + disposals.Add(_postgreSql.DisposeAsync().AsTask()); + if (_mongoDbStarted) + disposals.Add(_mongoDb.DisposeAsync().AsTask()); + await Task.WhenAll(disposals); + _sqlServerStarted = false; + _postgreSqlStarted = false; + _mongoDbStarted = false; + } +} + +[CollectionDefinition(Name)] +public sealed class DiagnosticsProviderCollection : ICollectionFixture +{ + public const string Name = "diagnostics-providers"; +} diff --git a/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogReplayTests.cs b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogReplayTests.cs new file mode 100644 index 000000000..1a3b0d7aa --- /dev/null +++ b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogReplayTests.cs @@ -0,0 +1,183 @@ +using Elsa.Diagnostics.StructuredLogs.Core.Exceptions; +using Elsa.Diagnostics.StructuredLogs.Core.Models; +using Elsa.Diagnostics.StructuredLogs.Core.Options; +using Groundwork.DiagnosticRecords; +using Groundwork.Sqlite.DiagnosticRecords; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork.Tests; + +public sealed class GroundworkStructuredLogReplayTests : GroundworkStructuredLogStoreTestBase +{ + [Fact] + public async Task Tied_entries_replay_in_committed_cursor_order() + { + await using var first = await CreateStoreAsync(Binding); + await using var second = await CreateStoreAsync(Binding); + var timestamp = DateTimeOffset.Parse("2026-07-12T12:00:00Z"); + + var commits = await Task.WhenAll( + first.AppendAsync(Entry(5, "first", timestamp)).AsTask(), + second.AppendAsync(Entry(5, "second", timestamp)).AsTask()); + var ordered = (await first.GetRecentAsync(StructuredLogFilter.None)).ToArray(); + var replayed = (await first.ReadAfterAsync(ordered[0].ReplayCursor, StructuredLogFilter.None, 100)).Entries; + + Assert.Single(replayed); + Assert.Equal(ordered[1].Message, replayed[0].Message); + Assert.NotEqual(commits[0].ReplayCursor, commits[1].ReplayCursor); + } + + [Fact] + public async Task Filtered_empty_page_advances_cursor_to_the_last_scanned_record() + { + await using var store = await CreateStoreAsync(Binding); + var timestamp = DateTimeOffset.Parse("2026-07-12T12:00:00Z"); + await store.AppendAsync(Entry(1, "filtered-out", timestamp) with { SourceId = "other" }); + var expected = await store.AppendAsync(Entry(2, "included", timestamp) with { SourceId = "selected" }); + + var emptyPage = await store.ReadAfterAsync( + null, + new StructuredLogFilter { SourceId = "selected" }, + 1); + var matchingPage = await store.ReadAfterAsync( + emptyPage.NextCursor, + new StructuredLogFilter { SourceId = "selected" }, + 1); + + Assert.Empty(emptyPage.Entries); + Assert.NotNull(emptyPage.NextCursor); + Assert.True(emptyPage.HasMore); + Assert.Equal([expected.ReplayCursor], matchingPage.Entries.Select(x => x.ReplayCursor)); + } + + [Fact] + public async Task Restart_preserves_cursor_replay_and_lifetime_logical_high_water() + { + StructuredLogEntry first; + await using (var beforeRestart = await CreateStoreAsync(Binding)) + first = await beforeRestart.AppendAsync(Entry(11, "before", DateTimeOffset.UnixEpoch)); + + await using var restarted = await CreateStoreAsync(Binding); + var second = await restarted.AppendAsync(Entry(12, "after", DateTimeOffset.UnixEpoch)); + var replay = await restarted.ReadAfterAsync(first.ReplayCursor, StructuredLogFilter.None, 100); + + Assert.Equal(12, await restarted.GetHighWaterMarkAsync()); + Assert.Equal([second.ReplayCursor], replay.Entries.Select(x => x.ReplayCursor)); + } + + [Fact] + public async Task Malformed_or_foreign_binding_fails_with_one_non_disclosing_outcome() + { + await using var source = await CreateStoreAsync(Binding); + await using var wrongTenant = await CreateStoreAsync(new("tenant-b", "shell-a", "structured-logs")); + await using var wrongScope = await CreateStoreAsync(new("tenant-a", "shell-b", "structured-logs")); + await using var wrongStream = await CreateStoreAsync(new("tenant-a", "shell-a", "audit-logs")); + var committed = await source.AppendAsync(Entry(1, "source", DateTimeOffset.UnixEpoch)); + + var errors = new[] + { + await Assert.ThrowsAsync(() => + source.ReadAfterAsync(default(StructuredLogReplayCursor), StructuredLogFilter.None, 100)), + await Assert.ThrowsAsync(() => + wrongTenant.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)), + await Assert.ThrowsAsync(() => + wrongScope.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)), + await Assert.ThrowsAsync(() => + wrongStream.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)) + }; + + Assert.All(errors, error => Assert.Equal("The structured log replay cursor is unavailable.", error.Message)); + } + + [Fact] + public async Task Trimmed_anchor_expires_while_lifetime_high_water_survives_restart() + { + StructuredLogEntry committed; + await using (var store = await CreateStoreAsync(Binding)) + { + committed = await store.AppendAsync(Entry(19, "trimmed", DateTimeOffset.UnixEpoch)); + await store.TrimAsync(0); + } + + await using var restarted = await CreateStoreAsync(Binding); + + Assert.Equal(19, await restarted.GetHighWaterMarkAsync()); + Assert.Empty(await restarted.GetRecentAsync(StructuredLogFilter.None)); + await Assert.ThrowsAsync(() => + restarted.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); + } + + [Fact] + public async Task Operational_query_failure_is_not_reported_as_cursor_unavailable() + { + var provider = await CreateProviderAsync(Binding); + var failure = new StructuredLogsException("The diagnostics database is unavailable."); + var failing = new QueryFailingStore(provider, failure); + await using var store = new GroundworkStructuredLogStore( + failing, + Options.Create(new StructuredLogsOptions()), + Binding); + var committed = await store.AppendAsync(Entry(1, "source", DateTimeOffset.UnixEpoch)); + + failing.FailQueries = true; + + var actual = await Assert.ThrowsAsync(() => + store.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); + Assert.Same(failure, actual); + } + + private sealed class QueryFailingStore(IDiagnosticRecordStore inner, Exception failure) : IDiagnosticRecordStore + { + public bool FailQueries { get; set; } + public DiagnosticRecordStoreHandlers Handlers => inner.Handlers; + + public ValueTask QueryAsync( + DiagnosticRecordQuery query, + CancellationToken cancellationToken = default) => + FailQueries + ? ValueTask.FromException(failure) + : inner.QueryAsync(query, cancellationToken); + } +} + +public abstract class GroundworkStructuredLogStoreTestBase : IAsyncLifetime +{ + private readonly string _databasePath = Path.Combine( + Path.GetTempPath(), + $"elsa-structured-logs-{Guid.NewGuid():N}.db"); + + protected static StructuredLogStoreBinding Binding { get; } = + new("tenant-a", "shell-a", "structured-logs"); + + protected async Task CreateStoreAsync(StructuredLogStoreBinding binding) + { + var provider = await CreateProviderAsync(binding); + return new(provider, Options.Create(new StructuredLogsOptions()), binding); + } + + protected Task CreateProviderAsync(StructuredLogStoreBinding binding) => + SqliteDiagnosticRecordStoreFactory.CreateAsync( + $"Data Source={_databasePath}", + GroundworkStructuredLogStore.CreateStreamDefinition(binding.StreamId)); + + protected static StructuredLogEntry Entry(long sequence, string message, DateTimeOffset timestamp) => new() + { + Sequence = sequence, + Timestamp = timestamp, + Level = LogLevel.Information, + Category = "Test.Category", + Message = message, + SourceId = "writer" + }; + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + if (File.Exists(_databasePath)) + File.Delete(_databasePath); + return Task.CompletedTask; + } +} diff --git a/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogStoreTests.cs b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogStoreTests.cs index f45e4328a..77146c895 100644 --- a/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogStoreTests.cs +++ b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/GroundworkStructuredLogStoreTests.cs @@ -5,101 +5,13 @@ using Elsa.Diagnostics.StructuredLogs.Core.Contracts; using Elsa.Diagnostics.StructuredLogs.Core.Exceptions; using Groundwork.DiagnosticRecords; -using Groundwork.Sqlite.DiagnosticRecords; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Xunit; namespace Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork.Tests; -public sealed class GroundworkStructuredLogStoreTests : IAsyncLifetime +public sealed class GroundworkStructuredLogStoreTests : GroundworkStructuredLogStoreTestBase { - private readonly string _databasePath = Path.Combine(Path.GetTempPath(), $"elsa-structured-logs-{Guid.NewGuid():N}.db"); - private static readonly StructuredLogStoreBinding Binding = new("tenant-a", "shell-a", "structured-logs"); - - [Fact] - public async Task Two_writers_with_equal_timestamp_and_sequence_replay_in_committed_cursor_order() - { - await using var first = await CreateStoreAsync(Binding); - await using var second = await CreateStoreAsync(Binding); - var timestamp = DateTimeOffset.Parse("2026-07-12T12:00:00Z"); - - var commits = await Task.WhenAll( - first.AppendAsync(Entry(5, "first", timestamp)).AsTask(), - second.AppendAsync(Entry(5, "second", timestamp)).AsTask()); - var ordered = (await first.GetRecentAsync(StructuredLogFilter.None)).ToArray(); - var replayed = (await first.ReadAfterAsync(ordered[0].ReplayCursor, StructuredLogFilter.None, 100)).Entries; - - Assert.Single(replayed); - Assert.Equal(ordered[1].Message, replayed[0].Message); - Assert.NotEqual(commits[0].ReplayCursor, commits[1].ReplayCursor); - } - - [Fact] - public async Task Restart_preserves_cursor_replay_and_lifetime_logical_high_water() - { - StructuredLogEntry first; - await using (var beforeRestart = await CreateStoreAsync(Binding)) - first = await beforeRestart.AppendAsync(Entry(11, "before", DateTimeOffset.UnixEpoch)); - - await using var restarted = await CreateStoreAsync(Binding); - var second = await restarted.AppendAsync(Entry(12, "after", DateTimeOffset.UnixEpoch)); - var replay = await restarted.ReadAfterAsync(first.ReplayCursor, StructuredLogFilter.None, 100); - - Assert.Equal(12, await restarted.GetHighWaterMarkAsync()); - Assert.Equal([second.ReplayCursor], replay.Entries.Select(x => x.ReplayCursor)); - } - - [Fact] - public async Task Trim_to_zero_and_restart_preserve_high_water_but_expire_replay_cursor() - { - StructuredLogEntry committed; - await using (var store = await CreateStoreAsync(Binding)) - { - committed = await store.AppendAsync(Entry(19, "trimmed", DateTimeOffset.UnixEpoch)); - await store.TrimAsync(0); - } - - await using var restarted = await CreateStoreAsync(Binding); - - Assert.Equal(19, await restarted.GetHighWaterMarkAsync()); - Assert.Empty(await restarted.GetRecentAsync(StructuredLogFilter.None)); - await Assert.ThrowsAsync(() => - restarted.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); - } - - [Fact] - public async Task Wrong_tenant_scope_and_stream_fail_without_disclosing_which_binding_mismatched() - { - await using var source = await CreateStoreAsync(Binding); - await using var wrongTenant = await CreateStoreAsync(new("tenant-b", "shell-a", "structured-logs")); - await using var wrongScope = await CreateStoreAsync(new("tenant-a", "shell-b", "structured-logs")); - await using var wrongStream = await CreateStoreAsync(new("tenant-a", "shell-a", "audit-logs")); - var committed = await source.AppendAsync(Entry(1, "source", DateTimeOffset.UnixEpoch)); - - var tenantError = await Assert.ThrowsAsync(() => - wrongTenant.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); - var scopeError = await Assert.ThrowsAsync(() => - wrongScope.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); - var streamError = await Assert.ThrowsAsync(() => - wrongStream.ReadAfterAsync(committed.ReplayCursor, StructuredLogFilter.None, 100)); - - Assert.Equal(tenantError.Message, scopeError.Message); - Assert.Equal(scopeError.Message, streamError.Message); - Assert.Equal("The structured log replay cursor is unavailable.", scopeError.Message); - } - - [Fact] - public async Task Default_cursor_is_rejected_at_the_groundwork_store_boundary() - { - await using var store = await CreateStoreAsync(Binding); - - var exception = await Assert.ThrowsAsync(() => - store.ReadAfterAsync(default(StructuredLogReplayCursor), StructuredLogFilter.None, 100)); - - Assert.Equal("The structured log replay cursor is unavailable.", exception.Message); - } - [Fact] public async Task Acknowledgement_loss_retries_the_same_operation_and_publishes_one_committed_cursor() { @@ -136,36 +48,6 @@ public async Task Hard_stop_settles_every_accepted_append_when_provider_ignores_ await hanging.Exited.WaitAsync(TimeSpan.FromSeconds(10)); } - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - if (File.Exists(_databasePath)) - File.Delete(_databasePath); - return Task.CompletedTask; - } - - private async Task CreateStoreAsync(StructuredLogStoreBinding binding) - { - var provider = await CreateProviderAsync(binding); - return new(provider, Options.Create(new StructuredLogsOptions()), binding); - } - - private Task CreateProviderAsync(StructuredLogStoreBinding binding) => - SqliteDiagnosticRecordStoreFactory.CreateAsync( - $"Data Source={_databasePath}", - GroundworkStructuredLogStore.CreateStreamDefinition(binding.StreamId)); - - private static StructuredLogEntry Entry(long sequence, string message, DateTimeOffset timestamp) => new() - { - Sequence = sequence, - Timestamp = timestamp, - Level = LogLevel.Information, - Category = "Test.Category", - Message = message, - SourceId = "writer" - }; - private static async Task WaitUntilAsync(Func predicate) { using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); diff --git a/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/StructuredLogsGroundworkStorageSchemaTests.cs b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/StructuredLogsGroundworkStorageSchemaTests.cs new file mode 100644 index 000000000..694875b31 --- /dev/null +++ b/tests/Elsa/Diagnostics/StructuredLogs/Persistence/Groundwork/Tests/StructuredLogsGroundworkStorageSchemaTests.cs @@ -0,0 +1,22 @@ +using Elsa.Diagnostics.StructuredLogs.Core.Models; +using Xunit; + +namespace Elsa.Diagnostics.StructuredLogs.Persistence.Groundwork.Tests; + +public sealed class StructuredLogsGroundworkStorageSchemaTests +{ + [Fact] + public void Schema_declares_bound_stream_projected_indexes_and_append_trim_ledgers() + { + var binding = new StructuredLogStoreBinding("tenant", "scope", "logs"); + var stream = StructuredLogsGroundworkStorageSchema.CreateStream(binding); + + Assert.Equal(binding.StreamId, stream.Stream.Value); + Assert.Equal( + StructuredLogsGroundworkStorageSchema.RequiredIndexFields.Order(StringComparer.Ordinal), + stream.Fields.Select(x => x.Name).Order(StringComparer.Ordinal)); + Assert.Equal( + new[] { "diagnostic-append-idempotency", "diagnostic-trim-idempotency" }, + StructuredLogsGroundworkStorageSchema.RequiredOperationLedgers.Order(StringComparer.Ordinal)); + } +} diff --git a/tests/Elsa/Diagnostics/StructuredLogs/Tests/Elsa.Diagnostics.StructuredLogs.Tests.csproj b/tests/Elsa/Diagnostics/StructuredLogs/Tests/Elsa.Diagnostics.StructuredLogs.Tests.csproj index 2a3486ae1..77dfd4e84 100644 --- a/tests/Elsa/Diagnostics/StructuredLogs/Tests/Elsa.Diagnostics.StructuredLogs.Tests.csproj +++ b/tests/Elsa/Diagnostics/StructuredLogs/Tests/Elsa.Diagnostics.StructuredLogs.Tests.csproj @@ -5,6 +5,7 @@ enable enable false + true @@ -21,6 +22,7 @@ + diff --git a/tests/Elsa/Diagnostics/StructuredLogs/Tests/StructuredLogsFeatureTests.cs b/tests/Elsa/Diagnostics/StructuredLogs/Tests/StructuredLogsFeatureTests.cs index 28ea22e2f..5f1bf736f 100644 --- a/tests/Elsa/Diagnostics/StructuredLogs/Tests/StructuredLogsFeatureTests.cs +++ b/tests/Elsa/Diagnostics/StructuredLogs/Tests/StructuredLogsFeatureTests.cs @@ -1,6 +1,8 @@ using Elsa.Diagnostics.StructuredLogs.Core.Contracts; using Elsa.Diagnostics.StructuredLogs.Core.Options; +using Elsa.Diagnostics.StructuredLogs.Core.Models; using Elsa.Diagnostics.StructuredLogs.Endpoints; +using Elsa.Diagnostics.Persistence.Observability; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -54,4 +56,29 @@ public void AppliesConfiguredOptions() Assert.Equal(LogLevel.Warning, options.MinimumLevel); Assert.Equal(123, options.BufferCapacity); } + + [Fact] + public async Task Production_composition_counts_each_subscriber_drop_once() + { + var services = new ServiceCollection(); + new StructuredLogsFeature().ConfigureServices(services); + services.Configure(options => options.SubscriberQueueCapacity = 1); + using var provider = services.BuildServiceProvider(); + var feed = provider.GetRequiredService(); + var publisher = provider.GetRequiredService(); + var counters = provider.GetRequiredService(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await using var subscriber = feed.Subscribe(StructuredLogFilter.None, cancellation.Token) + .GetAsyncEnumerator(cancellation.Token); + + publisher.Publish(TestEntries.Create(sequence: 1)); + publisher.Publish(TestEntries.Create(sequence: 2)); + publisher.Publish(TestEntries.Create(sequence: 3)); + Assert.True(await subscriber.MoveNextAsync()); + Assert.True(await subscriber.MoveNextAsync()); + + var signal = Assert.IsType(subscriber.Current.Dropped); + Assert.Equal(signal.DroppedCount, + counters.Snapshot().Losses[DiagnosticsPersistenceLossReason.SubscriberDelivery]); + } }