Skip to content

Commit 7a20fc7

Browse files
sfmskywalkersfmskywalker
committed
Fix tenant service mutation race
1 parent 1000d29 commit 7a20fc7

2 files changed

Lines changed: 157 additions & 9 deletions

File tree

‎src/modules/Elsa.Common/Multitenancy/Implementations/DefaultTenantService.cs‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ public class DefaultTenantService(IServiceScopeFactory scopeFactory, ITenantScop
77
{
88
private readonly AsyncServiceScope _serviceScope = scopeFactory.CreateAsyncScope();
99
private readonly SemaphoreSlim _initializationLock = new(1, 1);
10-
private readonly SemaphoreSlim _refreshLock = new(1, 1);
10+
private readonly SemaphoreSlim _tenantMutationLock = new(1, 1);
1111
private IDictionary<string, Tenant>? _tenantsDictionary;
1212
private IDictionary<Tenant, TenantScope>? _tenantScopesDictionary;
1313

@@ -60,22 +60,33 @@ public async Task ActivateTenantsAsync(CancellationToken cancellationToken = def
6060

6161
public async Task DeactivateTenantsAsync(CancellationToken cancellationToken = default)
6262
{
63-
var dictionary = await GetTenantsDictionaryAsync(cancellationToken);
64-
var tenants = dictionary.Values.ToArray();
63+
var dictionary = await GetTenantsDictionaryForMutationAsync(cancellationToken);
64+
await _tenantMutationLock.WaitAsync(cancellationToken);
65+
66+
try
67+
{
68+
var tenants = dictionary.Values.ToArray();
6569

66-
foreach (var tenant in tenants)
67-
await UnregisterTenantAsync(tenant, false, cancellationToken);
70+
foreach (var tenant in tenants)
71+
{
72+
await UnregisterTenantAsync(tenant, false, cancellationToken);
73+
}
74+
}
75+
finally
76+
{
77+
_tenantMutationLock.Release();
78+
}
6879
}
6980

7081
public async Task RefreshAsync(CancellationToken cancellationToken = default)
7182
{
72-
await _refreshLock.WaitAsync(cancellationToken);
83+
var currentTenants = await GetTenantsDictionaryForMutationAsync(cancellationToken);
84+
await _tenantMutationLock.WaitAsync(cancellationToken);
7385

7486
try
7587
{
7688
await using var scope = scopeFactory.CreateAsyncScope();
7789
var tenantsProvider = scope.ServiceProvider.GetRequiredService<ITenantsProvider>();
78-
var currentTenants = await GetTenantsDictionaryAsync(cancellationToken);
7990
var currentTenantIds = currentTenants.Keys;
8091
var tenantsFromProvider = (await tenantsProvider.ListAsync(cancellationToken)).ToList();
8192
var newTenants = tenantsFromProvider.Count == 0
@@ -99,10 +110,22 @@ public async Task RefreshAsync(CancellationToken cancellationToken = default)
99110
}
100111
finally
101112
{
102-
_refreshLock.Release();
113+
_tenantMutationLock.Release();
103114
}
104115
}
105116

117+
private async Task<IDictionary<string, Tenant>> GetTenantsDictionaryForMutationAsync(CancellationToken cancellationToken)
118+
{
119+
var dictionary = await GetTenantsDictionaryAsync(cancellationToken);
120+
121+
// The dictionary is published before initialization completes so lifecycle event handlers can read it.
122+
// Wait for any concurrent initializer before allowing a mutation to proceed.
123+
await _initializationLock.WaitAsync(cancellationToken);
124+
_initializationLock.Release();
125+
126+
return dictionary;
127+
}
128+
106129
private async Task<IDictionary<string, Tenant>> GetTenantsDictionaryAsync(CancellationToken cancellationToken)
107130
{
108131
if (_tenantsDictionary == null)
@@ -157,4 +180,4 @@ private async Task UnregisterTenantAsync(Tenant tenant, bool isDeleted, Cancella
157180
}
158181
}
159182
}
160-
}
183+
}

‎test/unit/Elsa.Common.UnitTests/Multitenancy/DefaultTenantServiceTests.cs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,126 @@ public async Task RefreshAsync_WhenProviderChangesFromTenantsToEmpty_KeepsDefaul
119119
}
120120
}
121121

122+
[Fact]
123+
public async Task DeactivateTenantsAsync_WhenRefreshIsInProgress_WaitsForRefreshBeforeDeactivating()
124+
{
125+
var tenant = new Tenant { Id = "tenant-1", Name = "Tenant 1" };
126+
var timeout = TimeSpan.FromSeconds(5);
127+
var refreshStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
128+
var completeRefresh = new TaskCompletionSource<IEnumerable<Tenant>>(TaskCreationOptions.RunContinuationsAsynchronously);
129+
var listRequests = 0;
130+
var tenantsProvider = Substitute.For<ITenantsProvider>();
131+
tenantsProvider.ListAsync(Arg.Any<CancellationToken>()).Returns(async _ =>
132+
{
133+
if (Interlocked.Increment(ref listRequests) == 1)
134+
{
135+
return new[] { tenant };
136+
}
137+
138+
refreshStarted.TrySetResult();
139+
return await completeRefresh.Task;
140+
});
141+
142+
var (tenantService, serviceProvider) = await CreateTenantServiceAsync(tenantsProvider);
143+
Task? refreshTask = null;
144+
Task? deactivateTask = null;
145+
146+
try
147+
{
148+
await tenantService.ListAsync();
149+
150+
refreshTask = tenantService.RefreshAsync();
151+
await refreshStarted.Task.WaitAsync(timeout);
152+
153+
deactivateTask = tenantService.DeactivateTenantsAsync();
154+
155+
Assert.False(deactivateTask.IsCompleted);
156+
157+
completeRefresh.SetResult([tenant]);
158+
await refreshTask.WaitAsync(timeout);
159+
await deactivateTask.WaitAsync(timeout);
160+
161+
Assert.Empty(await tenantService.ListAsync());
162+
}
163+
finally
164+
{
165+
completeRefresh.TrySetResult([tenant]);
166+
167+
if (refreshTask != null)
168+
{
169+
await refreshTask.WaitAsync(timeout);
170+
}
171+
172+
if (deactivateTask != null)
173+
{
174+
await deactivateTask.WaitAsync(timeout);
175+
}
176+
177+
if (tenantService is IAsyncDisposable disposable)
178+
{
179+
await disposable.DisposeAsync();
180+
}
181+
182+
await serviceProvider.DisposeAsync();
183+
}
184+
}
185+
186+
[Fact]
187+
public async Task DeactivateTenantsAsync_WhenInitializationIsInProgress_WaitsForInitializationBeforeDeactivating()
188+
{
189+
var tenant = new Tenant { Id = "tenant-1", Name = "Tenant 1" };
190+
var timeout = TimeSpan.FromSeconds(5);
191+
var initializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
192+
var completeInitialization = new TaskCompletionSource<IEnumerable<Tenant>>(TaskCreationOptions.RunContinuationsAsynchronously);
193+
var tenantsProvider = Substitute.For<ITenantsProvider>();
194+
tenantsProvider.ListAsync(Arg.Any<CancellationToken>()).Returns(async _ =>
195+
{
196+
initializationStarted.TrySetResult();
197+
return await completeInitialization.Task;
198+
});
199+
200+
var (tenantService, serviceProvider) = await CreateTenantServiceAsync(tenantsProvider);
201+
Task<IEnumerable<Tenant>>? initializationTask = null;
202+
Task? deactivateTask = null;
203+
204+
try
205+
{
206+
initializationTask = tenantService.ListAsync();
207+
await initializationStarted.Task.WaitAsync(timeout);
208+
209+
deactivateTask = tenantService.DeactivateTenantsAsync();
210+
211+
Assert.False(deactivateTask.IsCompleted);
212+
213+
completeInitialization.SetResult([tenant]);
214+
await initializationTask.WaitAsync(timeout);
215+
await deactivateTask.WaitAsync(timeout);
216+
217+
Assert.Empty(await tenantService.ListAsync());
218+
}
219+
finally
220+
{
221+
completeInitialization.TrySetResult([tenant]);
222+
223+
if (initializationTask != null)
224+
{
225+
await initializationTask.WaitAsync(timeout);
226+
}
227+
228+
if (deactivateTask != null)
229+
{
230+
await deactivateTask.WaitAsync(timeout);
231+
}
232+
233+
if (tenantService is IAsyncDisposable disposable)
234+
{
235+
await disposable.DisposeAsync();
236+
}
237+
238+
await serviceProvider.DisposeAsync();
239+
}
240+
}
241+
122242
private static Task<(ITenantService TenantService, ServiceProvider ServiceProvider)> CreateTenantServiceAsync(IEnumerable<Tenant> tenants, Func<List<Tenant>>? tenantsFactory = null)
123243
{
124244
var tenantList = tenants.ToList();
@@ -127,6 +247,11 @@ public async Task RefreshAsync_WhenProviderChangesFromTenantsToEmpty_KeepsDefaul
127247
var tenantsProvider = Substitute.For<ITenantsProvider>();
128248
tenantsProvider.ListAsync(Arg.Any<CancellationToken>()).Returns(_ => getTenants());
129249

250+
return CreateTenantServiceAsync(tenantsProvider);
251+
}
252+
253+
private static Task<(ITenantService TenantService, ServiceProvider ServiceProvider)> CreateTenantServiceAsync(ITenantsProvider tenantsProvider)
254+
{
130255
var services = new ServiceCollection();
131256
services.AddSingleton(_ => tenantsProvider);
132257
services.AddSingleton<ITenantScopeFactory, DefaultTenantScopeFactory>();

0 commit comments

Comments
 (0)