private async Task BasicScenario(bool enabled)
{
var options = new ClusterMembershipOptions {
DefunctSiloCleanupPeriod = enabled ? new TimeSpan?(TimeSpan.FromMinutes(90)) : null, DefunctSiloExpiration = TimeSpan.FromDays(1)
};
var timers = new List <DelegateAsyncTimer>();
var timerCalls = new ConcurrentQueue <(TimeSpan?DelayOverride, TaskCompletionSource <bool> Completion)>();
var timerFactory = new DelegateAsyncTimerFactory(
(period, name) =>
{
Assert.Equal(options.DefunctSiloCleanupPeriod.Value, period);
var t = new DelegateAsyncTimer(
overridePeriod =>
{
var task = new TaskCompletionSource <bool>();
timerCalls.Enqueue((overridePeriod, task));
return(task.Task);
});
timers.Add(t);
return(t);
});
var table = new InMemoryMembershipTable();
var cleanupAgent = new MembershipTableCleanupAgent(
Options.Create(options),
table,
this.loggerFactory.CreateLogger <MembershipTableCleanupAgent>(),
timerFactory);
var lifecycle = new SiloLifecycleSubject(this.loggerFactory.CreateLogger <SiloLifecycleSubject>());
((ILifecycleParticipant <ISiloLifecycle>)cleanupAgent).Participate(lifecycle);
await lifecycle.OnStart();
Assert.DoesNotContain(table.Calls, c => c.Method.Equals(nameof(IMembershipTable.CleanupDefunctSiloEntries)));
Assert.Equal(enabled, timerCalls.TryDequeue(out var timer));
timer.Completion?.TrySetResult(true);
var stopped = lifecycle.OnStop();
while (timerCalls.TryDequeue(out timer))
{
timer.Completion.TrySetResult(false);
}
if (enabled)
{
Assert.Contains(table.Calls, c => c.Method.Equals(nameof(IMembershipTable.CleanupDefunctSiloEntries)));
}
else
{
Assert.DoesNotContain(table.Calls, c => c.Method.Equals(nameof(IMembershipTable.CleanupDefunctSiloEntries)));
}
await stopped;
}
public async Task MembershipTableManager_ExistingCluster()
{
var otherSilos = new[]
{
Entry(Silo("127.0.0.1:200@100"), SiloStatus.Active),
Entry(Silo("127.0.0.1:300@100"), SiloStatus.ShuttingDown),
Entry(Silo("127.0.0.1:400@100"), SiloStatus.Joining),
Entry(Silo("127.0.0.1:500@100"), SiloStatus.Dead),
};
var membershipTable = new InMemoryMembershipTable(new TableVersion(123, "123"), otherSilos);
await this.BasicScenarioTest(membershipTable, gracefulShutdown : false);
}
private async Task BasicScenarioTest(InMemoryMembershipTable membershipTable, bool gracefulShutdown = true)
{
var timers = new List <DelegateAsyncTimer>();
var timerCalls = new ConcurrentQueue <(TimeSpan?DelayOverride, TaskCompletionSource <bool> Completion)>();
var timerFactory = new DelegateAsyncTimerFactory(
(period, name) =>
{
var timer = new DelegateAsyncTimer(
overridePeriod =>
{
var task = new TaskCompletionSource <bool>();
timerCalls.Enqueue((overridePeriod, task));
return(task.Task);
});
timers.Add(timer);
return(timer);
});
var manager = new MembershipTableManager(
localSiloDetails: this.localSiloDetails,
clusterMembershipOptions: Options.Create(new ClusterMembershipOptions()),
membershipTable: membershipTable,
fatalErrorHandler: this.fatalErrorHandler,
gossiper: this.membershipGossiper,
log: this.loggerFactory.CreateLogger <MembershipTableManager>(),
timerFactory: timerFactory,
this.lifecycle);
// Validate that the initial snapshot is valid and contains the local silo.
var initialSnapshot = manager.MembershipTableSnapshot;
Assert.NotNull(initialSnapshot);
Assert.NotNull(initialSnapshot.Entries);
Assert.NotNull(initialSnapshot.LocalSilo);
Assert.Equal(SiloStatus.Created, initialSnapshot.LocalSilo.Status);
Assert.Equal(this.localSiloDetails.Name, initialSnapshot.LocalSilo.SiloName);
Assert.Equal(this.localSiloDetails.DnsHostName, initialSnapshot.LocalSilo.HostName);
Assert.Equal(SiloStatus.Created, manager.CurrentStatus);
Assert.NotNull(manager.MembershipTableUpdates);
var changes = manager.MembershipTableUpdates;
var currentEnumerator = changes.GetAsyncEnumerator();
Assert.True(currentEnumerator.MoveNextAsync().Result);
Assert.Equal(currentEnumerator.Current.Version, manager.MembershipTableSnapshot.Version);
Assert.Empty(membershipTable.Calls);
// All of these checks were performed before any lifecycle methods have a chance to run.
// This is in order to verify that a service accessing membership in its constructor will
// see the correct results regardless of initialization order.
((ILifecycleParticipant <ISiloLifecycle>)manager).Participate(this.lifecycle);
await this.lifecycle.OnStart();
var calls = membershipTable.Calls;
Assert.NotEmpty(calls);
Assert.True(calls.Count >= 2);
Assert.Equal(nameof(IMembershipTable.InitializeMembershipTable), calls[0].Method);
Assert.Equal(nameof(IMembershipTable.ReadAll), calls[1].Method);
// During initialization, a first read from the table will be performed, transitioning
// membership to a valid version.currentEnumerator = changes.GetAsyncEnumerator();
currentEnumerator = changes.GetAsyncEnumerator();
Assert.True(currentEnumerator.MoveNextAsync().Result);
var update1 = currentEnumerator.Current;
// Transition to joining.
this.membershipGossiper.ClearReceivedCalls();
await manager.UpdateStatus(SiloStatus.Joining);
await this.membershipGossiper.ReceivedWithAnyArgs().GossipToRemoteSilos(default, default, default);
public async Task MembershipTableManager_NewCluster()
{
var membershipTable = new InMemoryMembershipTable(new TableVersion(123, "123"));
await this.BasicScenarioTest(membershipTable, gracefulShutdown : true);
}
