public async Task DoesNotFailBeforeFirstResolverUpdate()
{
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var allSilos = this.oracle.GetApproximateSiloStatuses(false);
var livingSilos = this.oracle.GetApproximateSiloStatuses(true);
Assert.Single(allSilos);
Assert.Single(livingSilos);
Assert.Equal(2, listener.Notifications.Count);
Assert.Single(listener.Silos);
await this.oracle.Stop();
allSilos = this.oracle.GetApproximateSiloStatuses(false);
livingSilos = this.oracle.GetApproximateSiloStatuses(true);
Assert.Single(allSilos);
Assert.Empty(livingSilos);
Assert.Equal(3, listener.Notifications.Count);
Assert.Single(listener.Silos);
}
public async Task UnknownSilosAreAccountedForDuringUpdate()
{
var now = new[] { DateTime.UtcNow };
this.unknownSiloMonitor.GetDateTime = () => now[0];
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
this.resolver.Notify(silos);
listener.WaitForVersion(4);
Assert.Equal(3, listener.Silos.Count);
// Query for an unknown silo. Doing so should result in the silo being recorded as unknown.
var unknownSilo1 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("1.1.1.1"), 3), 3);
Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1));
this.resolver.Notify(silos);
// The status should not have changed.
Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1));
now[0] += this.fabricMembershipOptions.UnknownSiloRemovalPeriod + TimeSpan.FromMilliseconds(1);
this.resolver.Notify(silos);
listener.WaitForVersion(5);
// After the delay, the silo should be declared dead.
Assert.Equal(SiloStatus.Dead, this.oracle.GetApproximateSiloStatus(unknownSilo1));
// Query for another unknown silo using a different mechanism.
var unknownSilo2 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 4);
Assert.False(this.oracle.IsFunctionalDirectory(unknownSilo2));
// Report a new silo with the same endpoint as the unknown silo but a higher generation.
silos = new[]
{
silos[0],
silos[1],
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 5),
SiloAddress.Zero,
"TippyTop")
};
this.resolver.Notify(silos);
listener.WaitForVersion(6);
Assert.True(this.oracle.IsDeadSilo(unknownSilo2));
}
public async Task HandlesSiloAdditionAndRemoval()
{
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
// The local silo transitioned to 'Joining' and then 'Active' and two other silos became 'Active',
// so the version must be 4
this.resolver.Notify(silos);
listener.WaitForVersion(4);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Active);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Send the same update again and verify that nothing changed.
this.resolver.Notify(silos);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
// The single removal will bump the version to 5.
this.resolver.Notify(new[] { silos[1] });
listener.WaitForVersion(5);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
this.resolver.Notify(new FabricSiloInfo[0]);
listener.WaitForVersion(6);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
}
public async Task HandlesSiloAdditionAndRemoval()
{
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
listener.VersionReached.Reset();
listener.WaitForVersion = 4;
this.resolver.Notify(silos);
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Active);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Send the same update again and verify that nothing changed.
this.resolver.Notify(silos);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
listener.WaitForVersion = 5;
this.resolver.Notify(new[] { silos[1] });
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
this.resolver.Notify(new FabricSiloInfo[0]);
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
}