public async Task UnknownSilosAreAccountedForDuringUpdate() { var now = new[] { DateTime.UtcNow }; this.unknownSiloMonitor.GetDateTime = () => now[0]; var listener = new MockStatusListener(); this.oracle.SubscribeToSiloStatusEvents(listener); await this.oracle.Start(); await this.oracle.BecomeActive(); var silos = new[] { CreateSiloInfo( SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1), SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1), "HappyNewSilo"), CreateSiloInfo( SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2), SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2), "OtherNewSilo"), }; this.resolver.Notify(silos); listener.WaitForVersion(4); Assert.Equal(3, listener.Silos.Count); // Query for an unknown silo. Doing so should result in the silo being recorded as unknown. var unknownSilo1 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("1.1.1.1"), 3), 3); Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1)); this.resolver.Notify(silos); // The status should not have changed. Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1)); now[0] += this.fabricMembershipOptions.UnknownSiloRemovalPeriod + TimeSpan.FromMilliseconds(1); this.resolver.Notify(silos); listener.WaitForVersion(5); // After the delay, the silo should be declared dead. Assert.Equal(SiloStatus.Dead, this.oracle.GetApproximateSiloStatus(unknownSilo1)); // Query for another unknown silo using a different mechanism. var unknownSilo2 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 4); Assert.False(this.oracle.IsFunctionalDirectory(unknownSilo2)); // Report a new silo with the same endpoint as the unknown silo but a higher generation. silos = new[] { silos[0], silos[1], CreateSiloInfo( SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 5), SiloAddress.Zero, "TippyTop") }; this.resolver.Notify(silos); listener.WaitForVersion(6); Assert.True(this.oracle.IsDeadSilo(unknownSilo2)); }
public async Task HandlesSiloAdditionAndRemoval() { var listener = new MockStatusListener(); this.oracle.SubscribeToSiloStatusEvents(listener); await this.oracle.Start(); await this.oracle.BecomeActive(); var multiClusters = this.oracle.GetApproximateMultiClusterGateways(); Assert.Equal(1, multiClusters.Count); Assert.Contains(this.siloDetails.SiloAddress, multiClusters); var silos = new[] { CreateSiloInfo( SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1), SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1), "HappyNewSilo"), CreateSiloInfo( SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2), SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2), "OtherNewSilo"), }; // The local silo transitioned to 'Joining' and then 'Active' and two other silos became 'Active', // so the version must be 4 this.resolver.Notify(silos); listener.WaitForVersion(4); Assert.Equal(3, listener.Silos.Count); Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys); Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]); AssertStatus(silos[0].SiloAddress, SiloStatus.Active); AssertStatus(silos[1].SiloAddress, SiloStatus.Active); multiClusters = this.oracle.GetApproximateMultiClusterGateways(); Assert.Equal(2, multiClusters.Count); // Send the same update again and verify that nothing changed. this.resolver.Notify(silos); Assert.Equal(3, listener.Silos.Count); Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys); Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]); multiClusters = this.oracle.GetApproximateMultiClusterGateways(); Assert.Equal(2, multiClusters.Count); // Remove a silo and verify that it's been removed. // The single removal will bump the version to 5. this.resolver.Notify(new[] { silos[1] }); listener.WaitForVersion(5); Assert.Equal(3, listener.Silos.Count); Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys); Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]); AssertStatus(silos[1].SiloAddress, SiloStatus.Active); Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]); AssertStatus(silos[0].SiloAddress, SiloStatus.Dead); multiClusters = this.oracle.GetApproximateMultiClusterGateways(); Assert.Equal(2, multiClusters.Count); // Remove a silo and verify that it's been removed. this.resolver.Notify(new FabricSiloInfo[0]); listener.WaitForVersion(6); Assert.Equal(3, listener.Silos.Count); Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys); Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]); AssertStatus(silos[0].SiloAddress, SiloStatus.Dead); Assert.Equal(SiloStatus.Dead, listener.Silos[silos[1].SiloAddress]); AssertStatus(silos[1].SiloAddress, SiloStatus.Dead); multiClusters = this.oracle.GetApproximateMultiClusterGateways(); Assert.Equal(1, multiClusters.Count); Assert.Contains(this.siloDetails.SiloAddress, multiClusters); }