public void SiloEventuallyBecomesDead()
{
var now = new[] { DateTime.UtcNow };
var options = new ServiceFabricMembershipOptions();
var monitor = new UnknownSiloMonitor(new OptionsWrapper <ServiceFabricMembershipOptions>(options), this.Logger)
{
GetDateTime = () => now[0]
};
var knownSilos = new Dictionary <SiloAddress, SiloStatus>
{
[NewSiloAddress("2.2.2.2", 9030, 2)] = SiloStatus.Active
};
// Report a silo as having an unknown status.
var deadSilo = NewSiloAddress("1.1.1.1", 9030, 1000);
monitor.ReportUnknownSilo(deadSilo);
// No time has passed and the set of known silos contains no information about the reported silo,
// therefore we expect no reports even after running multiple times.
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
// Advance passed the expiration time.
now[0] += options.UnknownSiloRemovalPeriod + TimeSpan.FromMilliseconds(1);
// The silo should be declared dead by the monitor.
Assert.Contains(deadSilo, monitor.DetermineDeadSilos(knownSilos));
}
public FabricMembershipOracleTests(ITestOutputHelper output)
{
this.Output = output;
this.siloDetails = new MockSiloDetails
{
Name = Guid.NewGuid().ToString("N"),
SiloAddress = SiloAddressUtils.NewLocalSiloAddress(SiloAddress.AllocateNewGeneration())
};
this.resolver = new MockResolver();
var globalConfig = new ClusterConfiguration().Globals;
globalConfig.MaxMultiClusterGateways = 2;
globalConfig.ClusterId = "MegaGoodCluster";
this.fabricMembershipOptions = new ServiceFabricMembershipOptions();
this.unknownSiloMonitor = new UnknownSiloMonitor(
new OptionsWrapper <ServiceFabricMembershipOptions>(this.fabricMembershipOptions),
new TestOutputLogger <UnknownSiloMonitor>(this.Output));
this.oracle = new FabricMembershipOracle(
this.siloDetails,
globalConfig,
this.resolver,
new NullLogger <FabricMembershipOracle>(),
this.unknownSiloMonitor);
}
public FabricMembershipOracleTests(ITestOutputHelper output)
{
this.Output = output;
this.siloDetails = new MockSiloDetails
{
Name = Guid.NewGuid().ToString("N"),
SiloAddress = SiloAddressUtils.NewLocalSiloAddress(SiloAddress.AllocateNewGeneration())
};
this.resolver = new MockResolver();
var multiClusterOptions = new MultiClusterOptions
{
HasMultiClusterNetwork = true,
MaxMultiClusterGateways = 2
};
this.fabricClusteringOptions = new ServiceFabricClusteringOptions();
this.unknownSiloMonitor = new UnknownSiloMonitor(
new OptionsWrapper <ServiceFabricClusteringOptions>(this.fabricClusteringOptions),
new TestOutputLogger <UnknownSiloMonitor>(this.Output));
this.oracle = new FabricMembershipOracle(
this.siloDetails,
this.resolver,
new NullLogger <FabricMembershipOracle>(),
this.unknownSiloMonitor,
Options.Create(multiClusterOptions));
}
public void SiloDeclaredDeadWhenSupersededOnSameEndpoint()
{
var now = new[] { DateTime.UtcNow };
var options = new ServiceFabricMembershipOptions();
var monitor = new UnknownSiloMonitor(new OptionsWrapper <ServiceFabricMembershipOptions>(options), this.Logger)
{
GetDateTime = () => now[0]
};
var knownSilos = new Dictionary <SiloAddress, SiloStatus>();
// Report a silo as having an unknown status.
var deadSilo = NewSiloAddress("1.1.1.1", 9030, 1000);
monitor.ReportUnknownSilo(deadSilo);
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
// Create a silo with the same IP and port but a lower generation.
var predecessorSilo = NewSiloAddress("1.1.1.1", 9030, 500);
knownSilos[predecessorSilo] = SiloStatus.Active;
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
knownSilos[predecessorSilo] = SiloStatus.Dead;
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
// Create a silo with the same IP and port but a higher generation.
var supersedingSilo = NewSiloAddress("1.1.1.1", 9030, 2000);
// A status of None is equivalent to no status, so no declarations should be made.
knownSilos[supersedingSilo] = SiloStatus.None;
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
// The silo should be declared dead by the monitor even if the newer silo is dead.
knownSilos[supersedingSilo] = SiloStatus.Dead;
Assert.Contains(deadSilo, monitor.DetermineDeadSilos(knownSilos));
// The silo should be declared dead by the monitor.
knownSilos[supersedingSilo] = SiloStatus.Active;
Assert.Contains(deadSilo, monitor.DetermineDeadSilos(knownSilos));
}
public void SiloNotDeclaredDeadWhenSupersededOnSameAddressDifferentPort()
{
var now = new[] { DateTime.UtcNow };
var options = new ServiceFabricMembershipOptions();
var monitor = new UnknownSiloMonitor(new OptionsWrapper <ServiceFabricMembershipOptions>(options), this.Logger)
{
GetDateTime = () => now[0]
};
var deadSilo = NewSiloAddress("1.1.1.1", 1111, 1000);
var supersedingSilo = NewSiloAddress("1.1.1.1", 9999, 2000);
var knownSilos = new Dictionary <SiloAddress, SiloStatus>
{
[supersedingSilo] = SiloStatus.Active
};
// Report a silo as having an unknown status.
monitor.ReportUnknownSilo(deadSilo);
// The silo is not declared dead.
Assert.Empty(monitor.DetermineDeadSilos(knownSilos));
}