public void RegisterMachineTransitionTests()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
// During transition, all machines will be added forcefully without regard for the consistency of the data
// structure
clusterState = clusterState.ForceRegisterMachine(new MachineId(3), MachineLocation.Create("A", 0), nowUtc);
clusterState.NextMachineId.Should().Be(4);
clusterState = clusterState.ForceRegisterMachine(new MachineId(8), MachineLocation.Create("B", 0), nowUtc);
clusterState.NextMachineId.Should().Be(9);
clusterState = clusterState.ForceRegisterMachine(new MachineId(16), MachineLocation.Create("C", 0), nowUtc);
clusterState.NextMachineId.Should().Be(17);
clusterState = clusterState.ForceRegisterMachine(new MachineId(20), MachineLocation.Create("D", 0), nowUtc);
clusterState.NextMachineId.Should().Be(21);
clusterState = clusterState.ForceRegisterMachine(new MachineId(23), MachineLocation.Create("D", 0), nowUtc);
clusterState.NextMachineId.Should().Be(24);
// After transition, adding proceeds as usual, by appending to the end basically
MachineId n1Id;
(clusterState, n1Id) = clusterState.RegisterMachine(MachineLocation.Create("Machine Gets Added After Transition", 0), nowUtc);
n1Id.Index.Should().Be(24);
clusterState.NextMachineId.Should().Be(25);
}
public void RecomputeChangesStatesAsExpected()
{
var clusterState = new ClusterStateMachine();
var cfg = new ClusterStateRecomputeConfiguration()
{
// Force recompute to run
RecomputeFrequency = TimeSpan.Zero,
};
var nowUtc = _clock.UtcNow;
// We want to test all possible state machine transitions. To do so, we generate a very specific instance
// of cluster state meant to transition the way we expect instead of actually simulating each possible
// branch.
var n1 = MachineLocation.Create("node2", 0);
MachineId n1Id;
(clusterState, n1Id) = clusterState.ForceRegisterMachineWithState(n1, nowUtc, MachineState.DeadUnavailable);
var n2 = MachineLocation.Create("node3", 0);
MachineId n2Id;
(clusterState, n2Id) = clusterState.ForceRegisterMachineWithState(n2, nowUtc, MachineState.DeadExpired);
var n3 = MachineLocation.Create("node4", 0);
MachineId n3Id;
(clusterState, n3Id) = clusterState.ForceRegisterMachineWithState(n3, nowUtc - cfg.ActiveToDeadExpiredInterval, MachineState.Open);
var n4 = MachineLocation.Create("node5", 0);
MachineId n4Id;
(clusterState, n4Id) = clusterState.ForceRegisterMachineWithState(n4, nowUtc - cfg.ActiveToClosedInterval, MachineState.Open);
var n5 = MachineLocation.Create("node6", 0);
MachineId n5Id;
(clusterState, n5Id) = clusterState.ForceRegisterMachineWithState(n5, nowUtc, MachineState.Open);
var n6 = MachineLocation.Create("node7", 0);
MachineId n6Id;
(clusterState, n6Id) = clusterState.ForceRegisterMachineWithState(n6, nowUtc - cfg.ClosedToDeadExpiredInterval, MachineState.Closed);
var n7 = MachineLocation.Create("node8", 0);
MachineId n7Id;
(clusterState, n7Id) = clusterState.ForceRegisterMachineWithState(n7, nowUtc, MachineState.Closed);
clusterState = clusterState.Recompute(cfg, nowUtc);
clusterState.GetStatus(n1Id).ThrowIfFailure().State.Should().Be(MachineState.DeadUnavailable);
clusterState.GetStatus(n2Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n3Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n4Id).ThrowIfFailure().State.Should().Be(MachineState.Closed);
clusterState.GetStatus(n5Id).ThrowIfFailure().State.Should().Be(MachineState.Open);
clusterState.GetStatus(n6Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n7Id).ThrowIfFailure().State.Should().Be(MachineState.Closed);
}
public Task SimpleGetUpdatesTests()
{
return(RunTest(async(context, clock, storage) =>
{
var m1 = await storage.RegisterMachineAsync(context, MachineLocation.Create("A", 1)).ThrowIfFailureAsync();
var m2 = await storage.RegisterMachineAsync(context, MachineLocation.Create("B", 1)).ThrowIfFailureAsync();
var r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest()
{
MaxMachineId = 0,
}).ThrowIfFailureAsync();
r.MaxMachineId.Should().Be(2);
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m1.Id, m1.Location)).Should().BeTrue();
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue();
r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest()
{
MaxMachineId = 1,
}).ThrowIfFailureAsync();
r.MaxMachineId.Should().Be(2);
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue();
}));
}
public Task SimpleHeartbeatTest()
{
return(RunTest(async(context, clock, storage) =>
{
var m1 = await storage.RegisterMachineAsync(context, MachineLocation.Create("A", 1)).ThrowIfFailureAsync();
var m2 = await storage.RegisterMachineAsync(context, MachineLocation.Create("B", 1)).ThrowIfFailureAsync();
// Transition m1 to Closed
var r = await storage.HeartbeatAsync(context, new HeartbeatMachineRequest()
{
MachineId = m1.Id,
Location = m1.Location,
Name = m1.Location.Path,
HeartbeatTime = clock.UtcNow,
DeclaredMachineState = MachineState.Closed,
}).ThrowIfFailureAsync();
r.PriorState.Should().Be(ClusterStateMachine.InitialState);
r.ClosedMachines.Value.Contains(m1.Id).Should().BeTrue();
r.InactiveMachines.Value.IsEmpty.Should().BeTrue();
// Transition m1 to Open
r = await storage.HeartbeatAsync(context, new HeartbeatMachineRequest()
{
MachineId = m1.Id,
Location = m1.Location,
Name = m1.Location.Path,
HeartbeatTime = clock.UtcNow,
DeclaredMachineState = MachineState.Open,
}).ThrowIfFailureAsync();
r.PriorState.Should().Be(MachineState.Closed);
r.ClosedMachines.Value.IsEmpty.Should().BeTrue();
r.InactiveMachines.Value.IsEmpty.Should().BeTrue();
// Transition m1 to DeadUnavailable
r = await storage.HeartbeatAsync(context, new HeartbeatMachineRequest()
{
MachineId = m1.Id,
Location = m1.Location,
Name = m1.Location.Path,
HeartbeatTime = clock.UtcNow,
DeclaredMachineState = MachineState.DeadUnavailable,
}).ThrowIfFailureAsync();
r.PriorState.Should().Be(MachineState.Open);
r.ClosedMachines.Value.IsEmpty.Should().BeTrue();
r.InactiveMachines.Value.Contains(m1.Id).Should().BeTrue();
// Transition m1 to Closed
r = await storage.HeartbeatAsync(context, new HeartbeatMachineRequest()
{
MachineId = m1.Id,
Location = m1.Location,
Name = m1.Location.Path,
HeartbeatTime = clock.UtcNow,
DeclaredMachineState = MachineState.Closed,
}).ThrowIfFailureAsync();
r.PriorState.Should().Be(MachineState.DeadUnavailable);
r.ClosedMachines.Value.Contains(m1.Id).Should().BeTrue();
r.InactiveMachines.Value.IsEmpty.Should().BeTrue();
// Transition m1 to Open
r = await storage.HeartbeatAsync(context, new HeartbeatMachineRequest()
{
MachineId = m1.Id,
Location = m1.Location,
Name = m1.Location.Path,
HeartbeatTime = clock.UtcNow,
DeclaredMachineState = MachineState.Open,
}).ThrowIfFailureAsync();
r.PriorState.Should().Be(MachineState.Closed);
r.ClosedMachines.Value.IsEmpty.Should().BeTrue();
r.InactiveMachines.Value.IsEmpty.Should().BeTrue();
}));
}