private async Task GivenSingleClient_ThenGetsAllResourcesAssigned()
{
// ARRANGE
string groupName = Guid.NewGuid().ToString();
await zkHelper.InitializeAsync("/rebalancer", TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(30));
await zkHelper.PrepareResourceGroupAsync(groupName, "res", 5);
List <string> expectedAssignedResources = new()
{
"res0",
"res1",
"res2",
"res3",
"res4"
};
// ACT
(RebalancerClient client, List <TestEvent> testEvents) = CreateClient();
await client.StartAsync(groupName, new ClientOptions { AutoRecoveryOnError = false });
await Task.Delay(TimeSpan.FromSeconds(10));
// ASSERT
Assert.Equal(1, testEvents.Count);
Assert.Equal(EventType.Assignment, testEvents[0].EventType);
Assert.True(ResourcesMatch(expectedAssignedResources, testEvents[0].Resources.ToList()));
await client.StopAsync(TimeSpan.FromSeconds(30));
}
private async Task GivenSixResourcesAndThreeClients_ThenEachClientGetsTwoResources()
{
// ARRANGE
string groupName = Guid.NewGuid().ToString();
await zkHelper.InitializeAsync("/rebalancer", TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(30));
await zkHelper.PrepareResourceGroupAsync(groupName, "res", 6);
// ACT
(RebalancerClient client1, List <TestEvent> testEvents1) = CreateClient();
(RebalancerClient client2, List <TestEvent> testEvents2) = CreateClient();
(RebalancerClient client3, List <TestEvent> testEvents3) = CreateClient();
await client1.StartAsync(groupName, new ClientOptions { AutoRecoveryOnError = false });
await client2.StartAsync(groupName, new ClientOptions { AutoRecoveryOnError = false });
await client3.StartAsync(groupName, new ClientOptions { AutoRecoveryOnError = false });
await Task.Delay(TimeSpan.FromSeconds(15));
// ASSERT
// check client 1
AssertAssigned(testEvents1, 2);
AssertAssigned(testEvents2, 2);
AssertAssigned(testEvents3, 2);
await client1.StopAsync();
await client2.StopAsync();
await client3.StopAsync();
}
private async Task RandomisedTest(RandomConfig config)
{
// ARRANGE
TestOutputLogger testLogger = new();
string groupName = Guid.NewGuid().ToString();
await zkHelper.InitializeAsync("/rebalancer", TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(30));
await zkHelper.PrepareResourceGroupAsync(groupName, "res", config.ResourceCount);
ResourceMonitor resourceMonitor = new();
List <int> resSuffixes = new();
for (int i = 0; i < config.ResourceCount; i++)
{
resourceMonitor.CreateResource($"res{i}");
resSuffixes.Add(i);
}
ClientOptions clientOptions = new()
{
AutoRecoveryOnError = true,
RestartDelay = TimeSpan.FromSeconds(10),
OnAssignmentDelay = config.OnAssignmentDelay
};
List <TestClient> clients = new();
for (int i = 0; i < config.ClientCount; i++)
{
clients.Add(new TestClient(resourceMonitor,
groupName,
clientOptions,
config.OnStartEventHandlerTime,
config.OnStopEventHandlerTime,
config.RandomiseEventHandlerTimes));
}
for (int i = 0; i < config.ClientCount; i++)
{
if (config.StartUpClientInterval.TotalMilliseconds > 0)
{
await Task.Delay(config.StartUpClientInterval);
}
await clients[i].StartAsync(testLogger);
}
await Task.Delay(TimeSpan.FromSeconds(30));
// ACT
Stopwatch sw = new();
sw.Start();
Random rand = new(Guid.NewGuid().GetHashCode());
int testCounter = 0;
while (sw.Elapsed < config.TestDuration)
{
testLogger.Info("TEST RUNNER", "Test " + testCounter);
// action == 0 -> add/remove a client
// action == 1 -> add/remove a resource
int action = rand.Next(2);
if (action == 0)
{
int clientIndex = rand.Next(config.ClientCount);
await clients[clientIndex].PerformActionAsync(testLogger);
}
else
{
// resAction == 0 && resources exist -> remove a resource
// else add a resource
int resAction = rand.Next(2);
if (resAction == 0 || !resSuffixes.Any())
{
int resSuffix = resSuffixes.Any() ? resSuffixes.Max() + 1 : 0;
resSuffixes.Add(resSuffix);
resourceMonitor.AddResource($"res{resSuffix}");
testLogger.Info("TEST RUNNER", "Adding a resource");
await zkHelper.AddResourceAsync(groupName, $"res{resSuffix}");
testLogger.Info("TEST RUNNER", "Added a resource");
}
else
{
int index = rand.Next(resSuffixes.Count);
int resSuffix = resSuffixes[index];
resSuffixes.RemoveAt(index);
resourceMonitor.RemoveResource($"res{resSuffix}");
testLogger.Info("TEST RUNNER", "Removing a resource");
await zkHelper.DeleteResourceAsync(groupName, $"res{resSuffix}");
testLogger.Info("TEST RUNNER", "Removed a resource");
}
}
// wait for the configured period of time before making asserts
// this gives the necessary time for rebalancing
TimeSpan currentTestInterval;
if (config.RandomiseInterval)
{
currentTestInterval =
TimeSpan.FromMilliseconds(rand.Next((int)config.MaxInterval.TotalMilliseconds));
}
else
{
currentTestInterval = config.MaxInterval;
}
await Task.Delay(currentTestInterval);
resourceMonitor.PrintEvents($"/home/jack/tmp/rebalancer-zk/test-{groupName}");
// check for double assignments. All test scenarios must check this. No matter
// what happens, we can never allow double assignments - ever
if (resourceMonitor.DoubleAssignmentsExist())
{
foreach (object violation in resourceMonitor.GetDoubleAssignments())
{
testLogger.Error("TEST RUNNER", violation.ToString());
}
}
Assert.False(resourceMonitor.DoubleAssignmentsExist());
// depending on the type of test, we'll ensure that all resources have been assigned
// some tests that have extremely short durations between events do not leave enough time
// for rebalancing and so do not perform this check. The conditional check will
// only perform this check when a long enough time period has been allowed for rebalancing to complete
if (config.CheckType == CheckType.FullCheck)
{
if (clients.Any(x => x.Started))
{
testLogger.Info("TEST RUNNER", "Perform all resources assigned check");
Assert.True(resourceMonitor.AllResourcesAssigned());
}
}
else if (config.CheckType == CheckType.ConditionalCheck)
{
if (testCounter % config.ConditionalCheckInterval == 0 && clients.Any(x => x.Started))
{
testLogger.Info("TEST RUNNER", "Grace period before all resources assigned check");
await Task.Delay(config.ConditionalCheckWaitPeriod);
testLogger.Info("TEST RUNNER", "Perform all resources assigned check");
Assert.True(resourceMonitor.AllResourcesAssigned());
}
}
testCounter++;
}
// clean up
for (int i = 0; i < config.ClientCount; i++)
{
await clients[i].StopAsync();
}
}