private async Task <RebalancingResult> AssignResourcesPhaseAsync(CancellationToken rebalancingToken,
ResourcesZnode resources,
ClientsZnode clients)
{
logger.Info(clientId, "Coordinator - Assign resources to clients");
Queue <string> resourcesToAssign = new(resources.Resources);
List <ResourceAssignment> resourceAssignments = new();
int clientIndex = 0;
while (resourcesToAssign.Any())
{
resourceAssignments.Add(new ResourceAssignment
{
ClientId = GetClientId(clients.ClientPaths[clientIndex]), Resource = resourcesToAssign.Dequeue()
});
clientIndex++;
if (clientIndex >= clients.ClientPaths.Count)
{
clientIndex = 0;
}
}
// write assignments back to resources znode
resources.ResourceAssignments.Assignments = resourceAssignments;
resourcesVersion = await zooKeeperService.SetResourcesAsync(resources);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
status.RebalancingStatus = RebalancingStatus.ResourcesGranted;
status.Version = await zooKeeperService.SetStatus(status);
if (onStartDelay.Ticks > 0)
{
logger.Info(clientId, $"Coordinator - Delaying on start for {(int)onStartDelay.TotalMilliseconds}ms");
await WaitFor(onStartDelay, rebalancingToken);
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
List <string> leaderAssignments =
resourceAssignments.Where(x => x.ClientId == clientId).Select(x => x.Resource).ToList();
await store.InvokeOnStartActionsAsync(clientId, "Coordinator", leaderAssignments, rebalancingToken,
coordinatorToken);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
return(RebalancingResult.Complete);
}
private async Task <RebalancingResult> RebalanceAsync(CancellationToken rebalancingToken)
{
Stopwatch sw = new();
sw.Start();
logger.Info(clientId, "Coordinator - Get clients and resources list");
ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync();
ResourcesZnode resources = await zooKeeperService.GetResourcesAsync(null, null);
if (resources.Version != resourcesVersion)
{
throw new ZkStaleVersionException(
"Resources znode version does not match expected value, indicates another client has been made coordinator and is executing a rebalancing.");
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
// if no resources were changed and there are more clients than resources then check
// to see if rebalancing is necessary. If existing assignments are still valid then
// a new client or the loss of a client with no assignments need not trigger a rebalancing
if (!IsRebalancingRequired(clients, resources))
{
logger.Info(clientId,
"Coordinator - No rebalancing required. No resource change. No change to existing clients. More clients than resources.");
return(RebalancingResult.Complete);
}
logger.Info(clientId,
$"Coordinator - Assign resources ({string.Join(",", resources.Resources)}) to clients ({string.Join(",", clients.ClientPaths.Select(GetClientId))})");
Queue <string> resourcesToAssign = new(resources.Resources);
List <ResourceAssignment> resourceAssignments = new();
int clientIndex = 0;
while (resourcesToAssign.Any())
{
resourceAssignments.Add(new ResourceAssignment
{
ClientId = GetClientId(clients.ClientPaths[clientIndex]), Resource = resourcesToAssign.Dequeue()
});
clientIndex++;
if (clientIndex >= clients.ClientPaths.Count)
{
clientIndex = 0;
}
}
// write assignments back to resources znode
resources.ResourceAssignments.Assignments = resourceAssignments;
resourcesVersion = await zooKeeperService.SetResourcesAsync(resources);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
await store.InvokeOnStopActionsAsync(clientId, "Coordinator");
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
if (onStartDelay.Ticks > 0)
{
logger.Info(clientId, $"Coordinator - Delaying on start for {(int)onStartDelay.TotalMilliseconds}ms");
await WaitFor(onStartDelay, rebalancingToken);
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
List <string> leaderAssignments = resourceAssignments
.Where(x => x.ClientId == clientId)
.Select(x => x.Resource)
.ToList();
await store.InvokeOnStartActionsAsync(clientId, "Coordinator", leaderAssignments, rebalancingToken,
coordinatorToken);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
return(RebalancingResult.Complete);
}