private bool IsRebalancingRequired(ClientsZnode clients, ResourcesZnode resources)
{
// if this is the first rebalancing as coordinator or the last one was not successful then rebalancing is required
if (this.store.GetAssignmentStatus() == AssignmentStatus.NoAssignmentYet || !lastRebalancingResult.HasValue || lastRebalancingResult.Value != RebalancingResult.Complete)
return true;
// any change to resources requires a rebalancing
if (resources.HasResourceChange())
return true;
// given a client was either added or removed
// if there are less clients than resources then we require a rebalancing
if (clients.ClientPaths.Count < resources.Resources.Count)
return true;
// given we have an equal or greater number clients than resources
// if an existing client is currently assigned more than one resource we require a rebalancing
if (resources.ResourceAssignments.Assignments.GroupBy(x => x.ClientId).Any(x => x.Count() > 1))
return true;
// given all existing assignments are one client to one resource
// if any client for the existing assignments is no longer around then we require a rebalancing
var clientIds = clients.ClientPaths.Select(GetClientId).ToList();
foreach (var assignment in resources.ResourceAssignments.Assignments)
{
if (!clientIds.Contains(assignment.ClientId, StringComparer.Ordinal))
return true;
}
// otherwise no rebalancing is required
return false;
}
private async Task <RebalancingResult> AssignResourcesPhaseAsync(CancellationToken rebalancingToken,
ResourcesZnode resources,
ClientsZnode clients)
{
logger.Info(this.clientId, "Coordinator - Assign resources to clients");
var resourcesToAssign = new Queue <string>(resources.Resources);
var resourceAssignments = new List <ResourceAssignment>();
var clientIndex = 0;
while (resourcesToAssign.Any())
{
resourceAssignments.Add(new ResourceAssignment()
{
ClientId = GetClientId(clients.ClientPaths[clientIndex]),
Resource = resourcesToAssign.Dequeue()
});
clientIndex++;
if (clientIndex >= clients.ClientPaths.Count)
{
clientIndex = 0;
}
}
// write assignments back to resources znode
resources.ResourceAssignments.Assignments = resourceAssignments;
this.resourcesVersion = await this.zooKeeperService.SetResourcesAsync(resources);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
this.status.RebalancingStatus = RebalancingStatus.ResourcesGranted;
this.status.Version = await this.zooKeeperService.SetStatus(this.status);
if (this.onStartDelay.Ticks > 0)
{
this.logger.Info(this.clientId, $"Coordinator - Delaying on start for {(int)this.onStartDelay.TotalMilliseconds}ms");
await WaitFor(this.onStartDelay, rebalancingToken);
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
var leaderAssignments = resourceAssignments.Where(x => x.ClientId == this.clientId).Select(x => x.Resource).ToList();
await this.store.InvokeOnStartActionsAsync(this.clientId, "Coordinator", leaderAssignments, rebalancingToken, this.coordinatorToken);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
return(RebalancingResult.Complete);
}
private async Task PerformLeaderCheckAsync()
{
bool checkComplete = false;
while (!checkComplete)
{
try
{
int maxClientNumber = -1;
string watchChild = string.Empty;
ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync();
foreach (string childPath in clients.ClientPaths)
{
int siblingClientNumber = int.Parse(childPath.Substring(childPath.Length - 10, 10));
if (siblingClientNumber > maxClientNumber && siblingClientNumber < clientNumber)
{
watchChild = childPath;
maxClientNumber = siblingClientNumber;
}
}
if (maxClientNumber == -1)
{
events.Add(FollowerEvent.IsNewLeader);
}
else
{
watchSiblingPath = watchChild;
siblingId = watchSiblingPath.Substring(watchChild.LastIndexOf("/", StringComparison.Ordinal));
await zooKeeperService.WatchSiblingNodeAsync(watchChild, this);
logger.Info(clientId, $"Follower - Set a watch on sibling node {watchSiblingPath}");
}
checkComplete = true;
}
catch (ZkNoEphemeralNodeWatchException)
{
// do nothing except wait, the next iteration will find
// another client or it wil detect that it itself is the new leader
await WaitFor(TimeSpan.FromSeconds(1));
}
catch (ZkSessionExpiredException)
{
events.Add(FollowerEvent.SessionExpired);
checkComplete = true;
}
catch (Exception ex)
{
logger.Error(clientId, "Follower - Failed looking for sibling to watch", ex);
events.Add(FollowerEvent.PotentialInconsistentState);
checkComplete = true;
}
}
}
private async Task <RebalancingResult> RebalanceAsync(CancellationToken rebalancingToken)
{
Stopwatch sw = new();
sw.Start();
logger.Info(clientId, "Coordinator - Get clients and resources list");
ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync();
ResourcesZnode resources = await zooKeeperService.GetResourcesAsync(null, null);
if (resources.Version != resourcesVersion)
{
throw new ZkStaleVersionException(
"Resources znode version does not match expected value, indicates another client has been made coordinator and is executing a rebalancing.");
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
// if no resources were changed and there are more clients than resources then check
// to see if rebalancing is necessary. If existing assignments are still valid then
// a new client or the loss of a client with no assignments need not trigger a rebalancing
if (!IsRebalancingRequired(clients, resources))
{
logger.Info(clientId,
"Coordinator - No rebalancing required. No resource change. No change to existing clients. More clients than resources.");
return(RebalancingResult.Complete);
}
logger.Info(clientId,
$"Coordinator - Assign resources ({string.Join(",", resources.Resources)}) to clients ({string.Join(",", clients.ClientPaths.Select(GetClientId))})");
Queue <string> resourcesToAssign = new(resources.Resources);
List <ResourceAssignment> resourceAssignments = new();
int clientIndex = 0;
while (resourcesToAssign.Any())
{
resourceAssignments.Add(new ResourceAssignment
{
ClientId = GetClientId(clients.ClientPaths[clientIndex]), Resource = resourcesToAssign.Dequeue()
});
clientIndex++;
if (clientIndex >= clients.ClientPaths.Count)
{
clientIndex = 0;
}
}
// write assignments back to resources znode
resources.ResourceAssignments.Assignments = resourceAssignments;
resourcesVersion = await zooKeeperService.SetResourcesAsync(resources);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
await store.InvokeOnStopActionsAsync(clientId, "Coordinator");
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
if (onStartDelay.Ticks > 0)
{
logger.Info(clientId, $"Coordinator - Delaying on start for {(int)onStartDelay.TotalMilliseconds}ms");
await WaitFor(onStartDelay, rebalancingToken);
}
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
List <string> leaderAssignments = resourceAssignments
.Where(x => x.ClientId == clientId)
.Select(x => x.Resource)
.ToList();
await store.InvokeOnStartActionsAsync(clientId, "Coordinator", leaderAssignments, rebalancingToken,
coordinatorToken);
if (rebalancingToken.IsCancellationRequested)
{
return(RebalancingResult.Cancelled);
}
return(RebalancingResult.Complete);
}
private async Task <StopPhaseResult> StopActivityPhaseAsync(CancellationToken rebalancingToken)
{
logger.Info(clientId, "Coordinator - Get active clients and resources");
ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync();
List <string> followerIds = clients.ClientPaths.Select(GetClientId).Where(x => x != clientId).ToList();
ResourcesZnode resources = await zooKeeperService.GetResourcesAsync(null, null);
logger.Info(clientId,
$"Coordinator - {followerIds.Count} followers in scope and {resources.Resources.Count} resources in scope");
logger.Info(clientId,
$"Coordinator - Assign resources ({string.Join(",", resources.Resources)}) to clients ({string.Join(",", clients.ClientPaths.Select(GetClientId))})");
if (resources.Version != resourcesVersion)
{
throw new ZkStaleVersionException(
"Resources znode version does not match expected value, indicates another client has been made coordinator and is executing a rebalancing.");
}
if (rebalancingToken.IsCancellationRequested)
{
return(new StopPhaseResult(RebalancingResult.Cancelled));
}
// if no resources were changed and there are more clients than resources then check
// to see if rebalancing is necessary. If existing assignments are still valid then
// a new client or the loss of a client with no assignments need not trigger a rebalancing
if (!IsRebalancingRequired(clients, resources))
{
logger.Info(clientId,
"Coordinator - No rebalancing required. No resource change. No change to existing assigned clients. More clients than resources.");
return(new StopPhaseResult(RebalancingResult.NotRequired));
}
logger.Info(clientId, "Coordinator - Command followers to stop");
status.RebalancingStatus = RebalancingStatus.StopActivity;
status.Version = await zooKeeperService.SetStatus(status);
if (rebalancingToken.IsCancellationRequested)
{
return(new StopPhaseResult(RebalancingResult.Cancelled));
}
await store.InvokeOnStopActionsAsync(clientId, "Coordinator");
// wait for confirmation that all followers have stopped or for time limit
while (!rebalancingToken.IsCancellationRequested)
{
List <string> stopped = await zooKeeperService.GetStoppedAsync();
if (AreClientsStopped(followerIds, stopped))
{
logger.Info(clientId, $"Coordinator - All {stopped.Count} in scope followers have stopped");
break;
}
// check that a client hasn't died mid-rebalancing, if so, trigger a new rebalancing and abort this one.
// else wait and check again
ClientsZnode latestClients = await zooKeeperService.GetActiveClientsAsync();
List <string> missingClients = GetMissing(followerIds, latestClients.ClientPaths);
if (missingClients.Any())
{
logger.Info(clientId,
$"Coordinator - {missingClients.Count} followers have disappeared. Missing: {string.Join(",", missingClients)}. Triggering new rebalancing.");
events.Add(CoordinatorEvent.RebalancingTriggered);
return(new StopPhaseResult(RebalancingResult.Cancelled));
}
List <string> pendingClientIds = GetMissing(followerIds, stopped);
logger.Info(clientId,
$"Coordinator - waiting for followers to stop: {string.Join(",", pendingClientIds)}");
await WaitFor(TimeSpan.FromSeconds(2)); // try again in 2s
}
if (rebalancingToken.IsCancellationRequested)
{
return(new StopPhaseResult(RebalancingResult.Cancelled));
}
StopPhaseResult phaseResult = new(RebalancingResult.Complete)
{
ResourcesZnode = resources, ClientsZnode = clients, FollowerIds = followerIds
};
return(phaseResult);
}
