private async Task PerformLeaderCheckAsync() { bool checkComplete = false; while (!checkComplete) { try { int maxClientNumber = -1; string watchChild = string.Empty; ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync(); foreach (string childPath in clients.ClientPaths) { int siblingClientNumber = int.Parse(childPath.Substring(childPath.Length - 10, 10)); if (siblingClientNumber > maxClientNumber && siblingClientNumber < clientNumber) { watchChild = childPath; maxClientNumber = siblingClientNumber; } } if (maxClientNumber == -1) { events.Add(FollowerEvent.IsNewLeader); } else { watchSiblingPath = watchChild; siblingId = watchSiblingPath.Substring(watchChild.LastIndexOf("/", StringComparison.Ordinal)); await zooKeeperService.WatchSiblingNodeAsync(watchChild, this); logger.Info(clientId, $"Follower - Set a watch on sibling node {watchSiblingPath}"); } checkComplete = true; } catch (ZkNoEphemeralNodeWatchException) { // do nothing except wait, the next iteration will find // another client or it wil detect that it itself is the new leader await WaitFor(TimeSpan.FromSeconds(1)); } catch (ZkSessionExpiredException) { events.Add(FollowerEvent.SessionExpired); checkComplete = true; } catch (Exception ex) { logger.Error(clientId, "Follower - Failed looking for sibling to watch", ex); events.Add(FollowerEvent.PotentialInconsistentState); checkComplete = true; } } }
private async Task <RebalancingResult> RebalanceAsync(CancellationToken rebalancingToken) { Stopwatch sw = new(); sw.Start(); logger.Info(clientId, "Coordinator - Get clients and resources list"); ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync(); ResourcesZnode resources = await zooKeeperService.GetResourcesAsync(null, null); if (resources.Version != resourcesVersion) { throw new ZkStaleVersionException( "Resources znode version does not match expected value, indicates another client has been made coordinator and is executing a rebalancing."); } if (rebalancingToken.IsCancellationRequested) { return(RebalancingResult.Cancelled); } // if no resources were changed and there are more clients than resources then check // to see if rebalancing is necessary. If existing assignments are still valid then // a new client or the loss of a client with no assignments need not trigger a rebalancing if (!IsRebalancingRequired(clients, resources)) { logger.Info(clientId, "Coordinator - No rebalancing required. No resource change. No change to existing clients. More clients than resources."); return(RebalancingResult.Complete); } logger.Info(clientId, $"Coordinator - Assign resources ({string.Join(",", resources.Resources)}) to clients ({string.Join(",", clients.ClientPaths.Select(GetClientId))})"); Queue <string> resourcesToAssign = new(resources.Resources); List <ResourceAssignment> resourceAssignments = new(); int clientIndex = 0; while (resourcesToAssign.Any()) { resourceAssignments.Add(new ResourceAssignment { ClientId = GetClientId(clients.ClientPaths[clientIndex]), Resource = resourcesToAssign.Dequeue() }); clientIndex++; if (clientIndex >= clients.ClientPaths.Count) { clientIndex = 0; } } // write assignments back to resources znode resources.ResourceAssignments.Assignments = resourceAssignments; resourcesVersion = await zooKeeperService.SetResourcesAsync(resources); if (rebalancingToken.IsCancellationRequested) { return(RebalancingResult.Cancelled); } await store.InvokeOnStopActionsAsync(clientId, "Coordinator"); if (rebalancingToken.IsCancellationRequested) { return(RebalancingResult.Cancelled); } if (onStartDelay.Ticks > 0) { logger.Info(clientId, $"Coordinator - Delaying on start for {(int)onStartDelay.TotalMilliseconds}ms"); await WaitFor(onStartDelay, rebalancingToken); } if (rebalancingToken.IsCancellationRequested) { return(RebalancingResult.Cancelled); } List <string> leaderAssignments = resourceAssignments .Where(x => x.ClientId == clientId) .Select(x => x.Resource) .ToList(); await store.InvokeOnStartActionsAsync(clientId, "Coordinator", leaderAssignments, rebalancingToken, coordinatorToken); if (rebalancingToken.IsCancellationRequested) { return(RebalancingResult.Cancelled); } return(RebalancingResult.Complete); }
private async Task <StopPhaseResult> StopActivityPhaseAsync(CancellationToken rebalancingToken) { logger.Info(clientId, "Coordinator - Get active clients and resources"); ClientsZnode clients = await zooKeeperService.GetActiveClientsAsync(); List <string> followerIds = clients.ClientPaths.Select(GetClientId).Where(x => x != clientId).ToList(); ResourcesZnode resources = await zooKeeperService.GetResourcesAsync(null, null); logger.Info(clientId, $"Coordinator - {followerIds.Count} followers in scope and {resources.Resources.Count} resources in scope"); logger.Info(clientId, $"Coordinator - Assign resources ({string.Join(",", resources.Resources)}) to clients ({string.Join(",", clients.ClientPaths.Select(GetClientId))})"); if (resources.Version != resourcesVersion) { throw new ZkStaleVersionException( "Resources znode version does not match expected value, indicates another client has been made coordinator and is executing a rebalancing."); } if (rebalancingToken.IsCancellationRequested) { return(new StopPhaseResult(RebalancingResult.Cancelled)); } // if no resources were changed and there are more clients than resources then check // to see if rebalancing is necessary. If existing assignments are still valid then // a new client or the loss of a client with no assignments need not trigger a rebalancing if (!IsRebalancingRequired(clients, resources)) { logger.Info(clientId, "Coordinator - No rebalancing required. No resource change. No change to existing assigned clients. More clients than resources."); return(new StopPhaseResult(RebalancingResult.NotRequired)); } logger.Info(clientId, "Coordinator - Command followers to stop"); status.RebalancingStatus = RebalancingStatus.StopActivity; status.Version = await zooKeeperService.SetStatus(status); if (rebalancingToken.IsCancellationRequested) { return(new StopPhaseResult(RebalancingResult.Cancelled)); } await store.InvokeOnStopActionsAsync(clientId, "Coordinator"); // wait for confirmation that all followers have stopped or for time limit while (!rebalancingToken.IsCancellationRequested) { List <string> stopped = await zooKeeperService.GetStoppedAsync(); if (AreClientsStopped(followerIds, stopped)) { logger.Info(clientId, $"Coordinator - All {stopped.Count} in scope followers have stopped"); break; } // check that a client hasn't died mid-rebalancing, if so, trigger a new rebalancing and abort this one. // else wait and check again ClientsZnode latestClients = await zooKeeperService.GetActiveClientsAsync(); List <string> missingClients = GetMissing(followerIds, latestClients.ClientPaths); if (missingClients.Any()) { logger.Info(clientId, $"Coordinator - {missingClients.Count} followers have disappeared. Missing: {string.Join(",", missingClients)}. Triggering new rebalancing."); events.Add(CoordinatorEvent.RebalancingTriggered); return(new StopPhaseResult(RebalancingResult.Cancelled)); } List <string> pendingClientIds = GetMissing(followerIds, stopped); logger.Info(clientId, $"Coordinator - waiting for followers to stop: {string.Join(",", pendingClientIds)}"); await WaitFor(TimeSpan.FromSeconds(2)); // try again in 2s } if (rebalancingToken.IsCancellationRequested) { return(new StopPhaseResult(RebalancingResult.Cancelled)); } StopPhaseResult phaseResult = new(RebalancingResult.Complete) { ResourcesZnode = resources, ClientsZnode = clients, FollowerIds = followerIds }; return(phaseResult); }