public ResourceManager(ZooKeeperService zooKeeperService,
ILogger logger,
OnChangeActions onChangeActions)
{
this.zooKeeperService = zooKeeperService;
this.logger = logger;
this.resources = new List <string>();
this.assignmentStatus = AssignmentStatus.NoAssignmentYet;
this.onChangeActions = onChangeActions;
}
private Task StartRoleTask(CancellationToken token,
OnChangeActions onChangeActions,
BlockingCollection <ClientEvent> clientEvents)
{
return(Task.Run(async() =>
{
while (!token.IsCancellationRequested && !clientEvents.IsAddingCompleted)
{
try
{
// take the most recent event, if multiple are queued up then we only need the latest
ClientEvent clientEvent = null;
while (clientEvents.Any())
{
try
{
clientEvent = clientEvents.Take(token);
}
catch (OperationCanceledException) { }
}
// if there was an event then call the appropriate role beahvaiour
if (clientEvent != null)
{
if (clientEvent.EventType == EventType.Coordinator)
{
//await this.coordinator.ExecuteCoordinatorRoleAsync(this.clientId,
// clientEvent,
// onChangeActions,
// token);
}
else
{
//await this.follower.ExecuteFollowerRoleAsync(this.clientId,
// clientEvent,
// onChangeActions,
// token);
}
}
else
{
await WaitFor(TimeSpan.FromSeconds(1), token);
}
}
catch (Exception ex)
{
this.logger.Error(ex);
await WaitFor(TimeSpan.FromSeconds(1), token);
}
}
}));
}
public async Task ExecuteCoordinatorRoleAsync(Guid coordinatorClientId,
ClientEvent clientEvent,
OnChangeActions onChangeActions,
CancellationToken token)
{
currentFencingToken = clientEvent.FencingToken;
Client self = await clientService.KeepAliveAsync(coordinatorClientId);
List <string> resourcesNow = (await resourceService.GetResourcesAsync(clientEvent.ResourceGroup))
.OrderBy(x => x)
.ToList();
List <Client> clientsNow = await GetLiveClientsAsync(clientEvent, coordinatorClientId);
List <Guid> clientIds = clientsNow.Select(x => x.ClientId).ToList();
clientIds.Add(coordinatorClientId);
if (clientsNow.Any(x => x.FencingToken > clientEvent.FencingToken))
{
clientEvent.CoordinatorToken.FencingTokenViolation = true;
return;
}
if (self.ClientStatus == ClientStatus.Terminated || stoppedDueToInternalErrorFlag)
{
stoppedDueToInternalErrorFlag = false;
await clientService.SetClientStatusAsync(coordinatorClientId, ClientStatus.Waiting);
logger.Debug(coordinatorClientId.ToString(),
"Status change: COORDINATOR was terminated due to an error");
await TriggerRebalancingAsync(coordinatorClientId, clientEvent, clientsNow, resourcesNow,
onChangeActions,
token);
}
else if (!resources.OrderBy(x => x).SequenceEqual(resourcesNow.OrderBy(x => x)))
{
logger.Debug(coordinatorClientId.ToString(),
$"Resource change: Old: {string.Join(",", resources.OrderBy(x => x))} New: {string.Join(",", resourcesNow.OrderBy(x => x))}");
await TriggerRebalancingAsync(coordinatorClientId, clientEvent, clientsNow, resourcesNow,
onChangeActions,
token);
}
else if (!clients.OrderBy(x => x).SequenceEqual(clientIds.OrderBy(x => x)))
{
logger.Debug(coordinatorClientId.ToString(),
$"Client change: Old: {string.Join(",", clients.OrderBy(x => x))} New: {string.Join(",", clientIds.OrderBy(x => x))}");
await TriggerRebalancingAsync(coordinatorClientId, clientEvent, clientsNow, resourcesNow,
onChangeActions,
token);
}
}
private void InvokeOnStop(OnChangeActions onChangeActions)
{
try
{
foreach (var onErrorAction in onChangeActions.OnStopActions)
{
onErrorAction.Invoke();
}
}
catch (Exception ex)
{
this.logger.Error(ex.ToString());
}
}
private void InvokeOnError(OnChangeActions onChangeActions, string message, bool autoRecoveryEnabled, Exception exception)
{
try
{
foreach (var onErrorAction in onChangeActions.OnErrorActions)
{
onErrorAction.Invoke(message, autoRecoveryEnabled, exception);
}
}
catch (Exception ex)
{
this.logger.Error(ex.ToString());
}
}
public async Task StartAsync(string resourceGroup,
OnChangeActions onChangeActions,
CancellationToken token,
ClientOptions clientOptions)
{
// just in case someone tries to start the client twice (with some concurrency)
lock (startLockObj)
{
if (this.started)
{
throw new RebalanserException("Client already started");
}
this.started = true;
}
this.resourceManager = new ResourceManager(this.zooKeeperService, this.logger, onChangeActions, this.rebalancingMode);
SetStateToNoSession();
this.resourceGroup = resourceGroup;
this.onStartDelay = clientOptions.OnAssignmentDelay;
mainTask = Task.Run(async() => await RunStateMachine(token, clientOptions));
await Task.Yield();
}
public async Task ExecuteCoordinatorRoleAsync(Guid coordinatorClientId,
ClientEvent clientEvent,
OnChangeActions onChangeActions,
CancellationToken token)
{
currentFencingToken = clientEvent.FencingToken;
var self = await clientService.KeepAliveAsync(coordinatorClientId);
var resourcesNow = (await resourceService.GetResourcesAsync(clientEvent.ResourceGroup)).OrderBy(x => x).ToList();
var clientsNow = await GetLiveClientsAsync(clientEvent, coordinatorClientId);
var clientIds = clientsNow.Select(x => x.ClientId).ToList();
clientIds.Add(coordinatorClientId);
if (clientsNow.Any(x => x.FencingToken > clientEvent.FencingToken))
{
clientEvent.CoordinatorToken.FencingTokenViolation = true;
return;
}
if (!resources.OrderBy(x => x).SequenceEqual(resourcesNow.OrderBy(x => x)))
{
logger.Debug($"Resource change: Old: {string.Join(",", resources.OrderBy(x => x))} New: {string.Join(",", resourcesNow.OrderBy(x => x))}");
await TriggerRebalancingAsync(coordinatorClientId, clientEvent, clientsNow, resourcesNow, onChangeActions, token);
}
else if (!clients.OrderBy(x => x).SequenceEqual(clientIds.OrderBy(x => x)))
{
logger.Debug($"Client change: Old: {string.Join(",", clients.OrderBy(x => x))} New: {string.Join(",", clientIds.OrderBy(x => x))}");
await TriggerRebalancingAsync(coordinatorClientId, clientEvent, clientsNow, resourcesNow, onChangeActions, token);
}
else
{
// no change, do nothing
}
}
public async Task StartAsync(string resourceGroup,
OnChangeActions onChangeActions,
CancellationToken parentToken,
ContextOptions contextOptions)
{
// just in case someone does some concurrency
lock (startLockObj)
{
if (this.started)
{
throw new RebalanserException("Context already started");
}
}
this.resourceGroup = resourceGroup;
await this.clientService.CreateClientAsync(this.resourceGroup, this.clientId);
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
mainTask = Task.Run(async() =>
{
while (!parentToken.IsCancellationRequested)
{
var childTaskCts = new CancellationTokenSource();
try
{
var clientEvents = new BlockingCollection <ClientEvent>();
var leaderElectionTask = StartLeadershipTask(childTaskCts.Token, clientEvents);
var roleTask = StartRoleTask(childTaskCts.Token, onChangeActions, clientEvents);
while (!parentToken.IsCancellationRequested &&
!leaderElectionTask.IsCompleted &&
!clientEvents.IsCompleted)
{
await Task.Delay(100);
}
// cancel child tasks
childTaskCts.Cancel();
if (parentToken.IsCancellationRequested)
{
logger.Info("Context shutting down due to cancellation");
}
else
{
if (leaderElectionTask.IsFaulted)
{
await NotifyOfErrorAsync(leaderElectionTask, "Shutdown due to leader election task fault", contextOptions.AutoRecoveryOnError, onChangeActions);
}
else if (roleTask.IsFaulted)
{
await NotifyOfErrorAsync(roleTask, "Shutdown due to coordinator/follower task fault", contextOptions.AutoRecoveryOnError, onChangeActions);
}
else
{
NotifyOfError(onChangeActions, "Unknown shutdown reason", contextOptions.AutoRecoveryOnError, null);
}
if (contextOptions.AutoRecoveryOnError)
{
await WaitFor(contextOptions.RestartDelay, parentToken);
}
else
{
break;
}
}
await leaderElectionTask;
await roleTask;
this.clientService.SetClientStatusAsync(this.clientId, ClientStatus.Terminated);
if (this.isCoordinator)
{
this.leaseService.RelinquishLeaseAsync(new RelinquishLeaseRequest()
{
ClientId = this.clientId,
FencingToken = this.coordinator.GetCurrentFencingToken(),
ResourceGroup = this.resourceGroup
});
}
}
catch (Exception ex)
{
NotifyOfError(onChangeActions, $"An unexpected error has caused shutdown. Automatic restart is set to {contextOptions.AutoRecoveryOnError}", contextOptions.AutoRecoveryOnError, ex);
if (contextOptions.AutoRecoveryOnError)
{
await WaitFor(contextOptions.RestartDelay, parentToken);
}
else
{
break;
}
}
}
});
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
private async Task InvokeOnErrorAsync(Task faultedTask, string message, bool autoRecoveryEnabled, OnChangeActions onChangeActions)
{
try
{
await faultedTask;
}
catch (Exception ex)
{
InvokeOnError(onChangeActions, message, autoRecoveryEnabled, ex);
}
}
private void NotifyOfError(OnChangeActions onChangeActions, string message, bool autoRecoveryEnabled, Exception exception)
{
InvokeOnError(onChangeActions, message, autoRecoveryEnabled, exception);
InvokeOnStop(onChangeActions);
}
private async Task NotifyOfErrorAsync(Task faultedTask, string message, bool autoRecoveryEnabled, OnChangeActions onChangeActions)
{
await InvokeOnErrorAsync(faultedTask, message, autoRecoveryEnabled, onChangeActions);
InvokeOnStop(onChangeActions);
}
private async Task TriggerRebalancingAsync(Guid coordinatorClientId,
ClientEvent clientEvent,
List <Client> clients,
List <string> resources,
OnChangeActions onChangeActions,
CancellationToken token)
{
logger.Info(coordinatorClientId.ToString(), "---------- Rebalancing triggered -----------");
// request stop of all clients
logger.Info(coordinatorClientId.ToString(), "COORDINATOR: Requested stop");
if (clients.Any())
{
ModifyClientResult result = await clientService.StopActivityAsync(clientEvent.FencingToken, clients);
if (result == ModifyClientResult.FencingTokenViolation)
{
clientEvent.CoordinatorToken.FencingTokenViolation = true;
return;
}
if (result == ModifyClientResult.Error)
{
logger.Error(coordinatorClientId.ToString(), "COORDINATOR: Rebalancing error");
return;
}
}
// stop all resource activity in local coordinator client
foreach (Action onStopAction in onChangeActions.OnStopActions)
{
onStopAction.Invoke();
}
// wait for all live clients to confirm stopped
bool allClientsWaiting = false;
List <Client> clientsNow = null;
while (!allClientsWaiting && !token.IsCancellationRequested)
{
await WaitFor(TimeSpan.FromSeconds(5), token);
clientsNow = await GetLiveClientsAsync(clientEvent, coordinatorClientId);
if (!clientsNow.Any())
{
allClientsWaiting = true;
}
else
{
allClientsWaiting = clientsNow.All(x => x.ClientStatus == ClientStatus.Waiting);
}
}
logger.Info(coordinatorClientId.ToString(), "COORDINATOR: Stop confirmed");
// assign resources first to coordinator then to other live clients
if (token.IsCancellationRequested)
{
return;
}
if (allClientsWaiting)
{
Queue <string> resourcesToAssign = new(resources);
List <ClientStartRequest> clientStartRequests = new();
int remainingClients = clientsNow.Count + 1;
int resourcesPerClient = Math.Max(1, resourcesToAssign.Count / remainingClients);
ClientStartRequest coordinatorRequest = new() { ClientId = coordinatorClientId };
while (coordinatorRequest.AssignedResources.Count < resourcesPerClient && resourcesToAssign.Any())
{
coordinatorRequest.AssignedResources.Add(resourcesToAssign.Dequeue());
}
clientStartRequests.Add(coordinatorRequest);
remainingClients--;
foreach (Client client in clientsNow)
{
resourcesPerClient = Math.Max(1, resourcesToAssign.Count / remainingClients);
ClientStartRequest request = new() { ClientId = client.ClientId };
while (request.AssignedResources.Count < resourcesPerClient && resourcesToAssign.Any())
{
request.AssignedResources.Add(resourcesToAssign.Dequeue());
}
clientStartRequests.Add(request);
remainingClients--;
}
if (token.IsCancellationRequested)
{
return;
}
logger.Info(coordinatorClientId.ToString(), "COORDINATOR: Resources assigned");
ModifyClientResult startResult =
await clientService.StartActivityAsync(clientEvent.FencingToken, clientStartRequests);
if (startResult == ModifyClientResult.FencingTokenViolation)
{
clientEvent.CoordinatorToken.FencingTokenViolation = true;
return;
}
if (startResult == ModifyClientResult.Error)
{
logger.Error(coordinatorClientId.ToString(), "COORDINATOR: Rebalancing error");
return;
}
store.SetResources(new SetResourcesRequest
{
AssignmentStatus = AssignmentStatus.ResourcesAssigned,
Resources = coordinatorRequest.AssignedResources
});
foreach (Action <IList <string> > onStartAction in onChangeActions.OnStartActions)
{
onStartAction.Invoke(coordinatorRequest.AssignedResources);
}
logger.Debug(coordinatorClientId.ToString(), "COORDINATOR: Local client started");
List <Guid> clientIds = clientsNow.Select(x => x.ClientId).ToList();
clientIds.Add(coordinatorClientId);
this.clients = clientIds;
this.resources = resources;
logger.Info(coordinatorClientId.ToString(), "---------- Activity Started -----------");
}
else
{
// log it
logger.Info(coordinatorClientId.ToString(), "!!!");
}
}
public async Task ExecuteFollowerRoleAsync(Guid followerClientId,
ClientEvent clientEvent,
OnChangeActions onChangeActions,
CancellationToken token)
{
var self = await clientService.KeepAliveAsync(followerClientId);
this.logger.Debug($"FOLLOWER : Keep Alive sent. Coordinator: {self.CoordinatorStatus} Client: {self.ClientStatus}");
if (self.CoordinatorStatus == CoordinatorStatus.StopActivity)
{
if (self.ClientStatus == ClientStatus.Active)
{
logger.Info("-------------- Stopping activity ---------------");
logger.Debug("FOLLOWER : Invoking on stop actions");
foreach (var stopAction in onChangeActions.OnStopActions)
{
stopAction.Invoke();
}
this.store.SetResources(new SetResourcesRequest()
{
AssignmentStatus = AssignmentStatus.AssignmentInProgress, Resources = new List <string>()
});
await clientService.SetClientStatusAsync(followerClientId, ClientStatus.Waiting);
logger.Info($"FOLLOWER : State= {self.ClientStatus} -> WAITING");
}
else
{
logger.Debug($"FOLLOWER : State= {self.ClientStatus}");
}
}
else if (self.CoordinatorStatus == CoordinatorStatus.ResourcesGranted)
{
if (self.ClientStatus == ClientStatus.Waiting)
{
if (self.AssignedResources.Any())
{
this.store.SetResources(new SetResourcesRequest()
{
AssignmentStatus = AssignmentStatus.ResourcesAssigned, Resources = self.AssignedResources
});
}
else
{
this.store.SetResources(new SetResourcesRequest()
{
AssignmentStatus = AssignmentStatus.NoResourcesAssigned, Resources = new List <string>()
});
}
await clientService.SetClientStatusAsync(followerClientId, ClientStatus.Active);
if (self.AssignedResources.Any())
{
logger.Info($"FOLLOWER : Granted resources={string.Join(",", self.AssignedResources)}");
}
else
{
logger.Info("FOLLOWER : No resources available to be assigned.");
}
foreach (var startAction in onChangeActions.OnStartActions)
{
startAction.Invoke();
}
logger.Info($"FOLLOWER : State={self.ClientStatus} -> ACTIVE");
logger.Info("-------------- Activity started ---------------");
}
else
{
logger.Debug($"FOLLOWER : State= {self.ClientStatus}");
}
}
}
public async Task StartAsync(string group, OnChangeActions onChangeActions, CancellationToken token, ContextOptions contextOptions)
{
throw new NotImplementedException();
}
