private async Task TerminateAsync(string terminationReason, bool aborted, Exception abortException = null)
{
try
{
this.state = ClientInternalState.Terminated;
if (aborted)
{
if (abortException != null)
{
logger.Error(this.clientId, $"Client aborting due: {terminationReason}. Exception: {abortException}");
}
else
{
logger.Error(this.clientId, $"Client aborting due: {terminationReason}");
}
}
else
{
logger.Info(this.clientId, $"Client terminating due: {terminationReason}");
}
await this.zooKeeperService.CloseSessionAsync();
await this.resourceManager.InvokeOnStopActionsAsync(this.clientId, "No role");
if (aborted)
{
await this.resourceManager.InvokeOnAbortActionsAsync(this.clientId, $"The client has aborted due to: {terminationReason}", abortException);
}
logger.Info(this.clientId, "Client terminated");
}
catch (TerminateClientException e)
{
logger.Error(this.clientId, "Client termination failure during invocation of on stop/abort actions", e);
}
finally
{
if (aborted)
{
this.aborted = true;
}
this.started = false;
}
}
private async Task RunStateMachine(CancellationToken token, ClientOptions clientOptions)
{
while (this.state != ClientInternalState.Terminated)
{
if (token.IsCancellationRequested)
{
await TerminateAsync("cancellation", false);
}
try
{
switch (this.state)
{
case ClientInternalState.NoSession:
var established = await EstablishSessionAsync(token);
switch (established)
{
case NewSessionResult.Established:
this.state = ClientInternalState.NoClientNode;
break;
case NewSessionResult.TimeOut:
this.state = ClientInternalState.NoSession;
await WaitRandomTime(TimeSpan.FromSeconds(5));
break;
default:
this.state = ClientInternalState.Error;
break;
}
break;
case ClientInternalState.NoClientNode:
var created = await CreateClientNodeAsync();
if (created)
{
this.state = ClientInternalState.NoRole;
}
else
{
this.state = ClientInternalState.Error;
}
break;
case ClientInternalState.NoRole:
var epochAttained = await CacheEpochLocallyAsync();
if (!epochAttained)
{
await EvaluateTerminationAsync(token, clientOptions, "Couldn't read the current epoch.");
}
var(electionResult, lowerSiblingPath) = await DetermineLeadershipAsync();
switch (electionResult)
{
case ElectionResult.IsLeader:
this.state = ClientInternalState.IsLeader;
this.watchSiblingNodePath = string.Empty;
break;
case ElectionResult.IsFollower:
this.state = ClientInternalState.IsFollower;
this.watchSiblingNodePath = lowerSiblingPath;
break;
default:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an unknown state");
break;
}
break;
case ClientInternalState.IsLeader:
var coordinatorExitReason = await BecomeCoordinatorAsync(token);
switch (coordinatorExitReason)
{
case CoordinatorExitReason.NoLongerCoordinator:
SetStateToNoSession(); // need a new client node
break;
case CoordinatorExitReason.Cancelled:
await TerminateAsync("cancellation", false);
break;
case CoordinatorExitReason.SessionExpired:
SetStateToNoSession();
break;
case CoordinatorExitReason.PotentialInconsistentState:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered a potentially inconsistent state");
break;
case CoordinatorExitReason.FatalError:
await TerminateAsync("fatal error", true);
break;
default:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an unknown state");
break;
}
break;
case ClientInternalState.IsFollower:
var followerExitReason = await BecomeFollowerAsync(token);
switch (followerExitReason)
{
case FollowerExitReason.PossibleRoleChange:
this.state = ClientInternalState.NoRole;
break;
case FollowerExitReason.Cancelled:
await TerminateAsync("cancellation", false);
break;
case FollowerExitReason.SessionExpired:
SetStateToNoSession();
break;
case FollowerExitReason.FatalError:
await TerminateAsync("fatal error", true);
break;
case FollowerExitReason.PotentialInconsistentState:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an potential inconsistent state");
break;
default:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an unknown state");
break;
}
break;
case ClientInternalState.Error:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an error state");
break;
default:
await EvaluateTerminationAsync(token, clientOptions, "The client has entered an unknown state");
break;
}
}
catch (ZkSessionExpiredException)
{
this.logger.Info(this.clientId, "ZooKeeper session lost");
SetStateToNoSession();
}
catch (ZkOperationCancelledException)
{
await TerminateAsync("cancellation", false);
}
catch (TerminateClientException e)
{
await TerminateAsync("Fatal error", true, e);
}
catch (InconsistentStateException e)
{
await EvaluateTerminationAsync(token, clientOptions, "An error has caused that may have left the client in an inconsistent state.", e);
}
catch (Exception e)
{
await EvaluateTerminationAsync(token, clientOptions, "An unexpected error has been caught", e);
}
}
}
private void SetStateToNoSession()
{
ResetMutableState();
this.state = ClientInternalState.NoSession;
}
