private async Task <bool> TryRecycle([NotNull] IManagedProcess managedProcess,
[CanBeNull] ServiceRegistrar serviceRegistrar)
{
_logger.LogInformation("The process {process} is due for recycling.", managedProcess);
int?ongoingRequests = await managedProcess.GetOngoingRequestCountAsync();
if (ongoingRequests > 0)
{
_logger.LogWarning(
"Process recycling is delayed due to one or more ongoing requests being processed.");
return(false);
}
// TODO: Send shutdown signal (that sets unhealthy = true and then waits another few seconds before shutting down to avoid races)
await ManagedProcessUtils.ShutDownAsync(managedProcess, serviceRegistrar,
TimeSpan.Zero);
// However, the advantage of killing is that we can re-start it straight away:
bool success = await managedProcess.StartAsync();
if (success &&
managedProcess is IServerProcess serverProcess)
{
serviceRegistrar?.Ensure(serverProcess);
}
return(success);
}
private async Task <bool> CareForUnhealthy(IManagedProcess process)
{
// TODO: Consider adding a new process while still shutting down, but only if ephemeral ports are used
_logger.LogInformation("(Re-)starting process with status 'not serving': {process}",
process);
string message =
await ManagedProcessUtils.ShutDownAsync(process, ServiceRegistrar,
MemberMaxShutdownTime);
if (!string.IsNullOrEmpty(message))
{
_logger.LogDebug(message);
}
if (process.StartupFailureCount > MemberMaxStartupRetries)
{
_logger.LogWarning("Startup retries have been exceeded. Not starting {process}",
process);
return(false);
}
return(await TryStart(process));
}
public async Task <bool> CheckHeartBeatAsync(IManagedProcess member)
{
var timeout = _cluster.MemberResponseTimeOut;
try
{
Stopwatch watch = Stopwatch.StartNew();
// Check for health first, it might be that the process is running but we have not
// started it.
var healthy = await TaskUtils.TimeoutAfter(member.IsServingAsync(), timeout);
if (healthy)
{
_logger.LogInformation(
"Heartbeat detected for {member}. All services are healthy [{milliseconds}ms].",
member, watch.ElapsedMilliseconds);
return(true);
}
if (member.IsKnownRunning)
{
_logger.LogWarning(
"Process is running for {member} but some services are un-healthy [{milliseconds}ms].",
member, watch.ElapsedMilliseconds);
return(await _unhealthyProcedure.Invoke(member));
}
_logger.LogWarning("Cluster member not running ({member}) [{milliseconds}ms].",
member, watch.ElapsedMilliseconds);
return(await _unavailableProcedure.Invoke(member));
}
catch (TimeoutException)
{
_logger.LogWarning("Heartbeat timed out for {member}.", member);
return(await _unavailableProcedure.Invoke(member));
}
catch (RpcException rpcException)
{
_logger.LogWarning(rpcException,
"RPC error in heartbeat detection for {member}: {exceptionMessage}",
member, rpcException.Message);
return(await _unavailableProcedure.Invoke(member));
}
catch (Exception e)
{
_logger.LogError(e, "Error in heartbeat detection for {member}: {exceptionMessage}",
member, e.Message);
return(false);
}
}
private async Task <bool> TryStart([NotNull] IManagedProcess process)
{
bool success = await process.StartAsync();
if (!success)
{
process.StartupFailureCount++;
}
else if (process is IServerProcess serverProcess)
{
ServiceRegistrar?.Ensure(serverProcess);
}
return(success);
}
public static async Task <string> ShutDownAsync(
[NotNull] IManagedProcess process,
[CanBeNull] ServiceRegistrar serviceRegistrar,
TimeSpan maxShutDownTime)
{
string message = null;
if (process.IsKnownRunning)
{
process.MonitoringSuspended = true;
try
{
if (process is IServerProcess serverProcess)
{
serviceRegistrar?.EnsureRemoved(serverProcess);
}
bool isShutDown = false;
if (maxShutDownTime > TimeSpan.Zero)
{
isShutDown = await process.TryShutdownAsync(maxShutDownTime);
}
if (!isShutDown)
{
message =
$"Process has not shut down within {maxShutDownTime.TotalSeconds}s. We had to kill it.";
process.Kill();
}
}
finally
{
process.MonitoringSuspended = false;
}
}
return(message);
}
private async Task <bool> CareForUnavailable([NotNull] IManagedProcess process)
{
_logger.LogInformation("(Re-)starting process due to request time-out: {process}",
process);
if (process is IServerProcess serverProcess)
{
ServiceRegistrar?.EnsureRemoved(serverProcess);
}
if (process.StartupFailureCount > MemberMaxStartupRetries)
{
_logger.LogWarning("Startup retries have been exceeded. Not starting {process}",
process);
}
else
{
return(await TryStart(process));
}
return(true);
}
public void Add(IManagedProcess managedProcess)
{
_managedProcesses.Add(managedProcess);
}