private async Task HeartBeatLoop()
{
await Task.Yield();
while (!_cancellationSource.IsCancellationRequested)
{
try
{
_logger.LogInformation("-------------Heartbeat [{time}]------------------",
DateTime.Now);
var members = _cluster.Members;
foreach (var member in members.ToList())
{
// This could be done in parallel. However, we're not in a hurry and the logs would get messy
if (member.MonitoringSuspended)
{
_logger.LogInformation(
$"Monitoring is suspended for {member} (probably shutting down).");
continue;
}
if (await CheckHeartBeatAsync(member))
{
if (member is IServerProcess serverProcess)
{
// Just to be save - in case they have been started previously.
_cluster.ServiceRegistrar?.Ensure(serverProcess);
}
}
}
foreach (IManagedProcess managedProcess in members.ToList())
{
if (!managedProcess.IsDueForRecycling)
{
continue;
}
ServiceRegistrar serviceRegistrar = _cluster.ServiceRegistrar;
await TryRecycle(managedProcess, serviceRegistrar);
}
}
catch (Exception exception)
{
_logger.LogError(exception, "Heartbeat loop failed");
}
await Task.Delay(_cluster.HeartBeatInterval);
}
}
public void Abort()
{
_heartbeat.ShutdownAsync();
foreach (var process in Members)
{
if (process is IServerProcess serverProcess)
{
ServiceRegistrar?.EnsureRemoved(serverProcess);
}
process.Kill();
}
}
private async Task <bool> TryStart([NotNull] IManagedProcess process)
{
bool success = await process.StartAsync();
if (!success)
{
process.StartupFailureCount++;
}
else if (process is IServerProcess serverProcess)
{
ServiceRegistrar?.Ensure(serverProcess);
}
return(success);
}
public async Task <bool> StartAsync()
{
_logger.LogInformation("Starting cluster...");
CheckRunningProcesses();
_logger.LogInformation("Initializing service registry...");
// Ensure KVS is running, if configured!
IKeyValueStore keyValueStore = await InitializeKeyValueStoreAsync(Members, _heartbeat);
ServiceRegistrar = new ServiceRegistrar(new ServiceRegistry(keyValueStore));
await _heartbeat.StartAsync();
return(true);
}
public static async Task <string> ShutDownAsync(
[NotNull] IManagedProcess process,
[CanBeNull] ServiceRegistrar serviceRegistrar,
TimeSpan maxShutDownTime)
{
string message = null;
if (process.IsKnownRunning)
{
process.MonitoringSuspended = true;
try
{
if (process is IServerProcess serverProcess)
{
serviceRegistrar?.EnsureRemoved(serverProcess);
}
bool isShutDown = false;
if (maxShutDownTime > TimeSpan.Zero)
{
isShutDown = await process.TryShutdownAsync(maxShutDownTime);
}
if (!isShutDown)
{
message =
$"Process has not shut down within {maxShutDownTime.TotalSeconds}s. We had to kill it.";
process.Kill();
}
}
finally
{
process.MonitoringSuspended = false;
}
}
return(message);
}
private async Task <bool> CareForUnavailable([NotNull] IManagedProcess process)
{
_logger.LogInformation("(Re-)starting process due to request time-out: {process}",
process);
if (process is IServerProcess serverProcess)
{
ServiceRegistrar?.EnsureRemoved(serverProcess);
}
if (process.StartupFailureCount > MemberMaxStartupRetries)
{
_logger.LogWarning("Startup retries have been exceeded. Not starting {process}",
process);
}
else
{
return(await TryStart(process));
}
return(true);
}
public async Task <bool> ShutdownAsync(TimeSpan timeout)
{
await _heartbeat.ShutdownAsync();
var shutDownResults = await Task.WhenAll(Members.Select(m =>
{
if (m is IServerProcess serverProcess)
{
ServiceRegistrar?.EnsureRemoved(serverProcess);
}
if (m.ClusterShutdownAction == ShutdownAction.Kill)
{
m.Kill();
}
//return m.TryShutdownAsync(timeout);
return(Task.FromResult(true));
}));
return(shutDownResults.All(r => r));
}