public async Task <bool> CheckHeartBeatAsync(IManagedProcess member) { var timeout = _cluster.MemberResponseTimeOut; try { Stopwatch watch = Stopwatch.StartNew(); // Check for health first, it might be that the process is running but we have not // started it. var healthy = await TaskUtils.TimeoutAfter(member.IsServingAsync(), timeout); if (healthy) { _logger.LogInformation( "Heartbeat detected for {member}. All services are healthy [{milliseconds}ms].", member, watch.ElapsedMilliseconds); return(true); } if (member.IsKnownRunning) { _logger.LogWarning( "Process is running for {member} but some services are un-healthy [{milliseconds}ms].", member, watch.ElapsedMilliseconds); return(await _unhealthyProcedure.Invoke(member)); } _logger.LogWarning("Cluster member not running ({member}) [{milliseconds}ms].", member, watch.ElapsedMilliseconds); return(await _unavailableProcedure.Invoke(member)); } catch (TimeoutException) { _logger.LogWarning("Heartbeat timed out for {member}.", member); return(await _unavailableProcedure.Invoke(member)); } catch (RpcException rpcException) { _logger.LogWarning(rpcException, "RPC error in heartbeat detection for {member}: {exceptionMessage}", member, rpcException.Message); return(await _unavailableProcedure.Invoke(member)); } catch (Exception e) { _logger.LogError(e, "Error in heartbeat detection for {member}: {exceptionMessage}", member, e.Message); return(false); } }