public async Task <ConcurrentBag <QualifiedService> > GetLoadQualifiedServices(
[NotNull] IEnumerable <ServiceLocation> serviceLocations,
TimeSpan workerResponseTimeout)
{
Stopwatch watch = Stopwatch.StartNew();
var result = new ConcurrentBag <QualifiedService>();
int failureCount = 0;
var getReportTasks = serviceLocations.Select(serviceLocation =>
TryAddLoadReport(serviceLocation, workerResponseTimeout, result));
var allReportRetrievals = await Task.WhenAll(getReportTasks);
watch.Stop();
ProcessUtils.EnsureThreadIdInName();
_logger.LogDebug("Received {loadReportCount} load reports in {seconds}ms.",
allReportRetrievals.Count(r => r), watch.ElapsedMilliseconds);
if (result.Count == 0 && allReportRetrievals.Any(r => r == false) &&
_lastException != null)
{
// Something more serious might be wrong and we cannot serve even one location
_logger.LogWarning(
"No service can be served AND {failureCount} exceptions occurred! Throwing last exception...",
failureCount);
throw _lastException;
}
return(result);
}
public override async Task <DiscoverServicesResponse> DiscoverTopServices(
DiscoverServicesRequest request, ServerCallContext context)
{
DiscoverServicesResponse response = null;
try
{
ProcessUtils.EnsureThreadIdInName();
Stopwatch watch = Stopwatch.StartNew();
response = new DiscoverServicesResponse();
IList <ServiceLocationMsg> result = await GetTopServiceLocationMessages(request);
response.ServiceLocations.AddRange(result);
_logger.LogDebug(
"Returning {count} service location(s) [{time}ms]: {serviceLocations}",
result.Count,
watch.ElapsedMilliseconds,
string.Concat(result.Select(s => $"{s.HostName}:{s.Port}, ")));
}
catch (Exception e)
{
_logger.LogError(e, "Error discovering service {serviceName}", request.ServiceName);
SetUnhealthy();
}
return(response);
}
private async Task <bool> TryAddLoadReport(
[NotNull] ServiceLocation serviceLocation,
TimeSpan workerResponseTimeout,
[NotNull] ConcurrentBag <QualifiedService> loadReportsByService)
{
Channel channel = GetChannel(serviceLocation);
try
{
// Check for health first, it might be that the process is running but we have not
// started it.
if (!await IsHealthy(serviceLocation, channel, workerResponseTimeout))
{
return(false);
}
LoadReportResponse loadReportResponse =
await GetLoadReport(serviceLocation, channel, workerResponseTimeout);
ProcessUtils.EnsureThreadIdInName();
if (loadReportResponse.ServerStats.RequestCapacity == 0)
{
_logger.LogDebug(
"Service location {serviceLocation} reports 0 capacity. It is ignored.",
serviceLocation);
return(false);
}
var qualifiedLocation =
new QualifiedService(serviceLocation, loadReportResponse.ServerStats)
{
KnownLoadRate = loadReportResponse.KnownLoadRate
};
loadReportsByService.Add(qualifiedLocation);
return(true);
}
catch (TimeoutException)
{
_logger.LogDebug(
"Service location {serviceLocation} took longer than {timeout}s. It is ignored.",
serviceLocation, workerResponseTimeout.TotalSeconds);
}
catch (Exception e)
{
_logger.LogWarning(e,
"Error checking service health / load report for {serviceLocation}",
serviceLocation);
_lastException = e;
}
return(false);
}
public async Task <ConcurrentBag <QualifiedService> > GetHealthyServiceLocations(
[NotNull] IEnumerable <ServiceLocation> allServices,
TimeSpan workerResponseTimeout,
int maxCount = -1)
{
var result = new ConcurrentBag <QualifiedService>();
int healthyCount = 0;
Stopwatch watch = Stopwatch.StartNew();
// NOTE: This could be done more in parallel, however it would not be possible to
// just break after maxCount and the overall resource consumption would go up.
foreach (var serviceLocation in allServices)
{
if (maxCount > 0 && healthyCount >= maxCount)
{
return(result);
}
bool isHealthy = await IsHealthy(serviceLocation, workerResponseTimeout);
result.Add(new QualifiedService(serviceLocation, isHealthy));
if (isHealthy)
{
healthyCount++;
}
}
ProcessUtils.EnsureThreadIdInName();
if (result.Count == 0 && _lastException != null)
{
// Something more serious might be wrong and we cannot serve even one location
_logger.LogWarning(
"No service can be served AND one or more exceptions occurred! Throwing last exception...");
throw _lastException;
}
watch.Stop();
_logger.LogDebug("Found {healthyCount} healthy services in {seconds}ms.",
result.Count, watch.ElapsedMilliseconds);
return(result);
}