/// <summary>
/// Start the execution of a work item.
/// </summary>
/// <param name="operationLogger">Instance of <see cref="IOperationLogger"/>.</param>
/// <param name="monotonicTimer">Instance of <see cref="IMonotonicTimer"/>.</param>
internal async void Start(IOperationLogger <RecurrentActivityManager> operationLogger, IMonotonicTimer monotonicTimer)
{
Contracts.Check(_isExecuting == false, "Expected work item was not executing");
_isExecuting = true;
LastExecution = monotonicTimer.CurrentTime;
try
{
await operationLogger.ExecuteAsync(
OperationName,
async() =>
{
await Func(_cts.Token);
});
}
catch (Exception e) when(!e.IsFatal())
{
// Exceptions are not expected, but are already logged above and service must remain executing.
}
finally
{
_isExecuting = false;
}
}
/// <inheritdoc/>
public async Task InvokeAsync(HttpContext context)
{
Contracts.CheckValue(context, nameof(context));
var aspNetCoreEndpoint = context.GetEndpoint();
if (aspNetCoreEndpoint == null)
{
throw new ReverseProxyException($"ASP .NET Core Endpoint wasn't set for the current request. This is a coding defect.");
}
var routeConfig = aspNetCoreEndpoint.Metadata.GetMetadata <RouteConfig>();
if (routeConfig == null)
{
throw new ReverseProxyException($"ASP .NET Core Endpoint is missing {typeof(RouteConfig).FullName} metadata. This is a coding defect.");
}
var backend = routeConfig.BackendOrNull;
if (backend == null)
{
throw new ReverseProxyException($"Route has no backend information.");
}
var dynamicState = backend.DynamicState.Value;
if (dynamicState == null)
{
throw new ReverseProxyException($"Route has no up to date information on its backend '{backend.BackendId}'. Perhaps the backend hasn't been probed yet? This can happen when a new backend is added but isn't ready to serve traffic yet.");
}
// TODO: Set defaults properly
BackendConfig.BackendLoadBalancingOptions loadBalancingOptions = default;
var backendConfig = backend.Config.Value;
if (backendConfig != null)
{
loadBalancingOptions = backendConfig.LoadBalancingOptions;
}
var endpoint = _operationLogger.Execute(
"ReverseProxy.PickEndpoint",
() => _loadBalancer.PickEndpoint(dynamicState.HealthyEndpoints, dynamicState.AllEndpoints, in loadBalancingOptions));
if (endpoint == null)
{
throw new ReverseProxyException($"No available endpoints.");
}
var endpointConfig = endpoint.Config.Value;
if (endpointConfig == null)
{
throw new ReverseProxyException($"Chosen endpoint has no configs set: '{endpoint.EndpointId}'");
}
// TODO: support StripPrefix and other url transformations
var targetUrl = BuildOutgoingUrl(context, endpointConfig.Address);
_logger.LogInformation($"Proxying to {targetUrl}");
var targetUri = new Uri(targetUrl, UriKind.Absolute);
using (var shortCts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted))
{
// TODO: Configurable timeout, measure from request start, make it unit-testable
shortCts.CancelAfter(TimeSpan.FromSeconds(30));
// TODO: Retry against other endpoints
try
{
// TODO: Apply caps
backend.ConcurrencyCounter.Increment();
endpoint.ConcurrencyCounter.Increment();
// TODO: Duplex channels should not have a timeout (?), but must react to Proxy force-shutdown signals.
var longCancellation = context.RequestAborted;
var proxyTelemetryContext = new ProxyTelemetryContext(
backendId: backend.BackendId,
routeId: routeConfig.Route.RouteId,
endpointId: endpoint.EndpointId);
await _operationLogger.ExecuteAsync(
"ReverseProxy.Proxy",
() => _httpProxy.ProxyAsync(context, targetUri, backend.ProxyHttpClientFactory, proxyTelemetryContext, shortCancellation: shortCts.Token, longCancellation: longCancellation));
}
finally
{
endpoint.ConcurrencyCounter.Decrement();
backend.ConcurrencyCounter.Decrement();
}
}
}
/// <summary>
/// A probe attempt to one endpoint.
/// </summary>
private async Task ProbeEndpointAsync(EndpointInfo endpoint, AsyncSemaphore semaphore, CancellationToken cancellationToken)
{
// Conduct a dither for every endpoint probe to optimize concurrency.
var randomDither = _randomFactory.CreateRandomInstance();
await _timer.Delay(TimeSpan.FromMilliseconds(randomDither.Next(_ditheringIntervalInMilliseconds)), cancellationToken);
var outcome = HealthProbeOutcome.Unknown;
string logDetail = null;
// Enforce max concurrency.
await semaphore.WaitAsync();
_logger.LogInformation($"The backend prober for backend: '{BackendId}', endpoint: `{endpoint.EndpointId}` has started.");
try
{
using (var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token))
{
// Set up timeout and start probing.
timeoutCts.CancelAfter(_httpTimeoutInterval, _timer);
var response = await _operationLogger.ExecuteAsync(
"ReverseProxy.Core.Service.HealthProbe",
() => _backendProbeHttpClient.GetAsync(new Uri(new Uri(endpoint.Config.Value.Address, UriKind.Absolute), _healthControllerUrl), timeoutCts.Token));
// Collect response status.
outcome = response.IsSuccessStatusCode ? HealthProbeOutcome.Success : HealthProbeOutcome.HttpFailure;
logDetail = $"Received status code {(int)response.StatusCode}";
}
}
catch (HttpRequestException ex)
{
// If there is a error during the http request process. Swallow the error and log error message.
outcome = HealthProbeOutcome.TransportFailure;
logDetail = ex.Message;
}
catch (OperationCanceledException)
{
// If the cancel requested by our StopAsync method. It is a expected graceful shut down.
if (_cts.IsCancellationRequested)
{
outcome = HealthProbeOutcome.Canceled;
logDetail = "Operation deliberately canceled";
throw;
}
else
{
outcome = HealthProbeOutcome.Timeout;
logDetail = $"Health probe timed out after {Config.HealthCheckOptions.Interval.TotalSeconds} second";
}
}
catch (Exception ex)
{
throw new Exception($"Prober for '{endpoint.EndpointId}' encounters unexpected exception.", ex);
}
finally
{
if (outcome != HealthProbeOutcome.Canceled)
{
// Update the health state base on the response.
var healthState = outcome == HealthProbeOutcome.Success ? EndpointHealth.Healthy : EndpointHealth.Unhealthy;
endpoint.DynamicState.Value = new EndpointDynamicState(healthState);
_logger.LogInformation($"Health probe result for endpoint '{endpoint.EndpointId}': {outcome}. Details: {logDetail}");
}
// The probe operation is done, release the semaphore to allow other probes to proceed.
semaphore.Release();
}
}
/// <inheritdoc/>
public async Task Invoke(HttpContext context)
{
Contracts.CheckValue(context, nameof(context));
var backend = context.Features.Get <BackendInfo>() ?? throw new InvalidOperationException("Backend unspecified.");
var endpoints = context.Features.Get <IAvailableBackendEndpointsFeature>()?.Endpoints
?? throw new InvalidOperationException("The AvailableBackendEndpoints collection was not set.");
var routeConfig = context.GetEndpoint()?.Metadata.GetMetadata <RouteConfig>()
?? throw new InvalidOperationException("RouteConfig unspecified.");
if (endpoints.Count == 0)
{
_logger.LogWarning("No available endpoints.");
context.Response.StatusCode = 503;
return;
}
var endpoint = endpoints[0];
if (endpoints.Count > 1)
{
_logger.LogWarning("More than one endpoint available, load balancing may not be configured correctly. Choosing randomly.");
endpoint = endpoints[_random.Next(endpoints.Count)];
}
var endpointConfig = endpoint.Config.Value;
if (endpointConfig == null)
{
throw new InvalidOperationException($"Chosen endpoint has no configs set: '{endpoint.EndpointId}'");
}
// TODO: support StripPrefix and other url transformations
var targetUrl = BuildOutgoingUrl(context, endpointConfig.Address);
_logger.LogInformation($"Proxying to {targetUrl}");
var targetUri = new Uri(targetUrl, UriKind.Absolute);
using (var shortCts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted))
{
// TODO: Configurable timeout, measure from request start, make it unit-testable
shortCts.CancelAfter(TimeSpan.FromSeconds(30));
// TODO: Retry against other endpoints
try
{
// TODO: Apply caps
backend.ConcurrencyCounter.Increment();
endpoint.ConcurrencyCounter.Increment();
// TODO: Duplex channels should not have a timeout (?), but must react to Proxy force-shutdown signals.
var longCancellation = context.RequestAborted;
var proxyTelemetryContext = new ProxyTelemetryContext(
backendId: backend.BackendId,
routeId: routeConfig.Route.RouteId,
endpointId: endpoint.EndpointId);
await _operationLogger.ExecuteAsync(
"ReverseProxy.Proxy",
() => _httpProxy.ProxyAsync(context, targetUri, backend.ProxyHttpClientFactory, proxyTelemetryContext, shortCancellation: shortCts.Token, longCancellation: longCancellation));
}
finally
{
endpoint.ConcurrencyCounter.Decrement();
backend.ConcurrencyCounter.Decrement();
}
}
}