private async Task <(StreamCopyResult, Exception)> CopyResponseBodyAsync(HttpContent destinationResponseContent, Stream clientResponseStream,
CancellationToken cancellation)
{
// SocketHttpHandler and similar transports always provide an HttpContent object, even if it's empty.
// In 3.1 this is only likely to return null in tests.
// As of 5.0 HttpResponse.Content never returns null.
// https://github.com/dotnet/runtime/blame/8fc68f626a11d646109a758cb0fc70a0aa7826f1/src/libraries/System.Net.Http/src/System/Net/Http/HttpResponseMessage.cs#L46
if (destinationResponseContent != null)
{
using var destinationResponseStream = await destinationResponseContent.ReadAsStreamAsync();
return(await StreamCopier.CopyAsync(isRequest : false, destinationResponseStream, clientResponseStream, _clock, cancellation));
}
return(StreamCopyResult.Success, null);
}
/// <summary>
/// Proxies an upgradable request to the upstream server, treating the upgraded stream as an opaque duplex channel.
/// </summary>
/// <remarks>
/// Upgradable request proxying comprises the following steps:
/// (1) Create outgoing HttpRequestMessage
/// (2) Copy request headers Downstream ---► Proxy ---► Upstream
/// (3) Send the outgoing request using HttpMessageInvoker Downstream ---► Proxy ---► Upstream
/// (4) Copy response status line Downstream ◄--- Proxy ◄--- Upstream
/// (5) Copy response headers Downstream ◄--- Proxy ◄--- Upstream
/// Scenario A: upgrade with upstream worked (got 101 response)
/// (A-6) Upgrade downstream channel (also sends response headers) Downstream ◄--- Proxy ◄--- Upstream
/// (A-7) Copy duplex streams Downstream ◄--► Proxy ◄--► Upstream
/// ---- or ----
/// Scenario B: upgrade with upstream failed (got non-101 response)
/// (B-6) Send response headers Downstream ◄--- Proxy ◄--- Upstream
/// (B-7) Copy response body Downstream ◄--- Proxy ◄--- Upstream
///
/// This takes care of WebSockets as well as any other upgradable protocol.
/// </remarks>
private async Task UpgradableProxyAsync(
HttpContext context,
IHttpUpgradeFeature upgradeFeature,
Uri targetUri,
HttpMessageInvoker httpClient,
ProxyTelemetryContext proxyTelemetryContext,
CancellationToken shortCancellation,
CancellationToken longCancellation)
{
Contracts.CheckValue(context, nameof(context));
Contracts.CheckValue(upgradeFeature, nameof(upgradeFeature));
Contracts.CheckValue(targetUri, nameof(targetUri));
Contracts.CheckValue(httpClient, nameof(httpClient));
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 1: Create outgoing HttpRequestMessage
var upstreamRequest = new HttpRequestMessage(HttpUtilities.GetHttpMethod(context.Request.Method), targetUri)
{
// Default to HTTP/1.1 for proxying upgradable requests. This is already the default as of .NET Core 3.1
Version = new Version(1, 1),
};
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 2: Copy request headers Downstream --► Proxy --► Upstream
CopyHeadersToUpstream(context, upstreamRequest);
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 3: Send the outgoing request using HttpMessageInvoker
var upstreamResponse = await httpClient.SendAsync(upstreamRequest, shortCancellation);
var upgraded = upstreamResponse.StatusCode == HttpStatusCode.SwitchingProtocols && upstreamResponse.Content != null;
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 4: Copy response status line Downstream ◄-- Proxy ◄-- Upstream
context.Response.StatusCode = (int)upstreamResponse.StatusCode;
context.Features.Get <IHttpResponseFeature>().ReasonPhrase = upstreamResponse.ReasonPhrase;
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 5: Copy response headers Downstream ◄-- Proxy ◄-- Upstream
CopyHeadersToDownstream(upstreamResponse, context.Response.Headers);
if (!upgraded)
{
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step B-6: Send response headers Downstream ◄-- Proxy ◄-- Upstream
// This is important to avoid any extra delays in sending response headers
// e.g. if the upstream server is slow to provide its response body.
await context.Response.StartAsync(shortCancellation);
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step B-7: Copy response body Downstream ◄-- Proxy ◄-- Upstream
await CopyBodyDownstreamAsync(upstreamResponse.Content, context.Response.Body, proxyTelemetryContext, longCancellation);
return;
}
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step A-6: Upgrade the downstream channel. This will send all response headers too.
using var downstreamStream = await upgradeFeature.UpgradeAsync();
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step A-7: Copy duplex streams
var upstreamStream = await upstreamResponse.Content.ReadAsStreamAsync();
var upstreamCopier = new StreamCopier(
_metrics,
new StreamCopyTelemetryContext(
direction: "upstream",
backendId: proxyTelemetryContext.BackendId,
routeId: proxyTelemetryContext.RouteId,
destinationId: proxyTelemetryContext.DestinationId));
var upstreamTask = upstreamCopier.CopyAsync(downstreamStream, upstreamStream, longCancellation);
var downstreamCopier = new StreamCopier(
_metrics,
new StreamCopyTelemetryContext(
direction: "downstream",
backendId: proxyTelemetryContext.BackendId,
routeId: proxyTelemetryContext.RouteId,
destinationId: proxyTelemetryContext.DestinationId));
var downstreamTask = downstreamCopier.CopyAsync(upstreamStream, downstreamStream, longCancellation);
await Task.WhenAll(upstreamTask, downstreamTask);
}
private async Task HandleUpgradedResponse(HttpContext context, HttpResponseMessage destinationResponse,
CancellationToken longCancellation)
{
ProxyTelemetry.Log.ProxyStage(ProxyStage.ResponseUpgrade);
// SocketHttpHandler and similar transports always provide an HttpContent object, even if it's empty.
// Note as of 5.0 HttpResponse.Content never returns null.
// https://github.com/dotnet/runtime/blame/8fc68f626a11d646109a758cb0fc70a0aa7826f1/src/libraries/System.Net.Http/src/System/Net/Http/HttpResponseMessage.cs#L46
if (destinationResponse.Content == null)
{
throw new InvalidOperationException("A response content is required for upgrades.");
}
// :: Step 7-A-1: Upgrade the client channel. This will also send response headers.
var upgradeFeature = context.Features.Get <IHttpUpgradeFeature>();
Stream upgradeResult;
try
{
upgradeResult = await upgradeFeature.UpgradeAsync();
}
catch (Exception ex)
{
destinationResponse.Dispose();
ReportProxyError(context, ProxyError.UpgradeResponseClient, ex);
return;
}
using var clientStream = upgradeResult;
// :: Step 7-A-2: Copy duplex streams
using var destinationStream = await destinationResponse.Content.ReadAsStreamAsync();
using var abortTokenSource = CancellationTokenSource.CreateLinkedTokenSource(longCancellation);
var requestTask = StreamCopier.CopyAsync(isRequest: true, clientStream, destinationStream, _clock, abortTokenSource.Token);
var responseTask = StreamCopier.CopyAsync(isRequest: false, destinationStream, clientStream, _clock, abortTokenSource.Token);
// Make sure we report the first failure.
var firstTask = await Task.WhenAny(requestTask, responseTask);
var requestFinishedFirst = firstTask == requestTask;
var secondTask = requestFinishedFirst ? responseTask : requestTask;
var(firstResult, firstException) = await firstTask;
if (firstResult != StreamCopyResult.Success)
{
ReportResult(context, requestFinishedFirst, firstResult, firstException);
// Cancel the other direction
abortTokenSource.Cancel();
// Wait for this to finish before exiting so the resources get cleaned up properly.
await secondTask;
}
else
{
var(secondResult, secondException) = await secondTask;
if (secondResult != StreamCopyResult.Success)
{
ReportResult(context, requestFinishedFirst, secondResult, secondException);
}
}
void ReportResult(HttpContext context, bool reqeuest, StreamCopyResult result, Exception exception)
{
var error = result switch
{
StreamCopyResult.InputError => reqeuest ? ProxyError.UpgradeRequestClient : ProxyError.UpgradeResponseDestination,
StreamCopyResult.OutputError => reqeuest ? ProxyError.UpgradeRequestDestination : ProxyError.UpgradeResponseClient,
StreamCopyResult.Canceled => reqeuest ? ProxyError.UpgradeRequestCanceled : ProxyError.UpgradeResponseCanceled,
_ => throw new NotImplementedException(result.ToString()),
};
ReportProxyError(context, error, exception);
}
}
/// <summary>
/// Proxies an upgradable request to the upstream server, treating the upgraded stream as an opaque duplex channel.
/// </summary>
/// <remarks>
/// Upgradable request proxying comprises the following steps:
/// (1) Create outgoing HttpRequestMessage
/// (2) Copy request headers Downstream ---► Proxy ---► Upstream
/// (3) Send the outgoing request using HttpMessageInvoker Downstream ---► Proxy ---► Upstream
/// (4) Copy response status line Downstream ◄--- Proxy ◄--- Upstream
/// (5) Copy response headers Downstream ◄--- Proxy ◄--- Upstream
/// Scenario A: upgrade with upstream worked (got 101 response)
/// (A-6) Upgrade downstream channel (also sends response headers) Downstream ◄--- Proxy ◄--- Upstream
/// (A-7) Copy duplex streams Downstream ◄--► Proxy ◄--► Upstream
/// ---- or ----
/// Scenario B: upgrade with upstream failed (got non-101 response)
/// (B-6) Send response headers Downstream ◄--- Proxy ◄--- Upstream
/// (B-7) Copy response body Downstream ◄--- Proxy ◄--- Upstream
///
/// This takes care of WebSockets as well as any other upgradable protocol.
/// </remarks>
private async Task UpgradableProxyAsync(
HttpContext context,
IHttpUpgradeFeature upgradeFeature,
HttpRequestMessage upstreamRequest,
Transforms transforms,
HttpMessageInvoker httpClient,
ProxyTelemetryContext proxyTelemetryContext,
CancellationToken shortCancellation,
CancellationToken longCancellation)
{
_ = context ?? throw new ArgumentNullException(nameof(context));
_ = upgradeFeature ?? throw new ArgumentNullException(nameof(upgradeFeature));
_ = upstreamRequest ?? throw new ArgumentNullException(nameof(upstreamRequest));
_ = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 2: Copy request headers Downstream --► Proxy --► Upstream
CopyHeadersToUpstream(context, upstreamRequest, transforms.CopyRequestHeaders, transforms.RequestHeaderTransforms);
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 3: Send the outgoing request using HttpMessageInvoker
var upstreamResponse = await httpClient.SendAsync(upstreamRequest, shortCancellation);
var upgraded = upstreamResponse.StatusCode == HttpStatusCode.SwitchingProtocols && upstreamResponse.Content != null;
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 4: Copy response status line Downstream ◄-- Proxy ◄-- Upstream
context.Response.StatusCode = (int)upstreamResponse.StatusCode;
context.Features.Get <IHttpResponseFeature>().ReasonPhrase = upstreamResponse.ReasonPhrase;
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step 5: Copy response headers Downstream ◄-- Proxy ◄-- Upstream
CopyHeadersToDownstream(upstreamResponse, context, transforms.ResponseHeaderTransforms);
if (!upgraded)
{
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step B-6: Send response headers Downstream ◄-- Proxy ◄-- Upstream
// This is important to avoid any extra delays in sending response headers
// e.g. if the upstream server is slow to provide its response body.
await context.Response.StartAsync(shortCancellation);
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step B-7: Copy response body Downstream ◄-- Proxy ◄-- Upstream
await CopyBodyDownstreamAsync(upstreamResponse.Content, context.Response.Body, proxyTelemetryContext, longCancellation);
return;
}
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step A-6: Upgrade the downstream channel. This will send all response headers too.
using var downstreamStream = await upgradeFeature.UpgradeAsync();
// :::::::::::::::::::::::::::::::::::::::::::::
// :: Step A-7: Copy duplex streams
var upstreamStream = await upstreamResponse.Content.ReadAsStreamAsync();
var upstreamCopier = new StreamCopier(
_metrics,
new StreamCopyTelemetryContext(
direction: "upstream",
clusterId: proxyTelemetryContext.ClusterId,
routeId: proxyTelemetryContext.RouteId,
destinationId: proxyTelemetryContext.DestinationId));
var upstreamTask = upstreamCopier.CopyAsync(downstreamStream, upstreamStream, longCancellation);
var downstreamCopier = new StreamCopier(
_metrics,
new StreamCopyTelemetryContext(
direction: "downstream",
clusterId: proxyTelemetryContext.ClusterId,
routeId: proxyTelemetryContext.RouteId,
destinationId: proxyTelemetryContext.DestinationId));
var downstreamTask = downstreamCopier.CopyAsync(upstreamStream, downstreamStream, longCancellation);
await Task.WhenAll(upstreamTask, downstreamTask);
}
