/// <summary> /// Proxies an upgradable request to the upstream server, treating the upgraded stream as an opaque duplex channel. /// </summary> /// <remarks> /// Upgradable request proxying comprises the following steps: /// (1) Create outgoing HttpRequestMessage /// (2) Copy request headers Downstream ---► Proxy ---► Upstream /// (3) Send the outgoing request using HttpMessageInvoker Downstream ---► Proxy ---► Upstream /// (4) Copy response status line Downstream ◄--- Proxy ◄--- Upstream /// (5) Copy response headers Downstream ◄--- Proxy ◄--- Upstream /// Scenario A: upgrade with upstream worked (got 101 response) /// (A-6) Upgrade downstream channel (also sends response headers) Downstream ◄--- Proxy ◄--- Upstream /// (A-7) Copy duplex streams Downstream ◄--► Proxy ◄--► Upstream /// ---- or ---- /// Scenario B: upgrade with upstream failed (got non-101 response) /// (B-6) Send response headers Downstream ◄--- Proxy ◄--- Upstream /// (B-7) Copy response body Downstream ◄--- Proxy ◄--- Upstream /// /// This takes care of WebSockets as well as any other upgradable protocol. /// </remarks> private async Task UpgradableProxyAsync( HttpContext context, IHttpUpgradeFeature upgradeFeature, Uri targetUri, HttpMessageInvoker httpClient, ProxyTelemetryContext proxyTelemetryContext, CancellationToken shortCancellation, CancellationToken longCancellation) { Contracts.CheckValue(context, nameof(context)); Contracts.CheckValue(upgradeFeature, nameof(upgradeFeature)); Contracts.CheckValue(targetUri, nameof(targetUri)); Contracts.CheckValue(httpClient, nameof(httpClient)); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 1: Create outgoing HttpRequestMessage var upstreamRequest = new HttpRequestMessage(HttpUtilities.GetHttpMethod(context.Request.Method), targetUri) { // Default to HTTP/1.1 for proxying upgradable requests. This is already the default as of .NET Core 3.1 Version = new Version(1, 1), }; // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 2: Copy request headers Downstream --► Proxy --► Upstream CopyHeadersToUpstream(context, upstreamRequest); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 3: Send the outgoing request using HttpMessageInvoker var upstreamResponse = await httpClient.SendAsync(upstreamRequest, shortCancellation); var upgraded = upstreamResponse.StatusCode == HttpStatusCode.SwitchingProtocols && upstreamResponse.Content != null; // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 4: Copy response status line Downstream ◄-- Proxy ◄-- Upstream context.Response.StatusCode = (int)upstreamResponse.StatusCode; context.Features.Get <IHttpResponseFeature>().ReasonPhrase = upstreamResponse.ReasonPhrase; // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 5: Copy response headers Downstream ◄-- Proxy ◄-- Upstream CopyHeadersToDownstream(upstreamResponse, context.Response.Headers); if (!upgraded) { // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step B-6: Send response headers Downstream ◄-- Proxy ◄-- Upstream // This is important to avoid any extra delays in sending response headers // e.g. if the upstream server is slow to provide its response body. await context.Response.StartAsync(shortCancellation); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step B-7: Copy response body Downstream ◄-- Proxy ◄-- Upstream await CopyBodyDownstreamAsync(upstreamResponse.Content, context.Response.Body, proxyTelemetryContext, longCancellation); return; } // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step A-6: Upgrade the downstream channel. This will send all response headers too. using var downstreamStream = await upgradeFeature.UpgradeAsync(); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step A-7: Copy duplex streams var upstreamStream = await upstreamResponse.Content.ReadAsStreamAsync(); var upstreamCopier = new StreamCopier( _metrics, new StreamCopyTelemetryContext( direction: "upstream", backendId: proxyTelemetryContext.BackendId, routeId: proxyTelemetryContext.RouteId, destinationId: proxyTelemetryContext.DestinationId)); var upstreamTask = upstreamCopier.CopyAsync(downstreamStream, upstreamStream, longCancellation); var downstreamCopier = new StreamCopier( _metrics, new StreamCopyTelemetryContext( direction: "downstream", backendId: proxyTelemetryContext.BackendId, routeId: proxyTelemetryContext.RouteId, destinationId: proxyTelemetryContext.DestinationId)); var downstreamTask = downstreamCopier.CopyAsync(upstreamStream, downstreamStream, longCancellation); await Task.WhenAll(upstreamTask, downstreamTask); }
private HttpRequestMessage CreateRequestMessage(HttpContext context, string destinationAddress, bool isUpgradeRequest, RequestProxyOptions proxyOptions) { // "http://a".Length = 8 if (destinationAddress == null || destinationAddress.Length < 8) { throw new ArgumentException(nameof(destinationAddress)); } // Default to HTTP/1.1 for proxying upgradeable requests. This is already the default as of .NET Core 3.1 // Otherwise request what's set in proxyOptions (e.g. default HTTP/2) and let HttpClient negotiate the protocol // based on VersionPolicy (for .NET 5 and higher). For example, downgrading to HTTP/1.1 if it cannot establish HTTP/2 with the target. // This is done without extra round-trips thanks to ALPN. We can detect a downgrade after calling HttpClient.SendAsync // (see Step 3 below). TBD how this will change when HTTP/3 is supported. var httpVersion = isUpgradeRequest ? ProtocolHelper.Http11Version : proxyOptions.Version; #if NET var httpVersionPolicy = isUpgradeRequest ? HttpVersionPolicy.RequestVersionOrLower : proxyOptions.VersionPolicy; #endif // TODO Perf: We could probably avoid splitting this and just append the final path and query UriHelper.FromAbsolute(destinationAddress, out var destinationScheme, out var destinationHost, out var destinationPathBase, out _, out _); // Query and Fragment are not supported here. var request = context.Request; var transforms = proxyOptions.Transforms.RequestTransforms; if (transforms.Count == 0) { var url = UriHelper.BuildAbsolute(destinationScheme, destinationHost, destinationPathBase, request.Path, request.QueryString); Log.Proxying(_logger, url); var uri = new Uri(url, UriKind.Absolute); return(new HttpRequestMessage(HttpUtilities.GetHttpMethod(context.Request.Method), uri) { Version = httpVersion, #if NET VersionPolicy = httpVersionPolicy, #endif }); } var transformContext = new RequestParametersTransformContext() { HttpContext = context, Version = httpVersion, Method = request.Method, Path = request.Path, Query = new QueryTransformContext(request), #if NET VersionPolicy = httpVersionPolicy, #endif }; foreach (var transform in transforms) { transform.Apply(transformContext); } var targetUrl = UriHelper.BuildAbsolute(destinationScheme, destinationHost, destinationPathBase, transformContext.Path, transformContext.Query.QueryString); Log.Proxying(_logger, targetUrl); var targetUri = new Uri(targetUrl, UriKind.Absolute); return(new HttpRequestMessage(HttpUtilities.GetHttpMethod(transformContext.Method), targetUri) { Version = transformContext.Version, #if NET VersionPolicy = transformContext.VersionPolicy, #endif }); }
/// <summary> /// Proxies a normal (i.e. non-upgradable) request to the upstream server, and the response back to our client. /// </summary> /// <remarks> /// Normal proxying comprises the following steps: /// (0) Disable ASP .NET Core limits for streaming requests /// (1) Create outgoing HttpRequestMessage /// (2) Setup copy of request body (background) Downstream --► Proxy --► Upstream /// (3) Copy request headers Downstream --► Proxy --► Upstream /// (4) Send the outgoing request using HttpMessageInvoker Downstream --► Proxy --► Upstream /// (5) Copy response status line Downstream ◄-- Proxy ◄-- Upstream /// (6) Copy response headers Downstream ◄-- Proxy ◄-- Upstream /// (7) Copy response body Downstream ◄-- Proxy ◄-- Upstream /// (8) Copy response trailer headers and finish response Downstream ◄-- Proxy ◄-- Upstream /// (9) Wait for completion of step 2: copying request body Downstream --► Proxy --► Upstream /// /// ASP .NET Core (Kestrel) will finally send response trailers (if any) /// after we complete the steps above and relinquish control. /// </remarks> private async Task NormalProxyAsync( HttpContext context, Uri targetUri, HttpMessageInvoker httpClient, ProxyTelemetryContext proxyTelemetryContext, CancellationToken shortCancellation, CancellationToken longCancellation) { Contracts.CheckValue(context, nameof(context)); Contracts.CheckValue(targetUri, nameof(targetUri)); Contracts.CheckValue(httpClient, nameof(httpClient)); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 0: Disable ASP .NET Core limits for streaming requests var isIncomingHttp2 = ProtocolHelper.IsHttp2(context.Request.Protocol); // NOTE: We heuristically assume gRPC-looking requests may require streaming semantics. // See https://github.com/microsoft/reverse-proxy/issues/118 for design discussion. var isStreamingRequest = isIncomingHttp2 && ProtocolHelper.IsGrpcContentType(context.Request.ContentType); if (isStreamingRequest) { DisableMinRequestBodyDataRateAndMaxRequestBodySize(context); } // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 1: Create outgoing HttpRequestMessage var upstreamRequest = new HttpRequestMessage(HttpUtilities.GetHttpMethod(context.Request.Method), targetUri) { // We request HTTP/2, but HttpClient will fallback to HTTP/1.1 if it cannot establish HTTP/2 with the target. // This is done without extra round-trips thanks to ALPN. We can detect a downgrade after calling HttpClient.SendAsync // (see Step 3 below). TBD how this will change when HTTP/3 is supported. Version = Http2Version, }; // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 2: Setup copy of request body (background) Downstream --► Proxy --► Upstream // Note that we must do this before step (3) because step (3) may also add headers to the HttpContent that we set up here. var bodyToUpstreamContent = SetupCopyBodyUpstream(context.Request.Body, upstreamRequest, in proxyTelemetryContext, isStreamingRequest, longCancellation); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 3: Copy request headers Downstream --► Proxy --► Upstream CopyHeadersToUpstream(context, upstreamRequest); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 4: Send the outgoing request using HttpClient ////this.logger.LogInformation($" Starting Proxy --> upstream request"); var upstreamResponse = await httpClient.SendAsync(upstreamRequest, shortCancellation); // Detect connection downgrade, which may be problematic for e.g. gRPC. if (isIncomingHttp2 && upstreamResponse.Version.Major != 2) { // TODO: Do something on connection downgrade... Log.HttpDowngradeDeteced(_logger); } // Assert that, if we are proxying content upstream, it must have started by now // (since HttpClient.SendAsync has already completed asynchronously). // If this check fails, there is a coding defect which would otherwise // cause us to wait forever in step 9, so fail fast here. if (bodyToUpstreamContent != null && !bodyToUpstreamContent.Started) { // TODO: bodyToUpstreamContent is never null. HttpClient might would not need to read the body in some scenarios, such as an early auth failure with Expect: 100-continue. throw new InvalidOperationException("Proxying the downstream request body to the upstream server hasn't started. This is a coding defect."); } // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 5: Copy response status line Downstream ◄-- Proxy ◄-- Upstream ////this.logger.LogInformation($" Setting downstream <-- Proxy status: {(int)upstreamResponse.StatusCode} {upstreamResponse.ReasonPhrase}"); context.Response.StatusCode = (int)upstreamResponse.StatusCode; context.Features.Get <IHttpResponseFeature>().ReasonPhrase = upstreamResponse.ReasonPhrase; // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 6: Copy response headers Downstream ◄-- Proxy ◄-- Upstream CopyHeadersToDownstream(upstreamResponse, context.Response.Headers); // NOTE: it may *seem* wise to call `context.Response.StartAsync()` at this point // since it looks like we are ready to send back response headers // (and this might help reduce extra delays while we wait to receive the body from upstream). // HOWEVER, this would produce the wrong result if it turns out that there is no content // from the upstream -- instead of sending headers and terminating the stream at once, // we would send headers thinking a body may be coming, and there is none. // This is problematic on gRPC connections when the upstream server encounters an error, // in which case it immediately returns the response headers and trailing headers, but no content, // and clients misbehave if the initial headers response does not indicate stream end. // TODO: Some of the tasks in steps (7) - (9) may go unobserved depending on what fails first. Needs more consideration. // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 7: Copy response body Downstream ◄-- Proxy ◄-- Upstream await CopyBodyDownstreamAsync(upstreamResponse.Content, context.Response.Body, proxyTelemetryContext, longCancellation); // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 8: Copy response trailer headers and finish response Downstream ◄-- Proxy ◄-- Upstream CopyTrailingHeadersToDownstream(upstreamResponse, context); if (isStreamingRequest) { // NOTE: We must call `CompleteAsync` so that Kestrel will flush all bytes to the client. // In the case where there was no response body, // this is also when headers and trailing headers are sent to the client. // Without this, the client might wait forever waiting for response bytes, // while we might wait forever waiting for request bytes, // leading to a stuck connection and no way to make progress. await context.Response.CompleteAsync(); } // ::::::::::::::::::::::::::::::::::::::::::::: // :: Step 9: Wait for completion of step 2: copying request body Downstream --► Proxy --► Upstream if (bodyToUpstreamContent != null) { ////this.logger.LogInformation($" Waiting for downstream --> Proxy --> upstream body proxying to complete"); await bodyToUpstreamContent.ConsumptionTask; } }
private HttpRequestMessage CreateRequestMessage(HttpContext context, string destinationAddress, IReadOnlyList <RequestParametersTransform> requestTransforms, RequestProxyOptions requestOptions) { // "http://a".Length = 8 if (destinationAddress == null || destinationAddress.Length < 8) { throw new ArgumentException(nameof(destinationAddress)); } var upgradeFeature = context.Features.Get <IHttpUpgradeFeature>(); var upgradeHeader = context.Request.Headers[HeaderNames.Upgrade].ToString(); var isUpgradeRequest = (upgradeFeature?.IsUpgradableRequest ?? false) // Mitigate https://github.com/microsoft/reverse-proxy/issues/255, IIS considers all requests upgradeable. && (string.Equals("WebSocket", upgradeHeader, StringComparison.OrdinalIgnoreCase) // https://github.com/microsoft/reverse-proxy/issues/467 for kubernetes APIs || upgradeHeader.StartsWith("SPDY/", StringComparison.OrdinalIgnoreCase)); // Default to HTTP/1.1 for proxying upgradeable requests. This is already the default as of .NET Core 3.1 // Otherwise request what's set in proxyOptions (e.g. default HTTP/2) and let HttpClient negotiate the protocol // based on VersionPolicy (for .NET 5 and higher). For example, downgrading to HTTP/1.1 if it cannot establish HTTP/2 with the target. // This is done without extra round-trips thanks to ALPN. We can detect a downgrade after calling HttpClient.SendAsync // (see Step 3 below). TBD how this will change when HTTP/3 is supported. var httpVersion = isUpgradeRequest ? ProtocolHelper.Http11Version : (requestOptions.Version ?? DefaultVersion); #if NET var httpVersionPolicy = isUpgradeRequest ? HttpVersionPolicy.RequestVersionOrLower : (requestOptions.VersionPolicy ?? DefaultVersionPolicy); #endif // TODO Perf: We could probably avoid splitting this and just append the final path and query UriHelper.FromAbsolute(destinationAddress, out var destinationScheme, out var destinationHost, out var destinationPathBase, out _, out _); // Query and Fragment are not supported here. var request = context.Request; if (requestTransforms.Count == 0) { var url = UriHelper.BuildAbsolute(destinationScheme, destinationHost, destinationPathBase, request.Path, request.QueryString); Log.Proxying(_logger, url); var uri = new Uri(url, UriKind.Absolute); return(new HttpRequestMessage(HttpUtilities.GetHttpMethod(context.Request.Method), uri) { Version = httpVersion, #if NET VersionPolicy = httpVersionPolicy, #endif }); } var transformContext = new RequestParametersTransformContext() { HttpContext = context, Version = httpVersion, Method = request.Method, Path = request.Path, Query = new QueryTransformContext(request), #if NET VersionPolicy = httpVersionPolicy, #endif }; foreach (var requestTransform in requestTransforms) { requestTransform.Apply(transformContext); } var targetUrl = UriHelper.BuildAbsolute(destinationScheme, destinationHost, destinationPathBase, transformContext.Path, transformContext.Query.QueryString); Log.Proxying(_logger, targetUrl); var targetUri = new Uri(targetUrl, UriKind.Absolute); return(new HttpRequestMessage(HttpUtilities.GetHttpMethod(transformContext.Method), targetUri) { Version = transformContext.Version, #if NET VersionPolicy = transformContext.VersionPolicy, #endif }); }