/// <summary>
/// A callback that is invoked prior to sending the proxied request. All HttpRequestMessage fields are
/// initialized except RequestUri, which will be initialized after the callback if no value is provided.
/// See <see cref="RequestUtilities.MakeDestinationAddress(string, PathString, QueryString)"/> for constructing a custom request Uri.
/// The string parameter represents the destination URI prefix that should be used when constructing the RequestUri.
/// The headers are copied by the base implementation, excluding some protocol headers like HTTP/2 pseudo headers (":authority").
/// </summary>
/// <param name="httpContext">The incoming request.</param>
/// <param name="proxyRequest">The outgoing proxy request.</param>
/// <param name="destinationPrefix">The uri prefix for the selected destination server which can be used to create the RequestUri.</param>
public virtual ValueTask TransformRequestAsync(HttpContext httpContext, HttpRequestMessage proxyRequest, string destinationPrefix)
{
foreach (var header in httpContext.Request.Headers)
{
var headerName = header.Key;
var headerValue = header.Value;
if (RequestUtilities.ShouldSkipRequestHeader(headerName))
{
continue;
}
RequestUtilities.AddHeader(proxyRequest, headerName, headerValue);
}
// https://datatracker.ietf.org/doc/html/rfc7230#section-3.3.3
// If a message is received with both a Transfer-Encoding and a
// Content-Length header field, the Transfer-Encoding overrides the
// Content-Length. Such a message might indicate an attempt to
// perform request smuggling (Section 9.5) or response splitting
// (Section 9.4) and ought to be handled as an error. A sender MUST
// remove the received Content-Length field prior to forwarding such
// a message downstream.
if (httpContext.Request.Headers.ContainsKey(HeaderNames.TransferEncoding) &&
httpContext.Request.Headers.ContainsKey(HeaderNames.ContentLength))
{
proxyRequest.Content?.Headers.Remove(HeaderNames.ContentLength);
}
// https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2.2
// The only exception to this is the TE header field, which MAY be
// present in an HTTP/2 request; when it is, it MUST NOT contain any
// value other than "trailers".
if (ProtocolHelper.IsHttp2OrGreater(httpContext.Request.Protocol))
{
var te = httpContext.Request.Headers.GetCommaSeparatedValues(HeaderNames.TE);
if (te != null)
{
for (var i = 0; i < te.Length; i++)
{
if (string.Equals(te[i], "trailers", StringComparison.OrdinalIgnoreCase))
{
var added = proxyRequest.Headers.TryAddWithoutValidation(HeaderNames.TE, te[i]);
Debug.Assert(added);
break;
}
}
}
}
return(default);
/// <summary>
/// Adds the given header to the HttpRequestMessage or HttpContent where applicable.
/// </summary>
public static void AddHeader(RequestTransformContext context, string headerName, StringValues values)
{
if (context is null)
{
throw new System.ArgumentNullException(nameof(context));
}
if (string.IsNullOrEmpty(headerName))
{
throw new System.ArgumentException($"'{nameof(headerName)}' cannot be null or empty.", nameof(headerName));
}
RequestUtilities.AddHeader(context.ProxyRequest, headerName, values);
}
private static void CopyResponseHeaders(HttpContext httpContext, HttpHeaders source, IHeaderDictionary destination)
{
var isHttp2OrGreater = ProtocolHelper.IsHttp2OrGreater(httpContext.Request.Protocol);
foreach (var header in source)
{
var headerName = header.Key;
if (RequestUtilities.ShouldSkipResponseHeader(headerName, isHttp2OrGreater))
{
continue;
}
destination.Append(headerName, header.Value.ToArray());
}
}
private async Task <(HttpRequestMessage, StreamCopyHttpContent)> CreateRequestMessageAsync(HttpContext context, string destinationPrefix,
HttpTransformer transformer, RequestProxyOptions requestOptions, bool isStreamingRequest, CancellationToken requestAborted)
{
// "http://a".Length = 8
if (destinationPrefix == null || destinationPrefix.Length < 8)
{
throw new ArgumentException(nameof(destinationPrefix));
}
var destinationRequest = new HttpRequestMessage();
destinationRequest.Method = HttpUtilities.GetHttpMethod(context.Request.Method);
var upgradeFeature = context.Features.Get <IHttpUpgradeFeature>();
var upgradeHeader = context.Request.Headers[HeaderNames.Upgrade].ToString();
var isUpgradeRequest = (upgradeFeature?.IsUpgradableRequest ?? false)
// Mitigate https://github.com/microsoft/reverse-proxy/issues/255, IIS considers all requests upgradeable.
&& (string.Equals("WebSocket", upgradeHeader, StringComparison.OrdinalIgnoreCase)
// https://github.com/microsoft/reverse-proxy/issues/467 for kubernetes APIs
|| upgradeHeader.StartsWith("SPDY/", StringComparison.OrdinalIgnoreCase));
// Default to HTTP/1.1 for proxying upgradeable requests. This is already the default as of .NET Core 3.1
// Otherwise request what's set in proxyOptions (e.g. default HTTP/2) and let HttpClient negotiate the protocol
// based on VersionPolicy (for .NET 5 and higher). For example, downgrading to HTTP/1.1 if it cannot establish HTTP/2 with the target.
// This is done without extra round-trips thanks to ALPN. We can detect a downgrade after calling HttpClient.SendAsync
// (see Step 3 below). TBD how this will change when HTTP/3 is supported.
destinationRequest.Version = isUpgradeRequest ? ProtocolHelper.Http11Version : (requestOptions?.Version ?? DefaultVersion);
#if NET
destinationRequest.VersionPolicy = isUpgradeRequest ? HttpVersionPolicy.RequestVersionOrLower : (requestOptions?.VersionPolicy ?? DefaultVersionPolicy);
#endif
// :: Step 2: Setup copy of request body (background) Client --► Proxy --► Destination
// Note that we must do this before step (3) because step (3) may also add headers to the HttpContent that we set up here.
var requestContent = SetupRequestBodyCopy(context.Request, isStreamingRequest, requestAborted);
destinationRequest.Content = requestContent;
// :: Step 3: Copy request headers Client --► Proxy --► Destination
await transformer.TransformRequestAsync(context, destinationRequest, destinationPrefix);
// Allow someone to custom build the request uri, otherwise provide a default for them.
var request = context.Request;
destinationRequest.RequestUri ??= RequestUtilities.MakeDestinationAddress(destinationPrefix, request.Path, request.QueryString);
Log.Proxying(_logger, destinationRequest.RequestUri.AbsoluteUri);
// TODO: What if they replace the HttpContent object? That would mess with our tracking and error handling.
return(destinationRequest, requestContent);
}
public void TryGetValues(params string[] headerValues)
{
var request = new HttpRequestMessage();
foreach (var value in headerValues)
{
request.Headers.TryAddWithoutValidation("foo", value);
}
request.Headers.TryAddWithoutValidation("bar", headerValues);
Assert.True(RequestUtilities.TryGetValues(request.Headers, "foo", out var actualValues));
Assert.Equal(headerValues, actualValues);
Assert.True(RequestUtilities.TryGetValues(request.Headers, "bar", out actualValues));
Assert.Equal(headerValues, actualValues);
}
/// <summary>
/// A callback that is invoked prior to sending the proxied request. All HttpRequestMessage
/// fields are initialized except RequestUri, which will be initialized after the
/// callback if no value is provided. The string parameter represents the destination
/// URI prefix that should be used when constructing the RequestUri. The headers
/// are copied by the base implementation, excluding some protocol headers like HTTP/2
/// pseudo headers (":authority").
/// </summary>
/// <param name="httpContext">The incoming request.</param>
/// <param name="proxyRequest">The outgoing proxy request.</param>
/// <param name="destinationPrefix">The uri prefix for the selected destination server which can be used to create
/// the RequestUri.</param>
public override async ValueTask TransformRequestAsync(HttpContext httpContext, HttpRequestMessage proxyRequest, string destinationPrefix)
{
// Copy all request headers
await base.TransformRequestAsync(httpContext, proxyRequest, destinationPrefix);
// Customize the query string:
var queryContext = new QueryTransformContext(httpContext.Request);
queryContext.Collection.Remove("param1");
queryContext.Collection["area"] = "xx2";
// Assign the custom uri. Be careful about extra slashes when concatenating here. RequestUtilities.MakeDestinationAddress is a safe default.
proxyRequest.RequestUri = RequestUtilities.MakeDestinationAddress("https://example.com", httpContext.Request.Path, queryContext.QueryString);
// Suppress the original request header, use the one from the destination Uri.
proxyRequest.Headers.Host = null;
}
public void InvalidPathCharacters()
{
var invalids = new char[]
{
// Controls
(char)0x00, (char)0x01, (char)0x02, (char)0x03, (char)0x04, (char)0x05, (char)0x06, (char)0x00, (char)0x07, (char)0x08, (char)0x09, (char)0x0A, (char)0x0B, (char)0x0C, (char)0x0D, (char)0x0E, (char)0x0F,
(char)0x10, (char)0x11, (char)0x02, (char)0x13, (char)0x14, (char)0x15, (char)0x16, (char)0x10, (char)0x17, (char)0x18, (char)0x19, (char)0x1A, (char)0x1B, (char)0x1C, (char)0x1D, (char)0x1E, (char)0x1F,
' ', '"', '#', '%', '<', '>', '?', '[', '\\', ']', '^', '`', '{', '|', '}'
};
foreach (var c in invalids)
{
var isValid = RequestUtilities.IsValidPathChar(c);
Assert.False(isValid, c.ToString());
}
}
/// <summary>
/// Removes and returns the current header value by first checking the HttpRequestMessage,
/// then the HttpContent, and falling back to the HttpContext only if
/// <see cref="RequestTransformContext.HeadersCopied"/> is not set.
/// This ordering allows multiple transforms to mutate the same header.
/// </summary>
/// <param name="context">The transform context.</param>
/// <param name="headerName">The name of the header to take.</param>
/// <returns>The requested header value, or StringValues.Empty if none.</returns>
public static StringValues TakeHeader(RequestTransformContext context, string headerName)
{
if (string.IsNullOrEmpty(headerName))
{
throw new ArgumentException($"'{nameof(headerName)}' cannot be null or empty.", nameof(headerName));
}
var proxyRequest = context.ProxyRequest;
if (RequestUtilities.TryGetValues(proxyRequest.Headers, headerName, out var existingValues))
{
proxyRequest.Headers.Remove(headerName);
}
else if (proxyRequest.Content is { } content&& RequestUtilities.TryGetValues(content.Headers, headerName, out existingValues))
{
content.Headers.Remove(headerName);
}
private static void RestoreUpgradeHeaders(HttpContext context, HttpResponseMessage response)
{
// We don't use NonValidated for the Connection header as we do want value validation.
// HttpHeaders.TryGetValues will handle the parsing and split the values for us.
if (RequestUtilities.TryGetValues(response.Headers, HeaderNames.Upgrade, out var upgradeValues) &&
response.Headers.TryGetValues(HeaderNames.Connection, out var connectionValues))
{
foreach (var value in connectionValues)
{
if (value.Equals("upgrade", StringComparison.OrdinalIgnoreCase))
{
context.Response.Headers.TryAdd(HeaderNames.Connection, value);
context.Response.Headers.TryAdd(HeaderNames.Upgrade, upgradeValues);
break;
}
}
}
}
public void Concat(string stringValues, string inputHeaderStringValues, string expectedOutput)
{
var request = new HttpRequestMessage();
foreach (var value in inputHeaderStringValues.Split(';'))
{
request.Headers.TryAddWithoutValidation("foo", value);
}
request.Headers.TryAddWithoutValidation("bar", inputHeaderStringValues.Split(';'));
var headerStringValues = request.Headers.NonValidated["foo"];
var actualValues = RequestUtilities.Concat(stringValues?.Split(';'), headerStringValues);
Assert.Equal(expectedOutput.Split(';'), actualValues);
headerStringValues = request.Headers.NonValidated["bar"];
actualValues = RequestUtilities.Concat(stringValues?.Split(';'), headerStringValues);
Assert.Equal(expectedOutput.Split(';'), actualValues);
}
public void ValidPathCharacters()
{
var valids = new char[]
{
'!', '$', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
':', ';', '=', '@',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
'_',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'~'
};
foreach (var c in valids)
{
var isValid = RequestUtilities.IsValidPathChar(c);
Assert.True(isValid, c.ToString());
}
}
/// <summary>
/// A callback that is invoked prior to sending the proxied request. All HttpRequestMessage fields are
/// initialized except RequestUri, which will be initialized after the callback if no value is provided.
/// See <see cref="RequestUtilities.MakeDestinationAddress(string, PathString, QueryString)"/> for constructing a custom request Uri.
/// The string parameter represents the destination URI prefix that should be used when constructing the RequestUri.
/// The headers are copied by the base implementation, excluding some protocol headers like HTTP/2 pseudo headers (":authority").
/// </summary>
/// <param name="httpContext">The incoming request.</param>
/// <param name="proxyRequest">The outgoing proxy request.</param>
/// <param name="destinationPrefix">The uri prefix for the selected destination server which can be used to create the RequestUri.</param>
public virtual ValueTask TransformRequestAsync(HttpContext httpContext, HttpRequestMessage proxyRequest, string destinationPrefix)
{
foreach (var header in httpContext.Request.Headers)
{
var headerName = header.Key;
var headerValue = header.Value;
if (StringValues.IsNullOrEmpty(headerValue))
{
continue;
}
// Filter out HTTP/2 pseudo headers like ":method" and ":path", those go into other fields.
if (headerName.Length > 0 && headerName[0] == ':')
{
continue;
}
RequestUtilities.AddHeader(proxyRequest, headerName, headerValue);
}
return(default);
// These intentionally do not call base because the logic here overlaps with the default header copy logic.
public override Task TransformRequestAsync(HttpContext context, HttpRequestMessage proxyRequest, string destinationPrefix)
{
var transformContext = new RequestParametersTransformContext()
{
DestinationPrefix = destinationPrefix,
HttpContext = context,
ProxyRequest = proxyRequest,
Path = context.Request.Path,
Query = new QueryTransformContext(context.Request),
};
foreach (var requestTransform in RequestTransforms)
{
requestTransform.Apply(transformContext);
}
// Allow a transform to directly set a custom RequestUri.
proxyRequest.RequestUri ??= RequestUtilities.MakeDestinationAddress(
transformContext.DestinationPrefix, transformContext.Path, transformContext.Query.QueryString);
CopyRequestHeaders(context, proxyRequest);
return(Task.CompletedTask);
}
/// <summary>
/// Adds the given header to the HttpRequestMessage or HttpContent where applicable.
/// </summary>
public static void AddHeader(RequestTransformContext context, string headerName, StringValues values)
{
RequestUtilities.AddHeader(context.ProxyRequest, headerName, values);
}
public void GetHttpMethod_Invalid_Throws(string method)
{
Assert.Throws <FormatException>(() => RequestUtilities.GetHttpMethod(method));
}
public void GetHttpMethod_Connect_Throws()
{
Assert.Throws <NotSupportedException>(() => RequestUtilities.GetHttpMethod("CONNECT"));
}
public void GetHttpMethod_Unknown_Works()
{
Assert.Same("Unknown", RequestUtilities.GetHttpMethod("Unknown").Method);
}
public void GetHttpMethod_Trace_Works()
{
Assert.Same(HttpMethod.Trace, RequestUtilities.GetHttpMethod("TRACE"));
}
public void GetHttpMethod_Delete_Works()
{
Assert.Same(HttpMethod.Delete, RequestUtilities.GetHttpMethod("DELETE"));
}
public void GetHttpMethod_Options_Works()
{
Assert.Same(HttpMethod.Options, RequestUtilities.GetHttpMethod("OPTIONS"));
}
public void GetHttpMethod_Put_Works()
{
Assert.Same(HttpMethod.Put, RequestUtilities.GetHttpMethod("PUT"));
}
/// <summary>
/// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
/// </summary>
public void Configure(IApplicationBuilder app, IHttpForwarder forwarder)
{
// Configure our own HttpMessageInvoker for outbound calls for proxy operations
var httpClient = new HttpMessageInvoker(new SocketsHttpHandler()
{
UseProxy = false,
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.None,
UseCookies = false
});
// Setup our own request transform class
var transformer = new CustomTransformer(); // or HttpTransformer.Default;
var requestOptions = new ForwarderRequestConfig {
ActivityTimeout = TimeSpan.FromSeconds(100)
};
app.UseRouting();
app.UseEndpoints(endpoints =>
{
endpoints.Map("/test/{**catch-all}", async httpContext =>
{
var error = await forwarder.SendAsync(httpContext, "https://example.com", httpClient, requestOptions,
static (context, proxyRequest) =>
{
// Customize the query string:
var queryContext = new QueryTransformContext(context.Request);
queryContext.Collection.Remove("param1");
queryContext.Collection["area"] = "xx2";
// Assign the custom uri. Be careful about extra slashes when concatenating here. RequestUtilities.MakeDestinationAddress is a safe default.
proxyRequest.RequestUri = RequestUtilities.MakeDestinationAddress("https://example.com", context.Request.Path, queryContext.QueryString);
// Suppress the original request header, use the one from the destination Uri.
proxyRequest.Headers.Host = null;
return(default);
});
// Check if the proxy operation was successful
if (error != ForwarderError.None)
{
var errorFeature = httpContext.Features.Get <IForwarderErrorFeature>();
var exception = errorFeature.Exception;
}
});
// When using IHttpForwarder for direct forwarding you are responsible for routing, destination discovery, load balancing, affinity, etc..
// For an alternate example that includes those features see BasicYarpSample.
endpoints.Map("/{**catch-all}", async httpContext =>
{
var error = await forwarder.SendAsync(httpContext, "https://example.com", httpClient, requestOptions, transformer);
// Check if the proxy operation was successful
if (error != ForwarderError.None)
{
var errorFeature = httpContext.Features.Get <IForwarderErrorFeature>();
var exception = errorFeature.Exception;
}
});
});
public void GetHttpMethod_Get_Works()
{
Assert.Same(HttpMethod.Get, RequestUtilities.GetHttpMethod("GET"));
}
public ResponseStatus Send(string username)
{
Status status = Status.NoKeyFound;
string url = string.Empty;
IDictionary <string, string> variables = new Dictionary <string, string>();
using (HttpRequest client = new HttpRequest())
{
#region Preparing client
client.IgnoreProtocolErrors = true;
client.Cookies = new CookieStorage();
#endregion
foreach (var request in _config.Requests)
{
#region Preparing Request
if (string.IsNullOrEmpty(request.UserAgent))
{
client.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36";
}
else
{
client.UserAgent = request.UserAgent;
}
if (request.Headers != null)
{
foreach (var header in request.Headers)
{
client.AddHeader(header.Key, Format(header.Value, username, ref variables));
}
}
if (request.Cookies != null)
{
foreach (var cookie in request.Cookies)
{
client.Cookies.Add(new System.Net.Cookie(cookie.Key, Format(cookie.Value, username, ref variables), "*/*", TextUtilities.UrlToHost(request.Url)));
}
}
#endregion
#region Request And Response
try
{
var response = client.Raw(RequestUtilities.GetMethod(request.Method), Format(request.Url, username, ref variables), new StringContent(Format(request.Content, username, ref variables))
{
ContentType = request.ContentType
});
string responseContent = response.ToString();
if (request.CaptureRaw)
{
responseContent = $"HTTP/{response.ProtocolVersion.ToString()} {Convert.ToInt32(response.StatusCode)} {response.StatusCode.ToString()}{Environment.NewLine}{RequestUtilities.GetHeaders(response.EnumerateHeaders())}{"Cookie: " + response.Cookies.GetCookieHeader(new Uri(Format(request.Url, username, ref variables)))}{Environment.NewLine}{Environment.NewLine}{responseContent}";
}
#region Getting Variables
if (request.GetbetweenVariables != null)
{
foreach (var x in request.GetbetweenVariables)
{
var variable = GetVariable(x, responseContent, username, ref variables, out bool found);
if (found)
{
variables.Add(variable);
}
}
}
if (request.RegexVariables != null)
{
foreach (var x in request.RegexVariables)
{
var variable = GetVariable(x, responseContent, username, ref variables, out bool found);
if (found)
{
variables.Add(variable);
}
}
}
#endregion
#region Checking For Keys
if (HasKey(responseContent, request.FailureKeys))
{
return new ResponseStatus()
{
Status = Status.Unsuccessful, Variables = new Dictionary <string, string>(variables), Url = response.Address.ToString()
}
}
;
if (HasKey(responseContent, request.SuccessKeys))
{
if (status == Status.NoKeyFound)
{
status = Status.Success;
url = response.Address.ToString();
}
}
#endregion
}
catch
{
return(new ResponseStatus()
{
Status = Status.RequestFailed, Variables = new Dictionary <string, string>(variables), Url = Format(request.Url, username, ref variables)
});
}
#endregion
}
}
return(new ResponseStatus()
{
Status = status, Variables = new Dictionary <string, string>(variables), Url = url
});
}
public void GetHttpMethod_Patch_Works()
{
Assert.Same(HttpMethod.Patch, RequestUtilities.GetHttpMethod("PATCH"));
}
public void GetHttpMethod_Head_Works()
{
Assert.Same(HttpMethod.Head, RequestUtilities.GetHttpMethod("HEAD"));
}
public void MakeDestinationAddress(string destinationPrefix, string path, string query, string expected)
{
var uri = RequestUtilities.MakeDestinationAddress(destinationPrefix, new PathString(path), new QueryString(query));
Assert.Equal(expected, uri.AbsoluteUri);
}
