// Supported formats:
// a=1&b=c (isUnprefixedExtraParameters)
// §a=1&b=c
// .link-next
// .link-next§§preserve
// .link-next (alwaysPreserveRemainingParameters)
// .link-next§§preserve§§a={z}
public static bool UpdateNextLink(ref LazyUri modifiableUrl, HtmlNode node, string rule, bool isUnprefixedExtraParameters = false, bool alwaysPreserveRemainingParameters = false)
{
var anyVarying = false;
bool preserve = alwaysPreserveRemainingParameters;
if (!isUnprefixedExtraParameters)
{
string additionalChanges = null;
if (!rule.StartsWith("§"))
{
if (rule.Contains("§§preserve"))
{
preserve = true;
rule = rule.Replace("§§preserve", string.Empty);
}
if (rule.Contains("§§"))
{
additionalChanges = rule.CaptureAfter("§§");
rule = rule.CaptureBefore("§§");
}
var nextlink = node.FindSingle(rule);
if (nextlink == null)
{
modifiableUrl = null; return(false);
}
var url = nextlink.TryGetLinkUrl();
if (url == null)
{
url = nextlink?.TryGetValue()?.AsUri();
}
if (!HttpUtils.IsHttp(url))
{
modifiableUrl = null; return(false);
}
if (!string.IsNullOrEmpty(url.Fragment))
{
url = url.GetLeftPart_UriPartial_Query().AsUri();
}
var defaults = preserve ? modifiableUrl.QueryParameters.Concat(modifiableUrl.FragmentParameters).ToList() : null;
modifiableUrl = new LazyUri(url);
if (defaults != null)
{
foreach (var kv in defaults)
{
if (kv.Key.StartsWith("$json-query-") && modifiableUrl.GetQueryParameter(kv.Key.CaptureBetween("-query-", "-")) != null)
{
continue;
}
if (modifiableUrl.GetQueryParameter(kv.Key) == null && modifiableUrl.GetFragmentParameter(kv.Key) == null)
{
if (kv.Key.StartsWith("$"))
{
modifiableUrl.AppendFragmentParameter(kv.Key, kv.Value);
}
else
{
modifiableUrl.AppendQueryParameter(kv.Key, kv.Value);
}
}
}
}
anyVarying = true;
if (additionalChanges == null)
{
return(anyVarying);
}
}
if (additionalChanges != null)
{
rule = additionalChanges;
}
else
{
rule = rule.Substring(1);
}
}
var z = HttpUtils.GetParameters(rule);
foreach (var kv in z)
{
var val = kv.Value;
var key = kv.Key;
if (key.StartsWith("£"))
{
key = "$" + key.Substring(1);
}
if (val == "{delete}")
{
if (key.StartsWith("$"))
{
modifiableUrl.RemoveFragmentParameter(key);
}
else
{
modifiableUrl.RemoveQueryParameter(key);
}
continue;
}
if (val.StartsWith("{") && val.EndsWith("}"))
{
val = val.Substring(1, val.Length - 2);
var optional = false;
var leaveUnchanged = false;
if (val.StartsWith("optional:"))
{
optional = true; val = val.CaptureAfter(":");
}
if (val.StartsWith("unchanged:"))
{
leaveUnchanged = true; val = val.CaptureAfter(":");
}
var v = node.TryGetValue(val);
anyVarying = true;
if (v == null)
{
if (leaveUnchanged)
{
continue;
}
if (optional)
{
if (key.StartsWith("$"))
{
modifiableUrl.RemoveFragmentParameter(key);
}
else
{
modifiableUrl.RemoveQueryParameter(key);
}
continue;
}
modifiableUrl = null;
return(anyVarying);
}
val = v;
}
if (key.StartsWith("$"))
{
modifiableUrl.AppendFragmentParameter(key, val);
}
else
{
modifiableUrl.AppendQueryParameter(key, val);
}
}
return(anyVarying);
}
internal static async Task <HttpResponseInfo> SendAsync(this LazyUri url, WebRequestOptions options, HttpRequestMessageBox messageBox, bool alwaysCatchAndForbidRedirects = false, bool keepResponseAliveOnError = false, bool synchronous = false)
{
HttpUtils.EnsureInitialized();
if (!synchronous)
{
await Utils.CheckLocalFileAccessAsync(url);
}
Utils.RaiseWebRequestEvent(url, false);
HttpResponseMessage result = null;
LazyUri previousResponse2 = null;
try
{
if (options == WebRequestOptions.DefaultOptions)
{
throw new ArgumentException();
}
if (options.WaitBefore.Ticks != 0 && !synchronous)
{
await TaskEx.Delay(options.WaitBefore);
}
LazyUri previousResponse1 = null;
previousResponse2 = url.Clone();
previousResponse2 = MaybeAddAdditionalQueryParameters(previousResponse2, options);
var redirectIndex = 0;
while (true)
{
#if WEBCLIENT
HttpContent requestContent = null;
#endif
var message = messageBox?.PrebuiltRequest ?? CreateRequestInternal(previousResponse2, options, true, redirectIndex
#if WEBCLIENT
, out requestContent
#endif
);
if (messageBox != null)
{
messageBox.Dispose();
messageBox.Message = message;
}
#if WEBCLIENT
if (requestContent != null)
{
if (requestContent.ContentType != null)
{
message.ContentType = requestContent.ContentType;
}
if (requestContent.ContentDisposition != null)
{
message.Headers["Content-Disposition"] = requestContent.ContentDisposition;
}
using (var req = await message.GetRequestStreamAsync())
{
await requestContent.CopyToAsync(req);
}
}
result = (HttpWebResponse)await message.GetResponseAsync();
#else
message.Properties["ShamanURL"] = url;
if (options.CustomHttpClient != null)
{
result = await options.CustomHttpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead);
}
else
{
if (defaultHttpClient == null)
{
defaultHttpClient = CreateHttpClient();
}
result = messageBox?.PrebuiltResponse ?? await defaultHttpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead);
}
#endif
#if !WEBCLIENT
if (result.Content != null && result.Content.Headers.ContentType != null && result.Content.Headers.ContentType.CharSet == "utf8")
{
result.Content.Headers.ContentType.CharSet = "utf-8";
}
#endif
if ((int)result.StatusCode >= 400)
{
if (!keepResponseAliveOnError)
{
result.Dispose();
}
// Hackish, change purpose of enumeration type
throw new WebException("The web server returned: " + result.StatusCode.ToString(), (WebExceptionStatus)result.StatusCode);
}
#if WEBCLIENT
var zz = result.Headers["Location"];
var redirectUrlNative = zz != null?HttpUtils.GetAbsoluteUri(url.PathConsistentUrl, zz) : null;
#else
var redirectUrlNative = result.Headers.Location;
#endif
if (redirectUrlNative == null)
{
return(new HttpResponseInfo()
{
RespondingUrl = previousResponse2, Response = result
});
}
else
{
if (alwaysCatchAndForbidRedirects)
{
return new HttpResponseInfo()
{
Response = result, RespondingUrl = previousResponse2, Exception = new WebException("Unexpected redirect", HttpUtils.Error_UnexpectedRedirect)
}
}
;
result.Dispose();
var redirectUrl = new LazyUri(redirectUrlNative);
if (!redirectUrl.IsAbsoluteUri)
{
redirectUrl = new LazyUri(new Uri(previousResponse2.PathConsistentUrl, redirectUrlNative));
}
if (options != null && !options.AllowRedirects)
{
throw new WebException("Unexpected redirect was received.", HttpUtils.Error_UnexpectedRedirect);
}
if (redirectIndex == Configuration_MaximumNumberOfRedirects)
{
throw new WebException("The maximum number of redirects has been reached.", HttpUtils.Error_MaximumNumberOfRedirectsExceeded);
}
if (!(redirectIndex == 0 && options != null && (options.PostData != null || options.PostString != null)))
{
if ((
(previousResponse1 != null && HttpUtils.UrisEqual(redirectUrl.PathAndQueryConsistentUrl, previousResponse1.PathAndQueryConsistentUrl)) ||
HttpUtils.UrisEqual(redirectUrl, previousResponse2)))
{
if (url.GetFragmentParameter("$allow-same-redirect") == "1")
{
if (!synchronous)
{
#if NET35
await TaskEx.Delay(Configuration_SameRedirectDelayTimeMs);
#else
await Task.Delay(Configuration_SameRedirectDelayTimeMs);
#endif
}
}
else
{
throw new WebException("The server isn't redirecting the requested resource properly.", HttpUtils.Error_RedirectLoopDetected);
}
}
}
previousResponse1 = previousResponse2;
previousResponse2 = redirectUrl;
redirectIndex++;
}
}
}
catch (Exception ex)
{
var orig = ex;
#if !WEBCLIENT
var hre = ex as HttpRequestException;
if (hre != null && hre.InnerException != null)
{
ex = hre.InnerException;
}
#endif
if (alwaysCatchAndForbidRedirects)
{
return new HttpResponseInfo()
{
Exception = ex, Response = result, RespondingUrl = previousResponse2
}
}
;
else if (ex == orig)
{
throw;
}
else
{
throw ex.Rethrow();
}
}
}