// Supported formats: // a=1&b=c (isUnprefixedExtraParameters) // §a=1&b=c // .link-next // .link-next§§preserve // .link-next (alwaysPreserveRemainingParameters) // .link-next§§preserve§§a={z} public static bool UpdateNextLink(ref LazyUri modifiableUrl, HtmlNode node, string rule, bool isUnprefixedExtraParameters = false, bool alwaysPreserveRemainingParameters = false) { var anyVarying = false; bool preserve = alwaysPreserveRemainingParameters; if (!isUnprefixedExtraParameters) { string additionalChanges = null; if (!rule.StartsWith("§")) { if (rule.Contains("§§preserve")) { preserve = true; rule = rule.Replace("§§preserve", string.Empty); } if (rule.Contains("§§")) { additionalChanges = rule.CaptureAfter("§§"); rule = rule.CaptureBefore("§§"); } var nextlink = node.FindSingle(rule); if (nextlink == null) { modifiableUrl = null; return(false); } var url = nextlink.TryGetLinkUrl(); if (url == null) { url = nextlink?.TryGetValue()?.AsUri(); } if (!HttpUtils.IsHttp(url)) { modifiableUrl = null; return(false); } if (!string.IsNullOrEmpty(url.Fragment)) { url = url.GetLeftPart_UriPartial_Query().AsUri(); } var defaults = preserve ? modifiableUrl.QueryParameters.Concat(modifiableUrl.FragmentParameters).ToList() : null; modifiableUrl = new LazyUri(url); if (defaults != null) { foreach (var kv in defaults) { if (kv.Key.StartsWith("$json-query-") && modifiableUrl.GetQueryParameter(kv.Key.CaptureBetween("-query-", "-")) != null) { continue; } if (modifiableUrl.GetQueryParameter(kv.Key) == null && modifiableUrl.GetFragmentParameter(kv.Key) == null) { if (kv.Key.StartsWith("$")) { modifiableUrl.AppendFragmentParameter(kv.Key, kv.Value); } else { modifiableUrl.AppendQueryParameter(kv.Key, kv.Value); } } } } anyVarying = true; if (additionalChanges == null) { return(anyVarying); } } if (additionalChanges != null) { rule = additionalChanges; } else { rule = rule.Substring(1); } } var z = HttpUtils.GetParameters(rule); foreach (var kv in z) { var val = kv.Value; var key = kv.Key; if (key.StartsWith("£")) { key = "$" + key.Substring(1); } if (val == "{delete}") { if (key.StartsWith("$")) { modifiableUrl.RemoveFragmentParameter(key); } else { modifiableUrl.RemoveQueryParameter(key); } continue; } if (val.StartsWith("{") && val.EndsWith("}")) { val = val.Substring(1, val.Length - 2); var optional = false; var leaveUnchanged = false; if (val.StartsWith("optional:")) { optional = true; val = val.CaptureAfter(":"); } if (val.StartsWith("unchanged:")) { leaveUnchanged = true; val = val.CaptureAfter(":"); } var v = node.TryGetValue(val); anyVarying = true; if (v == null) { if (leaveUnchanged) { continue; } if (optional) { if (key.StartsWith("$")) { modifiableUrl.RemoveFragmentParameter(key); } else { modifiableUrl.RemoveQueryParameter(key); } continue; } modifiableUrl = null; return(anyVarying); } val = v; } if (key.StartsWith("$")) { modifiableUrl.AppendFragmentParameter(key, val); } else { modifiableUrl.AppendQueryParameter(key, val); } } return(anyVarying); }
internal static async Task <HttpResponseInfo> SendAsync(this LazyUri url, WebRequestOptions options, HttpRequestMessageBox messageBox, bool alwaysCatchAndForbidRedirects = false, bool keepResponseAliveOnError = false, bool synchronous = false) { HttpUtils.EnsureInitialized(); if (!synchronous) { await Utils.CheckLocalFileAccessAsync(url); } Utils.RaiseWebRequestEvent(url, false); HttpResponseMessage result = null; LazyUri previousResponse2 = null; try { if (options == WebRequestOptions.DefaultOptions) { throw new ArgumentException(); } if (options.WaitBefore.Ticks != 0 && !synchronous) { await TaskEx.Delay(options.WaitBefore); } LazyUri previousResponse1 = null; previousResponse2 = url.Clone(); previousResponse2 = MaybeAddAdditionalQueryParameters(previousResponse2, options); var redirectIndex = 0; while (true) { #if WEBCLIENT HttpContent requestContent = null; #endif var message = messageBox?.PrebuiltRequest ?? CreateRequestInternal(previousResponse2, options, true, redirectIndex #if WEBCLIENT , out requestContent #endif ); if (messageBox != null) { messageBox.Dispose(); messageBox.Message = message; } #if WEBCLIENT if (requestContent != null) { if (requestContent.ContentType != null) { message.ContentType = requestContent.ContentType; } if (requestContent.ContentDisposition != null) { message.Headers["Content-Disposition"] = requestContent.ContentDisposition; } using (var req = await message.GetRequestStreamAsync()) { await requestContent.CopyToAsync(req); } } result = (HttpWebResponse)await message.GetResponseAsync(); #else message.Properties["ShamanURL"] = url; if (options.CustomHttpClient != null) { result = await options.CustomHttpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead); } else { if (defaultHttpClient == null) { defaultHttpClient = CreateHttpClient(); } result = messageBox?.PrebuiltResponse ?? await defaultHttpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead); } #endif #if !WEBCLIENT if (result.Content != null && result.Content.Headers.ContentType != null && result.Content.Headers.ContentType.CharSet == "utf8") { result.Content.Headers.ContentType.CharSet = "utf-8"; } #endif if ((int)result.StatusCode >= 400) { if (!keepResponseAliveOnError) { result.Dispose(); } // Hackish, change purpose of enumeration type throw new WebException("The web server returned: " + result.StatusCode.ToString(), (WebExceptionStatus)result.StatusCode); } #if WEBCLIENT var zz = result.Headers["Location"]; var redirectUrlNative = zz != null?HttpUtils.GetAbsoluteUri(url.PathConsistentUrl, zz) : null; #else var redirectUrlNative = result.Headers.Location; #endif if (redirectUrlNative == null) { return(new HttpResponseInfo() { RespondingUrl = previousResponse2, Response = result }); } else { if (alwaysCatchAndForbidRedirects) { return new HttpResponseInfo() { Response = result, RespondingUrl = previousResponse2, Exception = new WebException("Unexpected redirect", HttpUtils.Error_UnexpectedRedirect) } } ; result.Dispose(); var redirectUrl = new LazyUri(redirectUrlNative); if (!redirectUrl.IsAbsoluteUri) { redirectUrl = new LazyUri(new Uri(previousResponse2.PathConsistentUrl, redirectUrlNative)); } if (options != null && !options.AllowRedirects) { throw new WebException("Unexpected redirect was received.", HttpUtils.Error_UnexpectedRedirect); } if (redirectIndex == Configuration_MaximumNumberOfRedirects) { throw new WebException("The maximum number of redirects has been reached.", HttpUtils.Error_MaximumNumberOfRedirectsExceeded); } if (!(redirectIndex == 0 && options != null && (options.PostData != null || options.PostString != null))) { if (( (previousResponse1 != null && HttpUtils.UrisEqual(redirectUrl.PathAndQueryConsistentUrl, previousResponse1.PathAndQueryConsistentUrl)) || HttpUtils.UrisEqual(redirectUrl, previousResponse2))) { if (url.GetFragmentParameter("$allow-same-redirect") == "1") { if (!synchronous) { #if NET35 await TaskEx.Delay(Configuration_SameRedirectDelayTimeMs); #else await Task.Delay(Configuration_SameRedirectDelayTimeMs); #endif } } else { throw new WebException("The server isn't redirecting the requested resource properly.", HttpUtils.Error_RedirectLoopDetected); } } } previousResponse1 = previousResponse2; previousResponse2 = redirectUrl; redirectIndex++; } } } catch (Exception ex) { var orig = ex; #if !WEBCLIENT var hre = ex as HttpRequestException; if (hre != null && hre.InnerException != null) { ex = hre.InnerException; } #endif if (alwaysCatchAndForbidRedirects) { return new HttpResponseInfo() { Exception = ex, Response = result, RespondingUrl = previousResponse2 } } ; else if (ex == orig) { throw; } else { throw ex.Rethrow(); } } }