/// <summary> /// Loads the HEAD of the requested URL, and returns the URI from the returned request header. /// </summary> /// <param name="url">The URL to load.</param> /// <param name="shortDescrip">Short description of the page being loaded.</param> /// <param name="suppressNotifications">Whether to suppress notifications.</param> /// <param name="token">Cancellation token.</param> /// <returns>Returns the URI, if the page is loaded. Otherwise null.</returns> private async Task <Uri?> GetRedirectedHeaderRequestUri(string url, string?shortDescrip, SuppressNotifications suppressNotifications, CancellationToken token) { var(uri, url2) = GetVerifiedUrl(url); NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications); // Limit to no more than N parallel requests await ss.WaitAsync(token).ConfigureAwait(false); try { Cookie?cookie = ForumCookies.GetCookie(uri); if (cookie != null) { ClientHandler.CookieContainer.Add(uri, cookie); } string?authorization = ForumAuthentications.GetAuthorization(uri); if (authorization != null) { httpClient.DefaultRequestHeaders.Add("Authorization", authorization); } int tries = 0; do { token.ThrowIfCancellationRequested(); if (tries > 0) { // Delay any additional attempts after the first. await Task.Delay(retryDelay, token).ConfigureAwait(false); // Notify the user if we're re-trying to load the page. NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications); } tries++; try { using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Head, uri); // As long as we got a response (whether 200 or 404), we can extract what // the server thinks the URL should be. using (HttpResponseMessage response = await httpClient.SendAsync(request, token).ConfigureAwait(false)) { return(response.RequestMessage.RequestUri); } } catch (HttpRequestException e) { NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications); throw; } catch (OperationCanceledException) { if (token.IsCancellationRequested) { // user request throw; } else { // timeout via cancellation logger.LogDebug($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}"); } } catch (TimeoutException) { logger.LogDebug($"Attempt to load {shortDescrip} timed out. Tries={tries}"); } } while (tries < retryLimit); } finally { httpClient.DefaultRequestHeaders.Remove("Authorization"); ss.Release(); } return(null); }
/// <summary> /// Asynchronously load a specific page. /// </summary> /// <param name="url">The URL of the page to load. Cannot be null.</param> /// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param> /// <param name="caching">Indicator of whether to query the cache for the requested page.</param> /// <param name="token">Cancellation token.</param> /// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param> /// <returns>Returns an HTML document, if it can be loaded.</returns> /// <exception cref="ArgumentNullException">If url is null or empty.</exception> /// <exception cref="ArgumentException">If url is not a valid absolute url.</exception> private async Task <string?> GetUrlContent(Uri uri, string url, string shortDescrip, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token) { string? result = null; int tries = 0; DateTime expires = CacheInfo.DefaultExpiration; NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications); // Limit to no more than N parallel requests await ss.WaitAsync(token).ConfigureAwait(false); try { Cookie?cookie = ForumCookies.GetCookie(uri); if (cookie != null) { ClientHandler.CookieContainer.Add(uri, cookie); } string?authorization = ForumAuthentications.GetAuthorization(uri); if (authorization != null && !httpClient.DefaultRequestHeaders.Contains("Authorization")) { httpClient.DefaultRequestHeaders.Add("Authorization", authorization); } Task <HttpResponseMessage>?getResponseTask = null; do { token.ThrowIfCancellationRequested(); if (tries > 0) { // Delay any additional attempts after the first. await Task.Delay(retryDelay, token).ConfigureAwait(false); // Notify the user if we're making another attempt to load the page. NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications); } tries++; try { getResponseTask = httpClient.GetAsync(uri, token).TimeoutAfter(timeout, token); logger.LogDebug($"Get URI {uri} task ID: {getResponseTask.Id}"); using (var response = await getResponseTask.ConfigureAwait(false)) { if (response.IsSuccessStatusCode) { result = await response.Content.ReadAsStringAsync().ConfigureAwait(false); // Get expires value // Cannot get Expires value until we move to .NET Standard 2.0. // If we get a successful result, we're done. break; } else if (PageLoadFailed(response)) { NotifyStatusChange(PageRequestStatusType.Failed, url, GetFailureMessage(response, shortDescrip, url), null, suppressNotifications); return(null); } else if (PageWasMoved(response)) { url = response.Content.Headers.ContentLocation.AbsoluteUri; uri = new Uri(url); } } } catch (OperationCanceledException) { if (token.IsCancellationRequested) { // user request throw; } else { // timeout via cancellation logger.LogDebug($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}"); } } catch (TimeoutException) { logger.LogDebug($"Attempt to load {shortDescrip} timed out. Tries={tries}"); } catch (HttpRequestException e) { NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications); throw; } } while (tries < retryLimit); logger.LogDebug($"Finished getting URI {uri} task ID: {getResponseTask?.Id ?? 0}"); if (result == null && tries >= retryLimit) { httpClient.CancelPendingRequests(); } } catch (OperationCanceledException) { // If it's not a user-requested cancellation, generate a failure message. if (!token.IsCancellationRequested) { NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications); } throw; } finally { ss.Release(); } token.ThrowIfCancellationRequested(); if (result == null) { NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications); return(null); } if (shouldCache == ShouldCache.Yes) { Cache.Add(url, result, expires); } NotifyStatusChange(PageRequestStatusType.Loaded, url, shortDescrip, null, suppressNotifications); return(result); }