/// <summary>
/// Loads the HEAD of the requested URL, and returns the URI from the returned request header.
/// </summary>
/// <param name="url">The URL to load.</param>
/// <param name="shortDescrip">Short description of the page being loaded.</param>
/// <param name="suppressNotifications">Whether to suppress notifications.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>Returns the URI, if the page is loaded. Otherwise null.</returns>
private async Task <Uri?> GetRedirectedHeaderRequestUri(string url, string?shortDescrip, SuppressNotifications suppressNotifications, CancellationToken token)
{
var(uri, url2) = GetVerifiedUrl(url);
NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications);
// Limit to no more than N parallel requests
await ss.WaitAsync(token).ConfigureAwait(false);
try
{
Cookie?cookie = ForumCookies.GetCookie(uri);
if (cookie != null)
{
ClientHandler.CookieContainer.Add(uri, cookie);
}
string?authorization = ForumAuthentications.GetAuthorization(uri);
if (authorization != null)
{
httpClient.DefaultRequestHeaders.Add("Authorization", authorization);
}
int tries = 0;
do
{
token.ThrowIfCancellationRequested();
if (tries > 0)
{
// Delay any additional attempts after the first.
await Task.Delay(retryDelay, token).ConfigureAwait(false);
// Notify the user if we're re-trying to load the page.
NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications);
}
tries++;
try
{
using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Head, uri);
// As long as we got a response (whether 200 or 404), we can extract what
// the server thinks the URL should be.
using (HttpResponseMessage response = await httpClient.SendAsync(request, token).ConfigureAwait(false))
{
return(response.RequestMessage.RequestUri);
}
}
catch (HttpRequestException e)
{
NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications);
throw;
}
catch (OperationCanceledException)
{
if (token.IsCancellationRequested)
{
// user request
throw;
}
else
{
// timeout via cancellation
logger.LogDebug($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}");
}
}
catch (TimeoutException)
{
logger.LogDebug($"Attempt to load {shortDescrip} timed out. Tries={tries}");
}
} while (tries < retryLimit);
}
finally
{
httpClient.DefaultRequestHeaders.Remove("Authorization");
ss.Release();
}
return(null);
}
/// <summary>
/// Asynchronously load a specific page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="token">Cancellation token.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <returns>Returns an HTML document, if it can be loaded.</returns>
/// <exception cref="ArgumentNullException">If url is null or empty.</exception>
/// <exception cref="ArgumentException">If url is not a valid absolute url.</exception>
private async Task <string?> GetUrlContent(Uri uri, string url, string shortDescrip,
ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
string? result = null;
int tries = 0;
DateTime expires = CacheInfo.DefaultExpiration;
NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications);
// Limit to no more than N parallel requests
await ss.WaitAsync(token).ConfigureAwait(false);
try
{
Cookie?cookie = ForumCookies.GetCookie(uri);
if (cookie != null)
{
ClientHandler.CookieContainer.Add(uri, cookie);
}
string?authorization = ForumAuthentications.GetAuthorization(uri);
if (authorization != null && !httpClient.DefaultRequestHeaders.Contains("Authorization"))
{
httpClient.DefaultRequestHeaders.Add("Authorization", authorization);
}
Task <HttpResponseMessage>?getResponseTask = null;
do
{
token.ThrowIfCancellationRequested();
if (tries > 0)
{
// Delay any additional attempts after the first.
await Task.Delay(retryDelay, token).ConfigureAwait(false);
// Notify the user if we're making another attempt to load the page.
NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications);
}
tries++;
try
{
getResponseTask = httpClient.GetAsync(uri, token).TimeoutAfter(timeout, token);
logger.LogDebug($"Get URI {uri} task ID: {getResponseTask.Id}");
using (var response = await getResponseTask.ConfigureAwait(false))
{
if (response.IsSuccessStatusCode)
{
result = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
// Get expires value
// Cannot get Expires value until we move to .NET Standard 2.0.
// If we get a successful result, we're done.
break;
}
else if (PageLoadFailed(response))
{
NotifyStatusChange(PageRequestStatusType.Failed, url,
GetFailureMessage(response, shortDescrip, url), null, suppressNotifications);
return(null);
}
else if (PageWasMoved(response))
{
url = response.Content.Headers.ContentLocation.AbsoluteUri;
uri = new Uri(url);
}
}
}
catch (OperationCanceledException)
{
if (token.IsCancellationRequested)
{
// user request
throw;
}
else
{
// timeout via cancellation
logger.LogDebug($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}");
}
}
catch (TimeoutException)
{
logger.LogDebug($"Attempt to load {shortDescrip} timed out. Tries={tries}");
}
catch (HttpRequestException e)
{
NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications);
throw;
}
} while (tries < retryLimit);
logger.LogDebug($"Finished getting URI {uri} task ID: {getResponseTask?.Id ?? 0}");
if (result == null && tries >= retryLimit)
{
httpClient.CancelPendingRequests();
}
}
catch (OperationCanceledException)
{
// If it's not a user-requested cancellation, generate a failure message.
if (!token.IsCancellationRequested)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
}
throw;
}
finally
{
ss.Release();
}
token.ThrowIfCancellationRequested();
if (result == null)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
return(null);
}
if (shouldCache == ShouldCache.Yes)
{
Cache.Add(url, result, expires);
}
NotifyStatusChange(PageRequestStatusType.Loaded, url, shortDescrip, null, suppressNotifications);
return(result);
}