private async Task <HttpFetchResult> TryFetchWithHttp2Client(Uri url, params string[] acceptHeaders)
        {
            try
            {
                using var http2Request = new HttpRequestMessage(HttpMethod.Get, url)
                      {
                          Version = new Version(2, 0)
                      };
                http2Request.AddAcceptHeaders(acceptHeaders);
                using var result = await http.SendAsync(http2Request);

                result.EnsureSuccessStatusCode();
                var contentString = await result.Content.ReadAsStringAsync();

                logger.LogInformation("Successfully fetched {url} via HTTP/2 fallback", url);
                return(new HttpFetchResult
                {
                    Result = contentString
                });
            }
            catch (Exception http2Error)
            {
                logger.LogWarning(http2Error, "Unable to fetch {url} using HTTP/2 fallback.", url);
                return(new HttpFetchResult
                {
                    Error = http2Error
                });
            }
        }
        /// <summary>
        /// Sets the user agent to exclude CURL, which some sites (e.g. Facebook) require for programmatic fetching to work.
        /// </summary>
        /// <param name="url">The URL to fetch</param>
        /// <param name="acceptHeaders">Additional headers to set on the fetch request.</param>
        /// <returns>A fetch result containing the fetch response.</returns>
        private async Task <HttpFetchResult> TryFetchWithCurlUserAgent(Uri url, string[] acceptHeaders)
        {
            try
            {
                // Append CURL to the user agent.
                http.SetUserAgent(userAgent + " curl/7.64.1");

                using var httpRequest = new HttpRequestMessage(HttpMethod.Get, url);
                httpRequest.AddAcceptHeaders(acceptHeaders);
                var httpResponse = await http.SendAsync(httpRequest);

                httpResponse.EnsureSuccessStatusCode();
                var content = await httpResponse.Content.ReadAsStringAsync();

                return(new HttpFetchResult
                {
                    Result = content
                });
            }
            catch (Exception fetchError)
            {
                logger.LogWarning(fetchError, "Unable to fetch {url} using CURL user agent fallback.", url);
                return(new HttpFetchResult
                {
                    Error = fetchError
                });
            }
            finally
            {
                // Reset the user agent back to the default user agent.
                http.SetUserAgent(userAgent);
            }
        }
        /// <summary>
        /// Attempts to fetch a resource at the specified URL.
        /// If the fetch fails, it will attempt to fetch using HTTP/2.
        /// Failures due to encoding errors will also attempt fetch using UTF-8 encoding as a fallback.
        /// If all fetches fail, the result will contain the exception.
        /// </summary>
        /// <param name="url"></param>
        /// <param name="acceptHeaders"></param>
        /// <returns></returns>
        private async Task <HttpFetchResult> TryFetch(Uri url, params string[] acceptHeaders)
        {
            try
            {
                using var httpRequest = new HttpRequestMessage(HttpMethod.Get, url);
                httpRequest.AddAcceptHeaders(acceptHeaders);
                var httpResponse = await http.SendAsync(httpRequest);

                // If it's a 403, we have special handling for this.
                if (httpResponse.StatusCode == System.Net.HttpStatusCode.Forbidden)
                {
                    var errorMessage = !string.IsNullOrWhiteSpace(httpResponse.ReasonPhrase) ?
                                       httpResponse.ReasonPhrase :
                                       "Web server's response was 403 Forbidden.";
                    throw new HttpForbiddenException(errorMessage);
                }

                httpResponse.EnsureSuccessStatusCode();
                var content = await httpResponse.Content.ReadAsStringAsync();

                return(new HttpFetchResult
                {
                    Result = content
                });
            }
            catch (InvalidOperationException invalidOpError) when(invalidOpError.Message.Contains("The character set provided in ContentType is invalid."))
            {
                // Invalid encoding? Sometimes webpages have incorrectly set their charset / content type.
                // See if we can just parse the thing using UTF-8.
                logger.LogWarning(invalidOpError, "Unable to parse using HTTP client due to invalid ContentType. Attempting to parse using UTF-8.");
                return(await TryFetchWithForcedUtf8(url, acceptHeaders));
            }
            catch (HttpForbiddenException forbiddenError)
            {
                logger.LogWarning(forbiddenError, "Received 403 Forbidden when fetching {url}. Attempting fetch with CURL user agent fallback.");
                return(await TryFetchWithCurlUserAgent(url, acceptHeaders)); // TODO: should we always try this when we encounter exception, not just forbidden exception?
            }
            catch (Exception httpException)
            {
                logger.LogWarning(httpException, "Failed to fetch {url}. Falling back to HTTP/2 fetch.", url);
                return(await TryFetchWithHttp2Client(url, acceptHeaders));
            }
        }