/// <summary> /// Download a string from a URL /// </summary> /// <param name="url">URL to download from</param> /// <param name="authToken">An authentication token sent with the "Authorization" header. /// Attempted to be looked up from the configuraiton if not specified</param> /// <param name="mimeType">A mime type sent with the "Accept" header</param> /// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param> /// <returns>The text content returned by the server</returns> public static string DownloadText(Uri url, string authToken = "", string mimeType = null, int timeout = 100000) { log.DebugFormat("About to download {0}", url.OriginalString); WebClient agent = new RedirectingTimeoutWebClient(timeout, mimeType); // Check whether to use an auth token for this host if (!string.IsNullOrEmpty(authToken) || (ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) && !string.IsNullOrEmpty(authToken))) { log.InfoFormat("Using auth token for {0}", url.Host); // Send our auth token to the GitHub API (or whoever else needs one) agent.Headers.Add("Authorization", $"token {authToken}"); } for (int whichAttempt = 0; whichAttempt < MaxRetries + 1; ++whichAttempt) { try { string content = agent.DownloadString(url.OriginalString); string header = agent.ResponseHeaders.ToString(); log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content); return(content); } catch (WebException wex) when(wex.Status != WebExceptionStatus.ProtocolError && whichAttempt < MaxRetries) { log.DebugFormat("Web request failed with non-protocol error, retrying in {0} milliseconds: {1}", RetryDelayMilliseconds * whichAttempt, wex.Message); Thread.Sleep(RetryDelayMilliseconds * whichAttempt); } } // Should never get here, because we don't catch any exceptions // in the final iteration of the above for loop. They should be // thrown to the calling code, or the call should succeed. return(null); }
public static string Download(string url, out string etag, string filename = null, IUser user = null) { TxFileManager FileTransaction = new TxFileManager(); user = user ?? new NullUser(); user.RaiseMessage("Downloading {0}", url); // Generate a temporary file if none is provided. if (filename == null) { filename = FileTransaction.GetTempFileName(); } log.DebugFormat("Downloading {0} to {1}", url, filename); try { var agent = new RedirectingTimeoutWebClient(); agent.DownloadFile(url, filename); etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", ""); } catch (Exception exc) { var wexc = exc as WebException; if (wexc?.Status == WebExceptionStatus.ProtocolError) { // Get redirect if redirected. // This is needed when redirecting from HTTPS to HTTP on .NET Core. var response = wexc.Response as HttpWebResponse; if (response?.StatusCode == HttpStatusCode.Redirect) { return(Download(response.GetResponseHeader("Location"), out etag, filename, user)); } // Otherwise it's a valid failure from the server (probably a 404), keep it } // Clean up our file, it's unlikely to be complete. // We do this even though we're using transactional files, as we may not be in a transaction. // It's okay if this fails. try { log.DebugFormat("Removing {0} after web error failure", filename); FileTransaction.Delete(filename); } catch { // Apparently we need a catch, even if we do nothing. } // Look for an exception regarding the authentication. if (Regex.IsMatch(exc.ToString(), "The authentication or decryption has failed.")) { throw new MissingCertificateKraken("Failed downloading " + url, exc); } // Not the exception we were looking for! Throw it further upwards! throw; } return(filename); }