/// <summary>
/// Download a string from a URL
/// </summary>
/// <param name="url">URL to download from</param>
/// <param name="authToken">An authentication token sent with the "Authorization" header.
/// Attempted to be looked up from the configuraiton if not specified</param>
/// <param name="mimeType">A mime type sent with the "Accept" header</param>
/// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param>
/// <returns>The text content returned by the server</returns>
public static string DownloadText(Uri url, string authToken = "", string mimeType = null, int timeout = 100000)
{
log.DebugFormat("About to download {0}", url.OriginalString);
WebClient agent = new RedirectingTimeoutWebClient(timeout, mimeType);
// Check whether to use an auth token for this host
if (!string.IsNullOrEmpty(authToken) ||
(ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) &&
!string.IsNullOrEmpty(authToken)))
{
log.InfoFormat("Using auth token for {0}", url.Host);
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {authToken}");
}
for (int whichAttempt = 0; whichAttempt < MaxRetries + 1; ++whichAttempt)
{
try
{
string content = agent.DownloadString(url.OriginalString);
string header = agent.ResponseHeaders.ToString();
log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
return(content);
}
catch (WebException wex) when(wex.Status != WebExceptionStatus.ProtocolError && whichAttempt < MaxRetries)
{
log.DebugFormat("Web request failed with non-protocol error, retrying in {0} milliseconds: {1}", RetryDelayMilliseconds * whichAttempt, wex.Message);
Thread.Sleep(RetryDelayMilliseconds * whichAttempt);
}
}
// Should never get here, because we don't catch any exceptions
// in the final iteration of the above for loop. They should be
// thrown to the calling code, or the call should succeed.
return(null);
}
public static string Download(string url, out string etag, string filename = null, IUser user = null)
{
TxFileManager FileTransaction = new TxFileManager();
user = user ?? new NullUser();
user.RaiseMessage("Downloading {0}", url);
// Generate a temporary file if none is provided.
if (filename == null)
{
filename = FileTransaction.GetTempFileName();
}
log.DebugFormat("Downloading {0} to {1}", url, filename);
try
{
var agent = new RedirectingTimeoutWebClient();
agent.DownloadFile(url, filename);
etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", "");
}
catch (Exception exc)
{
var wexc = exc as WebException;
if (wexc?.Status == WebExceptionStatus.ProtocolError)
{
// Get redirect if redirected.
// This is needed when redirecting from HTTPS to HTTP on .NET Core.
var response = wexc.Response as HttpWebResponse;
if (response?.StatusCode == HttpStatusCode.Redirect)
{
return(Download(response.GetResponseHeader("Location"), out etag, filename, user));
}
// Otherwise it's a valid failure from the server (probably a 404), keep it
}
// Clean up our file, it's unlikely to be complete.
// We do this even though we're using transactional files, as we may not be in a transaction.
// It's okay if this fails.
try
{
log.DebugFormat("Removing {0} after web error failure", filename);
FileTransaction.Delete(filename);
}
catch
{
// Apparently we need a catch, even if we do nothing.
}
// Look for an exception regarding the authentication.
if (Regex.IsMatch(exc.ToString(), "The authentication or decryption has failed."))
{
throw new MissingCertificateKraken("Failed downloading " + url, exc);
}
// Not the exception we were looking for! Throw it further upwards!
throw;
}
return(filename);
}
