public static string DownloadText(Uri url, string authToken = "")
{
log.DebugFormat("About to download {0}", url.OriginalString);
try
{
WebClient agent = MakeDefaultHttpClient();
// Check whether to use an auth token for this host
if (!string.IsNullOrEmpty(authToken) ||
(Win32Registry.TryGetAuthToken(url.Host, out authToken) &&
!string.IsNullOrEmpty(authToken)))
{
log.InfoFormat("Using auth token for {0}", url.Host);
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {authToken}");
}
return(agent.DownloadString(url.OriginalString));
}
catch (Exception)
{
try
{
log.InfoFormat("Download failed, trying with curlsharp...");
var content = string.Empty;
var client = Curl.CreateEasy(url.OriginalString, delegate(byte[] buf, int size, int nmemb, object extraData)
{
content += Encoding.UTF8.GetString(buf);
return(size * nmemb);
});
using (client)
{
var result = client.Perform();
if (result != CurlCode.Ok)
{
throw new Exception("Curl download failed with error " + result);
}
log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);
return(content);
}
}
catch (Exception e)
{
throw new Kraken("Downloading using cURL failed", e);
}
}
}
public static string DownloadText(Uri url, string authToken = "")
{
log.DebugFormat("About to download {0}", url.OriginalString);
try
{
WebClient agent = MakeDefaultHttpClient();
// Check whether to use an auth token for this host
if (!string.IsNullOrEmpty(authToken) ||
(ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) &&
!string.IsNullOrEmpty(authToken)))
{
log.InfoFormat("Using auth token for {0}", url.Host);
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {authToken}");
}
return(agent.DownloadString(url.OriginalString));
}
catch (Exception e)
{
log.InfoFormat(e.ToString());
log.InfoFormat("Download failed, trying with curlsharp...");
var content = string.Empty;
var client = Curl.CreateEasy(url.OriginalString, delegate(byte[] buf, int size, int nmemb, object extraData)
{
content += Encoding.UTF8.GetString(buf);
return(size * nmemb);
});
client.SetOpt(CurlOption.FailOnError, true);
using (client)
{
var result = client.Perform();
var returnCode = client.ResponseCode;
if (result != CurlCode.Ok)
{
throw new WebException(
String.Format("Curl download failed with error {0} ({1})", result, returnCode),
e
);
}
log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);
return(content);
}
}
}
public static string DownloadText(string url)
{
Log.DebugFormat("About to download {0}", url);
var agent = MakeDefaultHttpClient();
try
{
return(agent.DownloadString(url));
}
catch (Exception)
{
try
{
Log.InfoFormat("Download failed, trying with curlsharp...");
var content = string.Empty;
var client = Curl.CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData)
{
content += Encoding.UTF8.GetString(buf);
return(size * nmemb);
});
using (client)
{
var result = client.Perform();
if (result != CurlCode.Ok)
{
throw new Exception("Curl download failed with error " + result);
}
Log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);
return(content);
}
}
catch (Exception e)
{
throw new Kraken("Downloading using cURL failed", e);
}
}
}
public static string Download(string url, string filename = null, IUser user = null)
{
user = user ?? new NullUser();
user.RaiseMessage("Downloading {0}", url);
// Generate a temporary file if none is provided.
if (filename == null)
{
filename = FileTransaction.GetTempFileName();
}
Log.DebugFormat("Downloading {0} to {1}", url, filename);
var agent = MakeDefaultHttpClient();
try
{
agent.DownloadFile(url, filename);
}
catch (Exception ex)
{
Log.InfoFormat("Download failed, trying with curlsharp...");
try
{
Curl.Init();
using (FileStream stream = File.OpenWrite(filename))
using (var curl = Curl.CreateEasy(url, stream))
{
CurlCode result = curl.Perform();
if (result != CurlCode.Ok)
{
throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
}
else
{
Log.Debug("curlsharp download successful");
}
}
Curl.CleanUp();
return(filename);
}
catch
{
// D'oh, failed again. Fall through to clean-up handling.
}
// Clean up our file, it's unlikely to be complete.
// We do this even though we're using transactional files, as we may not be in a transaction.
// It's okay if this fails.
try
{
Log.DebugFormat("Removing {0} after web error failure", filename);
FileTransaction.Delete(filename);
}
catch
{
// Apparently we need a catch, even if we do nothing.
}
// Look for an exception regarding the authentication.
if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
{
throw new MissingCertificateKraken("Failed downloading " + url, ex);
}
// Not the exception we were looking for! Throw it further upwards!
throw;
}
return(filename);
}
/// <summary>
/// Use curlsharp to handle our downloads.
/// </summary>
private void DownloadCurl()
{
log.Debug("Curlsharp async downloader engaged");
// Make sure our environment is set up.
Curl.Init();
// We'd *like* to use CurlMulti, but it just hangs when I try to retrieve
// messages from it. So we're spawning a thread for each curleasy that does
// the same thing. Ends up this is a little easier in handling, anyway.
for (int i = 0; i < downloads.Count; i++)
{
log.DebugFormat("Downloading {0}", downloads[i].url);
// Encode spaces to avoid confusing URL parsers
User.RaiseMessage("Downloading \"{0}\" (libcurl)",
downloads[i].url.ToString().Replace(" ", "%20"));
// Open our file, and make an easy object...
FileStream stream = File.OpenWrite(downloads[i].path);
CurlEasy easy = Curl.CreateEasy(downloads[i].url, stream);
// We need a separate variable for our closure, this is it.
int index = i;
// Curl recommends xferinfofunction, but this doesn't seem to
// be supported by curlsharp, so we use the progress function
// instead.
easy.ProgressFunction = delegate(object extraData, double dlTotal, double dlNow, double ulTotal, double ulNow)
{
log.DebugFormat("Progress function called... {0}/{1}", dlNow, dlTotal);
int percent;
if (dlTotal > 0)
{
percent = (int)dlNow * 100 / (int)dlTotal;
}
else
{
log.Debug("Unknown download size, skipping progress.");
return(0);
}
FileProgressReport(
index,
percent,
Convert.ToInt64(dlNow),
Convert.ToInt64(dlTotal)
);
// If the user has told us to cancel, then bail out now.
if (download_canceled)
{
log.InfoFormat("Bailing out of download {0} at user request", index);
// Bail out!
return(1);
}
// Returning 0 means we want to continue the download.
return(0);
};
// Download, little curl, fulfill your destiny!
Thread thread = new Thread(new ThreadStart(delegate
{
CurlWatchThread(index, easy, stream);
}));
// Keep track of our threads so we can clean them up later.
curl_threads.Add(thread);
// Background threads will mostly look after themselves.
thread.IsBackground = true;
// Let's go!
thread.Start();
}
}
public static string Download(string url, out string etag, string filename = null, IUser user = null)
{
TxFileManager FileTransaction = new TxFileManager();
user = user ?? new NullUser();
user.RaiseMessage("Downloading {0}", url);
// Generate a temporary file if none is provided.
if (filename == null)
{
filename = FileTransaction.GetTempFileName();
}
log.DebugFormat("Downloading {0} to {1}", url, filename);
try
{
var agent = MakeDefaultHttpClient();
agent.DownloadFile(url, filename);
etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", "");
}
// Explicit redirect handling. This is needed when redirecting from HTTPS to HTTP on .NET Core.
catch (WebException ex)
{
if (ex.Status != WebExceptionStatus.ProtocolError)
{
throw;
}
HttpWebResponse response = ex.Response as HttpWebResponse;
if (response?.StatusCode != HttpStatusCode.Redirect)
{
throw;
}
return(Download(response.GetResponseHeader("Location"), out etag, filename, user));
}
catch (Exception e)
{
log.InfoFormat("Native download failed, trying with CurlSharp...");
etag = null;
try
{
Curl.Init();
using (FileStream stream = File.OpenWrite(filename))
{
string header = string.Empty;
var client = Curl.CreateEasy(url, stream, delegate(byte[] buf, int size, int nmemb, object extraData)
{
header += Encoding.UTF8.GetString(buf);
return(size * nmemb);
});
using (client)
{
var result = client.Perform();
var returnCode = client.ResponseCode;
if (result != CurlCode.Ok || returnCode >= 300)
{
// Always log if it's an error.
log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
WebException curlException =
new WebException($"Curl download failed with status {returnCode}.");
throw new NativeAndCurlDownloadFailedKraken(
new List <Exception> {
e, curlException
},
url.ToString(), header, null, returnCode
);
}
else
{
// Only log if debug flag is set.
log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
}
}
}
Curl.CleanUp();
return(filename);
}
catch
{
// D'oh, failed again. Fall through to clean-up handling.
}
// Clean up our file, it's unlikely to be complete.
// We do this even though we're using transactional files, as we may not be in a transaction.
// It's okay if this fails.
try
{
log.DebugFormat("Removing {0} after web error failure", filename);
FileTransaction.Delete(filename);
}
catch
{
// Apparently we need a catch, even if we do nothing.
}
// Look for an exception regarding the authentication.
if (Regex.IsMatch(e.ToString(), "The authentication or decryption has failed."))
{
throw new MissingCertificateKraken("Failed downloading " + url, e);
}
// Not the exception we were looking for! Throw it further upwards!
throw;
}
return(filename);
}
public static string DownloadText(Uri url, string authToken = "")
{
log.DebugFormat("About to download {0}", url.OriginalString);
try
{
WebClient agent = MakeDefaultHttpClient();
// Check whether to use an auth token for this host
if (!string.IsNullOrEmpty(authToken) ||
(ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) &&
!string.IsNullOrEmpty(authToken)))
{
log.InfoFormat("Using auth token for {0}", url.Host);
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {authToken}");
}
string content = agent.DownloadString(url.OriginalString);
string header = agent.ResponseHeaders.ToString();
log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
return(content);
}
catch (Exception e)
{
log.InfoFormat("Native download failed, trying with CurlSharp...");
string content = string.Empty;
string header = string.Empty;
var client = Curl.CreateEasy(url.OriginalString,
delegate(byte[] buf, int size, int nmemb, object extraData)
{
content += Encoding.UTF8.GetString(buf);
return(size * nmemb);
},
delegate(byte[] buf, int size, int nmemb, object extraData)
{
header += Encoding.UTF8.GetString(buf);
return(size * nmemb);
}
);
using (client)
{
var result = client.Perform();
var returnCode = client.ResponseCode;
if (result != CurlCode.Ok || returnCode >= 300)
{
// Always log if it's an error.
log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
WebException curlException = new WebException($"Curl download failed with status {returnCode}.");
throw new NativeAndCurlDownloadFailedKraken(
new List <Exception> {
e, curlException
},
url.ToString(), header, content, returnCode
);
}
else
{
// Only log if debug flag is set
log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
return(content);
}
}
}
}
public static string Download(string url, out string etag, string filename = null, IUser user = null)
{
TxFileManager FileTransaction = new TxFileManager();
user = user ?? new NullUser();
user.RaiseMessage("Downloading {0}", url);
// Generate a temporary file if none is provided.
if (filename == null)
{
filename = FileTransaction.GetTempFileName();
}
log.DebugFormat("Downloading {0} to {1}", url, filename);
try
{
var agent = MakeDefaultHttpClient();
agent.DownloadFile(url, filename);
etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", "");
}
// Explicit redirect handling. This is needed when redirecting from HTTPS to HTTP on .NET Core.
catch (WebException ex)
{
if (ex.Status != WebExceptionStatus.ProtocolError)
{
throw;
}
HttpWebResponse response = ex.Response as HttpWebResponse;
if (response.StatusCode != HttpStatusCode.Redirect)
{
throw;
}
return(Net.Download(response.GetResponseHeader("Location"), out etag, filename, user));
}
catch (Exception ex)
{
log.InfoFormat("Download failed, trying with curlsharp...");
etag = null;
try
{
Curl.Init();
using (FileStream stream = File.OpenWrite(filename))
using (var curl = Curl.CreateEasy(url, stream))
{
CurlCode result = curl.Perform();
if (result != CurlCode.Ok)
{
throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
}
else
{
log.Debug("curlsharp download successful");
}
}
Curl.CleanUp();
return(filename);
}
catch
{
// D'oh, failed again. Fall through to clean-up handling.
}
// Clean up our file, it's unlikely to be complete.
// We do this even though we're using transactional files, as we may not be in a transaction.
// It's okay if this fails.
try
{
log.DebugFormat("Removing {0} after web error failure", filename);
FileTransaction.Delete(filename);
}
catch
{
// Apparently we need a catch, even if we do nothing.
}
// Look for an exception regarding the authentication.
if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
{
throw new MissingCertificateKraken("Failed downloading " + url, ex);
}
// Not the exception we were looking for! Throw it further upwards!
throw;
}
return(filename);
}