Beispiel #1
0
        public static string DownloadText(Uri url, string authToken = "")
        {
            log.DebugFormat("About to download {0}", url.OriginalString);

            try
            {
                WebClient agent = MakeDefaultHttpClient();

                // Check whether to use an auth token for this host
                if (!string.IsNullOrEmpty(authToken) ||
                    (Win32Registry.TryGetAuthToken(url.Host, out authToken) &&
                     !string.IsNullOrEmpty(authToken)))
                {
                    log.InfoFormat("Using auth token for {0}", url.Host);
                    // Send our auth token to the GitHub API (or whoever else needs one)
                    agent.Headers.Add("Authorization", $"token {authToken}");
                }

                return(agent.DownloadString(url.OriginalString));
            }
            catch (Exception)
            {
                try
                {
                    log.InfoFormat("Download failed, trying with curlsharp...");

                    var content = string.Empty;

                    var client = Curl.CreateEasy(url.OriginalString, delegate(byte[] buf, int size, int nmemb, object extraData)
                    {
                        content += Encoding.UTF8.GetString(buf);
                        return(size * nmemb);
                    });

                    using (client)
                    {
                        var result = client.Perform();

                        if (result != CurlCode.Ok)
                        {
                            throw new Exception("Curl download failed with error " + result);
                        }

                        log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);

                        return(content);
                    }
                }
                catch (Exception e)
                {
                    throw new Kraken("Downloading using cURL failed", e);
                }
            }
        }
Beispiel #2
0
        public static string DownloadText(Uri url, string authToken = "")
        {
            log.DebugFormat("About to download {0}", url.OriginalString);

            try
            {
                WebClient agent = MakeDefaultHttpClient();

                // Check whether to use an auth token for this host
                if (!string.IsNullOrEmpty(authToken) ||
                    (ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) &&
                     !string.IsNullOrEmpty(authToken)))
                {
                    log.InfoFormat("Using auth token for {0}", url.Host);
                    // Send our auth token to the GitHub API (or whoever else needs one)
                    agent.Headers.Add("Authorization", $"token {authToken}");
                }

                return(agent.DownloadString(url.OriginalString));
            }
            catch (Exception e)
            {
                log.InfoFormat(e.ToString());
                log.InfoFormat("Download failed, trying with curlsharp...");

                var content = string.Empty;

                var client = Curl.CreateEasy(url.OriginalString, delegate(byte[] buf, int size, int nmemb, object extraData)
                {
                    content += Encoding.UTF8.GetString(buf);
                    return(size * nmemb);
                });

                client.SetOpt(CurlOption.FailOnError, true);

                using (client)
                {
                    var result     = client.Perform();
                    var returnCode = client.ResponseCode;

                    if (result != CurlCode.Ok)
                    {
                        throw new WebException(
                                  String.Format("Curl download failed with error {0} ({1})", result, returnCode),
                                  e
                                  );
                    }

                    log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);

                    return(content);
                }
            }
        }
Beispiel #3
0
        public static string DownloadText(string url)
        {
            Log.DebugFormat("About to download {0}", url);

            var agent = MakeDefaultHttpClient();

            try
            {
                return(agent.DownloadString(url));
            }
            catch (Exception)
            {
                try
                {
                    Log.InfoFormat("Download failed, trying with curlsharp...");

                    var content = string.Empty;

                    var client = Curl.CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData)
                    {
                        content += Encoding.UTF8.GetString(buf);
                        return(size * nmemb);
                    });

                    using (client)
                    {
                        var result = client.Perform();

                        if (result != CurlCode.Ok)
                        {
                            throw new Exception("Curl download failed with error " + result);
                        }

                        Log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);

                        return(content);
                    }
                }
                catch (Exception e)
                {
                    throw new Kraken("Downloading using cURL failed", e);
                }
            }
        }
Beispiel #4
0
        public static string Download(string url, string filename = null, IUser user = null)
        {
            user = user ?? new NullUser();
            user.RaiseMessage("Downloading {0}", url);

            // Generate a temporary file if none is provided.
            if (filename == null)
            {
                filename = FileTransaction.GetTempFileName();
            }

            Log.DebugFormat("Downloading {0} to {1}", url, filename);

            var agent = MakeDefaultHttpClient();

            try
            {
                agent.DownloadFile(url, filename);
            }
            catch (Exception ex)
            {
                Log.InfoFormat("Download failed, trying with curlsharp...");

                try
                {
                    Curl.Init();

                    using (FileStream stream = File.OpenWrite(filename))
                        using (var curl = Curl.CreateEasy(url, stream))
                        {
                            CurlCode result = curl.Perform();
                            if (result != CurlCode.Ok)
                            {
                                throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
                            }
                            else
                            {
                                Log.Debug("curlsharp download successful");
                            }
                        }

                    Curl.CleanUp();
                    return(filename);
                }
                catch
                {
                    // D'oh, failed again. Fall through to clean-up handling.
                }

                // Clean up our file, it's unlikely to be complete.
                // We do this even though we're using transactional files, as we may not be in a transaction.
                // It's okay if this fails.
                try
                {
                    Log.DebugFormat("Removing {0} after web error failure", filename);
                    FileTransaction.Delete(filename);
                }
                catch
                {
                    // Apparently we need a catch, even if we do nothing.
                }

                // Look for an exception regarding the authentication.
                if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
                {
                    throw new MissingCertificateKraken("Failed downloading " + url, ex);
                }

                // Not the exception we were looking for! Throw it further upwards!
                throw;
            }

            return(filename);
        }
Beispiel #5
0
        /// <summary>
        /// Use curlsharp to handle our downloads.
        /// </summary>
        private void DownloadCurl()
        {
            log.Debug("Curlsharp async downloader engaged");

            // Make sure our environment is set up.

            Curl.Init();

            // We'd *like* to use CurlMulti, but it just hangs when I try to retrieve
            // messages from it. So we're spawning a thread for each curleasy that does
            // the same thing. Ends up this is a little easier in handling, anyway.

            for (int i = 0; i < downloads.Count; i++)
            {
                log.DebugFormat("Downloading {0}", downloads[i].url);
                // Encode spaces to avoid confusing URL parsers
                User.RaiseMessage("Downloading \"{0}\" (libcurl)",
                                  downloads[i].url.ToString().Replace(" ", "%20"));

                // Open our file, and make an easy object...
                FileStream stream = File.OpenWrite(downloads[i].path);
                CurlEasy   easy   = Curl.CreateEasy(downloads[i].url, stream);

                // We need a separate variable for our closure, this is it.
                int index = i;

                // Curl recommends xferinfofunction, but this doesn't seem to
                // be supported by curlsharp, so we use the progress function
                // instead.
                easy.ProgressFunction = delegate(object extraData, double dlTotal, double dlNow, double ulTotal, double ulNow)
                {
                    log.DebugFormat("Progress function called... {0}/{1}", dlNow, dlTotal);

                    int percent;

                    if (dlTotal > 0)
                    {
                        percent = (int)dlNow * 100 / (int)dlTotal;
                    }
                    else
                    {
                        log.Debug("Unknown download size, skipping progress.");
                        return(0);
                    }

                    FileProgressReport(
                        index,
                        percent,
                        Convert.ToInt64(dlNow),
                        Convert.ToInt64(dlTotal)
                        );

                    // If the user has told us to cancel, then bail out now.
                    if (download_canceled)
                    {
                        log.InfoFormat("Bailing out of download {0} at user request", index);
                        // Bail out!
                        return(1);
                    }

                    // Returning 0 means we want to continue the download.
                    return(0);
                };

                // Download, little curl, fulfill your destiny!
                Thread thread = new Thread(new ThreadStart(delegate
                {
                    CurlWatchThread(index, easy, stream);
                }));

                // Keep track of our threads so we can clean them up later.
                curl_threads.Add(thread);

                // Background threads will mostly look after themselves.
                thread.IsBackground = true;

                // Let's go!
                thread.Start();
            }
        }
Beispiel #6
0
        public static string Download(string url, out string etag, string filename = null, IUser user = null)
        {
            TxFileManager FileTransaction = new TxFileManager();

            user = user ?? new NullUser();
            user.RaiseMessage("Downloading {0}", url);

            // Generate a temporary file if none is provided.
            if (filename == null)
            {
                filename = FileTransaction.GetTempFileName();
            }

            log.DebugFormat("Downloading {0} to {1}", url, filename);

            try
            {
                var agent = MakeDefaultHttpClient();
                agent.DownloadFile(url, filename);
                etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", "");
            }
            // Explicit redirect handling. This is needed when redirecting from HTTPS to HTTP on .NET Core.
            catch (WebException ex)
            {
                if (ex.Status != WebExceptionStatus.ProtocolError)
                {
                    throw;
                }

                HttpWebResponse response = ex.Response as HttpWebResponse;
                if (response?.StatusCode != HttpStatusCode.Redirect)
                {
                    throw;
                }

                return(Download(response.GetResponseHeader("Location"), out etag, filename, user));
            }
            catch (Exception e)
            {
                log.InfoFormat("Native download failed, trying with CurlSharp...");
                etag = null;

                try
                {
                    Curl.Init();

                    using (FileStream stream = File.OpenWrite(filename))
                    {
                        string header = string.Empty;

                        var client = Curl.CreateEasy(url, stream, delegate(byte[] buf, int size, int nmemb, object extraData)
                        {
                            header += Encoding.UTF8.GetString(buf);
                            return(size * nmemb);
                        });

                        using (client)
                        {
                            var result     = client.Perform();
                            var returnCode = client.ResponseCode;

                            if (result != CurlCode.Ok || returnCode >= 300)
                            {
                                // Always log if it's an error.
                                log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");

                                WebException curlException =
                                    new WebException($"Curl download failed with status {returnCode}.");
                                throw new NativeAndCurlDownloadFailedKraken(
                                          new List <Exception> {
                                    e, curlException
                                },
                                          url.ToString(), header, null, returnCode
                                          );
                            }
                            else
                            {
                                // Only log if debug flag is set.
                                log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
                            }
                        }
                    }

                    Curl.CleanUp();
                    return(filename);
                }
                catch
                {
                    // D'oh, failed again. Fall through to clean-up handling.
                }

                // Clean up our file, it's unlikely to be complete.
                // We do this even though we're using transactional files, as we may not be in a transaction.
                // It's okay if this fails.
                try
                {
                    log.DebugFormat("Removing {0} after web error failure", filename);
                    FileTransaction.Delete(filename);
                }
                catch
                {
                    // Apparently we need a catch, even if we do nothing.
                }

                // Look for an exception regarding the authentication.
                if (Regex.IsMatch(e.ToString(), "The authentication or decryption has failed."))
                {
                    throw new MissingCertificateKraken("Failed downloading " + url, e);
                }

                // Not the exception we were looking for! Throw it further upwards!
                throw;
            }

            return(filename);
        }
Beispiel #7
0
        public static string DownloadText(Uri url, string authToken = "")
        {
            log.DebugFormat("About to download {0}", url.OriginalString);

            try
            {
                WebClient agent = MakeDefaultHttpClient();

                // Check whether to use an auth token for this host
                if (!string.IsNullOrEmpty(authToken) ||
                    (ServiceLocator.Container.Resolve <IConfiguration>().TryGetAuthToken(url.Host, out authToken) &&
                     !string.IsNullOrEmpty(authToken)))
                {
                    log.InfoFormat("Using auth token for {0}", url.Host);
                    // Send our auth token to the GitHub API (or whoever else needs one)
                    agent.Headers.Add("Authorization", $"token {authToken}");
                }

                string content = agent.DownloadString(url.OriginalString);
                string header  = agent.ResponseHeaders.ToString();

                log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);

                return(content);
            }
            catch (Exception e)
            {
                log.InfoFormat("Native download failed, trying with CurlSharp...");

                string content = string.Empty;
                string header  = string.Empty;

                var client = Curl.CreateEasy(url.OriginalString,
                                             delegate(byte[] buf, int size, int nmemb, object extraData)
                {
                    content += Encoding.UTF8.GetString(buf);
                    return(size * nmemb);
                },
                                             delegate(byte[] buf, int size, int nmemb, object extraData)
                {
                    header += Encoding.UTF8.GetString(buf);
                    return(size * nmemb);
                }
                                             );

                using (client)
                {
                    var result     = client.Perform();
                    var returnCode = client.ResponseCode;

                    if (result != CurlCode.Ok || returnCode >= 300)
                    {
                        // Always log if it's an error.
                        log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);

                        WebException curlException = new WebException($"Curl download failed with status {returnCode}.");
                        throw new NativeAndCurlDownloadFailedKraken(
                                  new List <Exception> {
                            e, curlException
                        },
                                  url.ToString(), header, content, returnCode
                                  );
                    }
                    else
                    {
                        // Only log if debug flag is set
                        log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
                        return(content);
                    }
                }
            }
        }
Beispiel #8
0
        public static string Download(string url, out string etag, string filename = null, IUser user = null)
        {
            TxFileManager FileTransaction = new TxFileManager();

            user = user ?? new NullUser();
            user.RaiseMessage("Downloading {0}", url);

            // Generate a temporary file if none is provided.
            if (filename == null)
            {
                filename = FileTransaction.GetTempFileName();
            }

            log.DebugFormat("Downloading {0} to {1}", url, filename);

            try
            {
                var agent = MakeDefaultHttpClient();
                agent.DownloadFile(url, filename);
                etag = agent.ResponseHeaders.Get("ETag")?.Replace("\"", "");
            }
            // Explicit redirect handling. This is needed when redirecting from HTTPS to HTTP on .NET Core.
            catch (WebException ex)
            {
                if (ex.Status != WebExceptionStatus.ProtocolError)
                {
                    throw;
                }

                HttpWebResponse response = ex.Response as HttpWebResponse;
                if (response.StatusCode != HttpStatusCode.Redirect)
                {
                    throw;
                }

                return(Net.Download(response.GetResponseHeader("Location"), out etag, filename, user));
            }
            catch (Exception ex)
            {
                log.InfoFormat("Download failed, trying with curlsharp...");
                etag = null;

                try
                {
                    Curl.Init();

                    using (FileStream stream = File.OpenWrite(filename))
                        using (var curl = Curl.CreateEasy(url, stream))
                        {
                            CurlCode result = curl.Perform();
                            if (result != CurlCode.Ok)
                            {
                                throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
                            }
                            else
                            {
                                log.Debug("curlsharp download successful");
                            }
                        }

                    Curl.CleanUp();
                    return(filename);
                }
                catch
                {
                    // D'oh, failed again. Fall through to clean-up handling.
                }

                // Clean up our file, it's unlikely to be complete.
                // We do this even though we're using transactional files, as we may not be in a transaction.
                // It's okay if this fails.
                try
                {
                    log.DebugFormat("Removing {0} after web error failure", filename);
                    FileTransaction.Delete(filename);
                }
                catch
                {
                    // Apparently we need a catch, even if we do nothing.
                }

                // Look for an exception regarding the authentication.
                if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
                {
                    throw new MissingCertificateKraken("Failed downloading " + url, ex);
                }

                // Not the exception we were looking for! Throw it further upwards!
                throw;
            }

            return(filename);
        }