Inheritance: System.Net.WebClient
Beispiel #1
0
        /// <summary>
        /// Download image
        /// </summary>
        /// <param name="url"></param>
        /// <param name="referer"></param>
        /// <param name="filename"></param>
        /// <param name="progressChanged"></param>
        /// <returns></returns>
        public string Download(string url, string referer, string filename, Action<string> progressChanged, CancellationToken cancelToken)
        {
            String message = "";

            checkOverwrite(filename, progressChanged, ref message);

            ExtendedWebClient client = new ExtendedWebClient();
            client.Referer = referer;

            var tempFilename = DeleteTempFile(filename, progressChanged);
            Util.CreateSubDir(tempFilename);
            int retry = 1;
            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    using (var stream = client.OpenRead(url))
                    {
                        Int64 bytes_total = downloadPreCheck(filename, client, progressChanged, ref message);

                        using (var f = File.Create(tempFilename))
                        {
                            stream.CopyTo(f);
                        }

                        downloadPostCheck(filename, tempFilename, bytes_total, progressChanged, ref message);
                    }
                    break;
                }
                catch (NijieException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    Log.Warn(string.Format("Error when downloading: {0} to {1} ==> {2}, Retrying {2} of {3}...", url, tempFilename, ex.Message, retry, Properties.Settings.Default.RetryCount));
                    DeleteTempFile(filename, progressChanged);

                    var prefixMsg = message.Clone();
                    for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
                    {
                        message = String.Format("{0} waiting: {1}", prefixMsg, i);
                        Thread.Sleep(1000);
                        if (cancelToken != null && cancelToken.IsCancellationRequested)
                        {
                            throw new NijieException(string.Format("Cancel requested, error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_CANCELLED);
                        }
                    }
                    ++retry;

                    if (retry > Properties.Settings.Default.RetryCount)
                        throw new NijieException(string.Format("Error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
                }
            }

            Thread.Sleep(100); // delay before renaming
            File.Move(tempFilename, filename);
            message = "Saved to: " + filename;
            if (progressChanged != null)
                progressChanged(message);
            return message;
        }
Beispiel #2
0
        private Tuple<HtmlDocument, WebResponse> getPage(string url)
        {
            Tuple<HtmlDocument, WebResponse> result = null;
            ExtendedWebClient client = new ExtendedWebClient();
            int retry = 1;
            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    var imagePage = client.DownloadData(url);
                    HtmlDocument doc = new HtmlDocument();
                    doc.LoadHtml(Encoding.UTF8.GetString(imagePage));
                    result = new Tuple<HtmlDocument, WebResponse>(doc, client.Response);
                    break;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    ++retry;
                    if (retry > Properties.Settings.Default.RetryCount)
                    {
                        throw;
                    }
                }
            }
            return result;
        }
Beispiel #3
0
        /// <summary>
        /// Check if File exists and need to redownload.
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="client"></param>
        /// <param name="progressChanged"></param>
        /// <param name="message"></param>
        /// <returns>Server File Size via "Content-Length" header.</returns>
        private Int64 downloadPreCheck(string filename, ExtendedWebClient client, Action<string> progressChanged, ref string message)
        {
            Int64 bytes_total = -1;
            // if compression enabled, the content-length is the compressed size.
            // so need to check after download to get the real size.
            if (File.Exists(filename) && !ExtendedWebClient.EnableCompression)
            {
                FileInfo oldFileInfo = new FileInfo(filename);
                if (client.ResponseHeaders["Content-Length"] != null)
                    bytes_total = Convert.ToInt64(client.ResponseHeaders["Content-Length"]);

                Log.Debug("Content-Length Filesize: " + bytes_total);

                // If have Content-length size, do pre-check.
                if (bytes_total > 0)
                {
                    // skip download if the filesize are the same.
                    if (oldFileInfo.Length == bytes_total && Properties.Settings.Default.OverwriteOnlyIfDifferentSize)
                    {
                        message += ", Identical size: " + bytes_total + ", skipping...";
                        if (progressChanged != null)
                            progressChanged(message);
                        throw new NijieException(message, NijieException.DOWNLOAD_SKIPPED);
                    }

                    // make backup for the old file
                    if (Properties.Settings.Default.MakeBackup)
                    {
                        var backupFilename = filename + "." + Util.DateTimeToUnixTimestamp(DateTime.Now);
                        message += ", different size: " + oldFileInfo.Length + " vs " + bytes_total + ", backing up to: " + backupFilename;
                        Log.Info(message);
                        if (progressChanged != null)
                            progressChanged(message);
                        File.Move(filename, backupFilename);
                    }
                    else
                    {
                        File.Delete(filename);
                    }
                }
            }
            return bytes_total;
        }
Beispiel #4
0
        /// <summary>
        /// Download data to memory byte array.
        /// </summary>
        /// <param name="url"></param>
        /// <param name="referer"></param>
        /// <returns>downloaded data.</returns>
        public byte[] DownloadData(string url, string referer)
        {
            int retry = 1;
            byte[] result = null;

            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    ExtendedWebClient client = new ExtendedWebClient();
                    client.Referer = referer;

                    result = client.DownloadData(url);
                    break;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    Log.Warn(String.Format("Error when downloading data: {0} ==> {1}, Retrying {2} of {3}...", url, ex.Message, retry, Properties.Settings.Default.RetryCount));

                    //for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
                    //{
                    //    Thread.Sleep(1000);
                    //}

                    ++retry;
                    if (retry > Properties.Settings.Default.RetryCount)
                        throw new NijieException(String.Format("Error when downloading data: {0} ==> {1}", url, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
                }
            }
            return result;
        }
Beispiel #5
0
        private bool DoLogin(NijieLoginInfo info)
        {
            IsLoggedIn = false;
            ExtendedWebClient client = new ExtendedWebClient();
            NameValueCollection loginInfo = new NameValueCollection();
            loginInfo.Add("email", info.UserName);
            loginInfo.Add("password", info.Password);
            if (info.RememberLogin)
                loginInfo.Add("save", "on");
            loginInfo.Add("ticket", info.Ticket);
            loginInfo.Add("url", info.ReturnUrl);

            var result = client.UploadValues(Util.FixUrl(NijieConstants.NIJIE_LOGIN_URL2, ROOT_DOMAIN, Properties.Settings.Default.UseHttps), "POST", loginInfo);
            //String data = Encoding.UTF8.GetString(result);

            var location = client.Response.ResponseUri.ToString();
            if (!String.IsNullOrWhiteSpace(location))
            {
                if (location.Contains(@"//nijie.info/login.php?"))
                    IsLoggedIn = false;
                else
                    IsLoggedIn = true;
            }

            var uri = new Uri(Util.FixUrl("//nijie.info", ROOT_DOMAIN, Properties.Settings.Default.UseHttps));
            ExtendedWebClient.CookieJar.Add(uri, new Cookie("R18", "1"));
            var cookies = ExtendedWebClient.CookieJar.GetCookies(uri);
            foreach (Cookie item in cookies)
            {
                //Cookie: NIJIEIJIEID=lp1ffmjc9gi7a3u9qkj8p566u3
                if (item.Name == "NIJIEIJIEID")
                {
                    NijieSessionID = item.Value;
                    item.Expires = DateTime.MaxValue;
                    break;
                }
            }

            PrintCookie("Login:");

            return IsLoggedIn;
        }
Beispiel #6
0
        private NijieLoginInfo PrepareLoginInfo(string userName, string password)
        {
            ExtendedWebClient client = new ExtendedWebClient();

            // not really used
            var uri = new Uri(Util.FixUrl("//nijie.info", ROOT_DOMAIN, Properties.Settings.Default.UseHttps));
            var tick = ((int)Util.DateTimeToUnixTimestamp(DateTime.Now)).ToString();
            ExtendedWebClient.CookieJar.Add(uri, new Cookie("nijie_token_secret", tick));
            ExtendedWebClient.CookieJar.Add(uri, new Cookie("nijie_token", tick));

            NijieLoginInfo info = new NijieLoginInfo() { UserName = userName, Password = password, ReturnUrl = "", Ticket = "", RememberLogin = true };

            HtmlDocument doc = getPage(Util.FixUrl(NijieConstants.NIJIE_LOGIN_URL, ROOT_DOMAIN, Properties.Settings.Default.UseHttps)).Item1;

            var tickets = doc.DocumentNode.SelectNodes("//input[@name='ticket']");
            if (tickets != null && tickets.Count > 0)
                info.Ticket = tickets[0].Attributes["value"].Value;

            var returnUrls = doc.DocumentNode.SelectNodes("//input[@name='url']");
            if (returnUrls != null && returnUrls.Count > 0)
                info.ReturnUrl = returnUrls[0].Attributes["value"].Value;

            PrintCookie("Prepare Login:");

            return info;
        }