/// <summary> /// Download image /// </summary> /// <param name="url"></param> /// <param name="referer"></param> /// <param name="filename"></param> /// <param name="progressChanged"></param> /// <returns></returns> public string Download(string url, string referer, string filename, Action<string> progressChanged, CancellationToken cancelToken) { String message = ""; checkOverwrite(filename, progressChanged, ref message); ExtendedWebClient client = new ExtendedWebClient(); client.Referer = referer; var tempFilename = DeleteTempFile(filename, progressChanged); Util.CreateSubDir(tempFilename); int retry = 1; while (retry <= Properties.Settings.Default.RetryCount) { try { using (var stream = client.OpenRead(url)) { Int64 bytes_total = downloadPreCheck(filename, client, progressChanged, ref message); using (var f = File.Create(tempFilename)) { stream.CopyTo(f); } downloadPostCheck(filename, tempFilename, bytes_total, progressChanged, ref message); } break; } catch (NijieException) { throw; } catch (Exception ex) { checkHttpStatusCode(url, ex); Log.Warn(string.Format("Error when downloading: {0} to {1} ==> {2}, Retrying {2} of {3}...", url, tempFilename, ex.Message, retry, Properties.Settings.Default.RetryCount)); DeleteTempFile(filename, progressChanged); var prefixMsg = message.Clone(); for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i) { message = String.Format("{0} waiting: {1}", prefixMsg, i); Thread.Sleep(1000); if (cancelToken != null && cancelToken.IsCancellationRequested) { throw new NijieException(string.Format("Cancel requested, error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_CANCELLED); } } ++retry; if (retry > Properties.Settings.Default.RetryCount) throw new NijieException(string.Format("Error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR); } } Thread.Sleep(100); // delay before renaming File.Move(tempFilename, filename); message = "Saved to: " + filename; if (progressChanged != null) progressChanged(message); return message; }
private Tuple<HtmlDocument, WebResponse> getPage(string url) { Tuple<HtmlDocument, WebResponse> result = null; ExtendedWebClient client = new ExtendedWebClient(); int retry = 1; while (retry <= Properties.Settings.Default.RetryCount) { try { var imagePage = client.DownloadData(url); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(Encoding.UTF8.GetString(imagePage)); result = new Tuple<HtmlDocument, WebResponse>(doc, client.Response); break; } catch (Exception ex) { checkHttpStatusCode(url, ex); ++retry; if (retry > Properties.Settings.Default.RetryCount) { throw; } } } return result; }
/// <summary> /// Check if File exists and need to redownload. /// </summary> /// <param name="filename"></param> /// <param name="client"></param> /// <param name="progressChanged"></param> /// <param name="message"></param> /// <returns>Server File Size via "Content-Length" header.</returns> private Int64 downloadPreCheck(string filename, ExtendedWebClient client, Action<string> progressChanged, ref string message) { Int64 bytes_total = -1; // if compression enabled, the content-length is the compressed size. // so need to check after download to get the real size. if (File.Exists(filename) && !ExtendedWebClient.EnableCompression) { FileInfo oldFileInfo = new FileInfo(filename); if (client.ResponseHeaders["Content-Length"] != null) bytes_total = Convert.ToInt64(client.ResponseHeaders["Content-Length"]); Log.Debug("Content-Length Filesize: " + bytes_total); // If have Content-length size, do pre-check. if (bytes_total > 0) { // skip download if the filesize are the same. if (oldFileInfo.Length == bytes_total && Properties.Settings.Default.OverwriteOnlyIfDifferentSize) { message += ", Identical size: " + bytes_total + ", skipping..."; if (progressChanged != null) progressChanged(message); throw new NijieException(message, NijieException.DOWNLOAD_SKIPPED); } // make backup for the old file if (Properties.Settings.Default.MakeBackup) { var backupFilename = filename + "." + Util.DateTimeToUnixTimestamp(DateTime.Now); message += ", different size: " + oldFileInfo.Length + " vs " + bytes_total + ", backing up to: " + backupFilename; Log.Info(message); if (progressChanged != null) progressChanged(message); File.Move(filename, backupFilename); } else { File.Delete(filename); } } } return bytes_total; }
/// <summary> /// Download data to memory byte array. /// </summary> /// <param name="url"></param> /// <param name="referer"></param> /// <returns>downloaded data.</returns> public byte[] DownloadData(string url, string referer) { int retry = 1; byte[] result = null; while (retry <= Properties.Settings.Default.RetryCount) { try { ExtendedWebClient client = new ExtendedWebClient(); client.Referer = referer; result = client.DownloadData(url); break; } catch (Exception ex) { checkHttpStatusCode(url, ex); Log.Warn(String.Format("Error when downloading data: {0} ==> {1}, Retrying {2} of {3}...", url, ex.Message, retry, Properties.Settings.Default.RetryCount)); //for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i) //{ // Thread.Sleep(1000); //} ++retry; if (retry > Properties.Settings.Default.RetryCount) throw new NijieException(String.Format("Error when downloading data: {0} ==> {1}", url, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR); } } return result; }
private bool DoLogin(NijieLoginInfo info) { IsLoggedIn = false; ExtendedWebClient client = new ExtendedWebClient(); NameValueCollection loginInfo = new NameValueCollection(); loginInfo.Add("email", info.UserName); loginInfo.Add("password", info.Password); if (info.RememberLogin) loginInfo.Add("save", "on"); loginInfo.Add("ticket", info.Ticket); loginInfo.Add("url", info.ReturnUrl); var result = client.UploadValues(Util.FixUrl(NijieConstants.NIJIE_LOGIN_URL2, ROOT_DOMAIN, Properties.Settings.Default.UseHttps), "POST", loginInfo); //String data = Encoding.UTF8.GetString(result); var location = client.Response.ResponseUri.ToString(); if (!String.IsNullOrWhiteSpace(location)) { if (location.Contains(@"//nijie.info/login.php?")) IsLoggedIn = false; else IsLoggedIn = true; } var uri = new Uri(Util.FixUrl("//nijie.info", ROOT_DOMAIN, Properties.Settings.Default.UseHttps)); ExtendedWebClient.CookieJar.Add(uri, new Cookie("R18", "1")); var cookies = ExtendedWebClient.CookieJar.GetCookies(uri); foreach (Cookie item in cookies) { //Cookie: NIJIEIJIEID=lp1ffmjc9gi7a3u9qkj8p566u3 if (item.Name == "NIJIEIJIEID") { NijieSessionID = item.Value; item.Expires = DateTime.MaxValue; break; } } PrintCookie("Login:"); return IsLoggedIn; }
private NijieLoginInfo PrepareLoginInfo(string userName, string password) { ExtendedWebClient client = new ExtendedWebClient(); // not really used var uri = new Uri(Util.FixUrl("//nijie.info", ROOT_DOMAIN, Properties.Settings.Default.UseHttps)); var tick = ((int)Util.DateTimeToUnixTimestamp(DateTime.Now)).ToString(); ExtendedWebClient.CookieJar.Add(uri, new Cookie("nijie_token_secret", tick)); ExtendedWebClient.CookieJar.Add(uri, new Cookie("nijie_token", tick)); NijieLoginInfo info = new NijieLoginInfo() { UserName = userName, Password = password, ReturnUrl = "", Ticket = "", RememberLogin = true }; HtmlDocument doc = getPage(Util.FixUrl(NijieConstants.NIJIE_LOGIN_URL, ROOT_DOMAIN, Properties.Settings.Default.UseHttps)).Item1; var tickets = doc.DocumentNode.SelectNodes("//input[@name='ticket']"); if (tickets != null && tickets.Count > 0) info.Ticket = tickets[0].Attributes["value"].Value; var returnUrls = doc.DocumentNode.SelectNodes("//input[@name='url']"); if (returnUrls != null && returnUrls.Count > 0) info.ReturnUrl = returnUrls[0].Attributes["value"].Value; PrintCookie("Prepare Login:"); return info; }