//int currFileRetry = 0; private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e) { BackgroundWorker worker = sender as BackgroundWorker; object[] param = (object[])e.Argument; Stream ms = (Stream)param[0]; DanbooruPostDaoOption option = (DanbooruPostDaoOption)param[1]; DanbooruPostDao newPosts = new DanbooruPostDao(ms, option); e.Result = newPosts; }
private void clientList_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e) { try { Program.Logger.Debug("Download list completed"); tsProgressBar.Visible = false; MemoryStream ms = new MemoryStream(e.Result); var option = new DanbooruPostDaoOption() { Provider = _currProvider, //Url = txtListFile.Text, Referer = _clientList.Referer, Query = txtQuery.Text, SearchTags = txtTags.Text, BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked }; tsStatus.Text = "Loading downloaded list..."; tsProgressBar.Style = ProgressBarStyle.Marquee; tsProgressBar.Visible = true; _isLoadingList = true; backgroundWorker1 = new BackgroundWorker(); backgroundWorker1.DoWork += new DoWorkEventHandler(backgroundWorker1_DoWork); backgroundWorker1.RunWorkerCompleted += new RunWorkerCompletedEventHandler(backgroundWorker1_RunWorkerCompleted); backgroundWorker1.RunWorkerAsync(new object[] { ms, option }); } catch (Exception ex) { var message = ex.Message; if (ex.InnerException != null) { message = ex.InnerException.Message; var wex = ex.InnerException as System.Net.WebException; if (wex != null && wex.Status == WebExceptionStatus.ProtocolError) { var status = wex.Response != null ? wex.Response.Headers["Status"] : null; var response = wex.Response as HttpWebResponse; if (status == "403" || (response != null && response.StatusCode == HttpStatusCode.Forbidden)) { message += Environment.NewLine + "Please check your login information."; } else { using (var responseStream = wex.Response.GetResponseStream()) { if (responseStream != null) { var option = new DanbooruPostDaoOption() { Provider = _currProvider, //Url = "", Referer = "", Query = "", SearchTags = "", BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked }; var resp = new DanbooruPostDao(responseStream, option); message = "Server Message: " + resp.ResponseMessage; if (status != "200") { message += "\nStatus Code: " + wex.Status.ToString() + " (" + status + ")"; } } } } } } MessageBox.Show(message, "Download List"); Program.Logger.Error(message, ex); UpdateLog("clientList_DownloadDataCompleted", "Error: " + (ex.InnerException == null ? ex.Message : ex.InnerException.Message), ex); UpdateLog("clientList_DownloadDataCompleted", "Referer: " + _clientList.Referer); chkAutoLoadNext.Checked = false; btnGet.Enabled = true; btnListCancel.Enabled = false; _isLoadingList = false; } }
private void clientList_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) { tsProgressBar.Visible = false; UpdateLog("clientList_DownloadFileCompleted", "Download Complete: " + e.UserState); txtListFile.Text = saveFileDialog1.FileName; gbxSearch.Enabled = true; gbxList.Enabled = true; btnGet.Enabled = true; btnListCancel.Enabled = false; if (chkAutoLoadList.Checked) { var option = new DanbooruPostDaoOption() { Provider = _currProvider, Url = txtListFile.Text, Referer = _clientList.Referer, Query = txtListFile.Text.Split('\\').Last(), SearchTags = "", BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked }; DanbooruPostDao newPosts = new DanbooruPostDao(option); LoadList(newPosts); } _isLoadingList = false; }
public void DoBatchJob(BindingList<DanbooruBatchJob> batchJob) { ToggleBatchJobButtonDelegate bjd = new ToggleBatchJobButtonDelegate(ToggleBatchJobButton); UpdateUiDelegate del = new UpdateUiDelegate(UpdateUi); UpdateUiDelegate2 del2 = new UpdateUiDelegate2(UpdateUi); ExtendedWebClient _clientPost = new ExtendedWebClient(); if (batchJob != null) { UpdateStatus2("Starting Batch Job"); for (int i = 0; i < batchJob.Count; i++) { batchJob[i].CurrentPage = 0; if (!batchJob[i].isCompleted) { UpdateLog("DoBatchJob", "Processing Batch Job#" + i); DanbooruPostDao prevDao = null; bool flag = true; int currPage = 0; int postCount = 0; do { // stop/pause event handling outside _pauseEvent.WaitOne(Timeout.Infinite); if (_shutdownEvent.WaitOne(0)) { batchJob[i].Status = " ==> Stopped."; // toggle button BeginInvoke(bjd, new object[] { true }); UpdateLog("DoBatchJob", "Batch Job Stopped."); UpdateStatus2("Batch Job Stopped."); return; } DanbooruPostDao d = null; int imgCount = 0; int skipCount = 0; string url; string query = ""; #region Construct the searchParam if (batchJob[i].Provider.BoardType == BoardType.Danbooru || batchJob[i].Provider.BoardType == BoardType.Shimmie2) { currPage = batchJob[i].CurrentPage; } else if (batchJob[i].Provider.BoardType == BoardType.Gelbooru) { if (batchJob[i].Provider.Preferred == PreferredMethod.Html) { currPage = batchJob[i].CurrentPage * postCount; } else { currPage = batchJob[i].CurrentPage; } } DanbooruSearchParam searchParam = GetSearchParamsFromJob(batchJob[i], currPage); url = batchJob[i].Provider.GetQueryUrl(searchParam); #endregion Construct the searchParam try { #region Get and load the image list batchJob[i].Status = "Getting list for page: " + searchParam.Page; BeginInvoke(del); UpdateLog("DoBatchJob", "Downloading list: " + url); d = GetBatchImageList(url, query, batchJob[i]); if (d == null) { // Cannot get list. UpdateLog("DoBatchJob", "Cannot load list"); batchJob[i].Status = "Cannot load list."; batchJob[i].isCompleted = false; batchJob[i].isError = true; flag = false; } else if (d.Posts == null || d.Posts.Count == 0) { // No more image UpdateLog("DoBatchJob", "No more image."); batchJob[i].Status = "No more image."; flag = false; //break; } else { if (prevDao != null) { // identical data returned, probably no more new image. if (prevDao.RawData != null && prevDao.RawData.Equals(d.RawData)) { UpdateLog("DoBatchJob", "Identical list, probably last page."); batchJob[i].Status = "Identical list, probably last page."; flag = false; //break; } } prevDao = d; batchJob[i].Total = d.PostCount; batchJob[i].CurrentPageTotal = d.Posts.Count; batchJob[i].CurrentPageOffset = d.Offset; #endregion Get and load the image list postCount = d.Posts.Count; foreach (DanbooruPost post in d.Posts) { // Update progress bar object[] myArray = new object[2]; myArray[0] = batchJob[i].ProcessedTotal; myArray[1] = d.PostCount < batchJob[i].Limit ? d.PostCount : batchJob[i].Limit; BeginInvoke(del2, myArray); // thread handling _pauseEvent.WaitOne(Timeout.Infinite); if (_shutdownEvent.WaitOne(0)) { batchJob[i].Status = " ==> Stopped."; // toggle button BeginInvoke(bjd, new object[] { true }); UpdateLog("DoBatchJob", "Batch Job Stopped."); UpdateStatus2("Batch Job Stopped."); return; } // check if have url and post is not deleted if (string.IsNullOrWhiteSpace(post.FileUrl) && (post.Status != "deleted" || chkProcessDeletedPost.Checked)) { ResolveFileUrlBatch(_clientPost, post); } //Choose the correct urls var targetUrl = post.FileUrl; if (_ImageSize == "Thumb" && !String.IsNullOrWhiteSpace(post.PreviewUrl)) { targetUrl = post.PreviewUrl; } else if (_ImageSize == "Jpeg" && !String.IsNullOrWhiteSpace(post.JpegUrl)) { targetUrl = post.JpegUrl; } else if (_ImageSize == "Sample" && !String.IsNullOrWhiteSpace(post.SampleUrl)) { targetUrl = post.SampleUrl; } batchJob[i].Status = "Downloading: " + targetUrl; BeginInvoke(del); //if (post.Provider == null) post.Provider = cbxProvider.Text; //if (post.Query == null) post.Query = txtQuery.Text; //if (post.SearchTags == null) post.SearchTags = txtTags.Text; bool download = true; // check if blacklisted if (download && post.Hidden) { ++skipCount; ++batchJob[i].Skipped; download = false; UpdateLog("DoBatchJob", "Download skipped, contains blacklisted tag: " + post.Tags + " Url: " + targetUrl); } string filename = ""; if (download && !string.IsNullOrWhiteSpace(targetUrl)) { var format = new DanbooruFilenameFormat() { FilenameFormat = batchJob[i].SaveFolder, Limit = Convert.ToInt32(txtFilenameLength.Text), BaseFolder = txtSaveFolder.Text, MissingTagReplacement = txtTagReplacement.Text, ArtistGroupLimit = Convert.ToInt32(txtArtistTagGrouping.Text), CharacterGroupLimit = Convert.ToInt32(txtCharaTagGrouping.Text), CopyrightGroupLimit = Convert.ToInt32(txtCopyTagGrouping.Text), CircleGroupLimit = Convert.ToInt32(txtCircleTagGrouping.Text), FaultsGroupLimit = Convert.ToInt32(txtFaultsTagGrouping.Text), IgnoredTags = DanbooruTagsDao.Instance.ParseTagsString(txtIgnoredTags.Text.Replace(Environment.NewLine, " ")), IgnoredTagsRegex = txtIgnoredTags.Text.Trim().Replace(Environment.NewLine, "|"), IgnoreTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsReplaceMode = chkReplaceMode.Checked, IgnoredTagsOnlyForGeneral = chkIgnoreForGeneralTag.Checked, TagReplaceUnderscoreToSpace = chkIsReplaceUnderscoreTag.Checked }; string extension = Helper.getFileExtensions(targetUrl); filename = Helper.MakeFilename(format, post) + extension; } // check if exist if (download && !chkOverwrite.Checked) { if (File.Exists(filename)) { ++skipCount; ++batchJob[i].Skipped; download = false; UpdateLog("DoBatchJob", "Download skipped, file exists: " + filename); } } if (download && String.IsNullOrWhiteSpace(targetUrl)) { ++skipCount; ++batchJob[i].Skipped; download = false; UpdateLog("DoBatchJob", "Download skipped, ID: " + post.Id + " No file_url, probably deleted"); } Uri uri = null; if (download && !Uri.TryCreate(targetUrl, UriKind.RelativeOrAbsolute, out uri)) { ++skipCount; ++batchJob[i].Skipped; download = false; UpdateLog("DoBatchJob", "Download skipped, ID: " + post.Id + " Invalid URL: " + targetUrl); } #region download if (download) { imgCount = DoDownloadBatch(targetUrl, batchJob[i], post, filename); } #endregion download // check if more than available post if (batchJob[i].ProcessedTotal >= d.PostCount && d.PostCount != 0) { UpdateLog("DoBatchJob", "No more post."); flag = false; break; } // check if over given limit if (batchJob[i].ProcessedTotal >= batchJob[i].Limit) { UpdateLog("DoBatchJob", "Limit Reached."); flag = false; break; } // check batch job delay int delay = 0; Int32.TryParse(Properties.Settings.Default.BatchJobDelay, out delay); if ((Properties.Settings.Default.DelayIncludeSkipped || download) && delay > 0) { UpdateLog("DoBatchJob", String.Format("Waiting for {0}ms for the next post.", delay)); Thread.Sleep(delay); } } } batchJob[i].Status = " ==> Done."; } catch (Exception ex) { string message = ex.Message; string responseMessage = ""; if (ex.InnerException != null) { message += Environment.NewLine + "Inner: " + ex.InnerException.Message; } message += Environment.NewLine + "Stack Trace: " + Environment.NewLine + ex.StackTrace; message += Environment.NewLine + "Query: " + batchJob[i].TagQuery; batchJob[i].isError = true; batchJob[i].isCompleted = false; if (ex.GetType() == typeof(System.Net.WebException)) { System.Net.WebException wex = (System.Net.WebException)ex; if (wex.Status == WebExceptionStatus.ProtocolError && wex.Response.Headers.AllKeys.Contains("Status") && wex.Response.Headers["Status"].ToString() == "500") { using (var response = wex.Response.GetResponseStream()) { if (response != null) { var option = new DanbooruPostDaoOption() { Provider = _currProvider, Query = query, SearchTags = batchJob[i].TagQuery, Url = url, Referer = "", BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked }; var resp = new DanbooruPostDao(response, option); responseMessage = resp.ResponseMessage; flag = false; } } } } if (ex.Message.Contains("(400)") || ex.Message.Contains("(403)") || ex.Message.Contains("(500)") || ex.Message.Contains("resolved")) { flag = false; } batchJob[i].Status = " ==> Error: " + (string.IsNullOrWhiteSpace(responseMessage) ? ex.Message : responseMessage) + Environment.NewLine; if (!string.IsNullOrWhiteSpace(responseMessage)) UpdateLog("DoBatchJob", "Server Message: " + responseMessage, ex); else UpdateLog("DoBatchJob", "Error: " + message, ex); if (cbxAbortOnError.Checked) { MessageBox.Show(message, "Batch Download"); break; } } finally { BeginInvoke(del); { // Update progress bar object[] myArray = new object[2]; myArray[0] = batchJob[i].ProcessedTotal; if (d != null) { myArray[1] = d.PostCount < batchJob[i].Limit ? d.PostCount : batchJob[i].Limit; } else { myArray[1] = -1; } BeginInvoke(del2, myArray); } } ++batchJob[i].CurrentPage; } while (flag); UpdateLog("DoBatchJob", "Batch Job #" + i + ": Done"); if (batchJob[i].isError) { batchJob[i].isCompleted = false; } else { batchJob[i].isCompleted = true; } BeginInvoke(del); } } } BeginInvoke(bjd, new object[] { true }); UpdateStatus2("Batch Job Completed!", true); { // hide progress bar object[] myArray = new object[2]; myArray[0] = -1; myArray[1] = -1; BeginInvoke(del2, myArray); } }
/// <summary> /// Load post from given DanbooruPostDao to _postsDao /// if chkAppendList, the post will be appended to current _postsDao /// </summary> /// <param name="postDao"></param> private void LoadList(DanbooruPostDao postDao) { Program.Logger.Debug("Loading list"); if (postDao.Posts.Count > 0) { _isLoadingList = true; if (_postsDao != null) { if (chkAppendList.Checked || chkAutoLoadNext.Checked) { UpdateLog("[LoadList]", "Appending: " + postDao.Option.SearchTags + " Offset: " + postDao.Offset); int oldCount = _postsDao.Posts.Count; foreach (DanbooruPost po in postDao.Posts) { _postsDao.Posts.Add(po); } dgvList.DataSource = _postsDao.Posts; if (chkLoadPreview.Checked && !_clientThumb.IsBusy && !_isLoadingThumb) LoadThumbnailLater(oldCount); if (chkAutoLoadNext.Checked) { tsCount.Text = "| Count = " + (oldCount + _postsDao.Posts.Count); tsTotalCount.Text = "| Total Count = " + _postsDao.PostCount; _currCount = postDao.Posts.Count; // only the new post } } else { _postsDao = postDao; dgvList.DataSource = _postsDao.Posts; if (chkLoadPreview.Checked && !_clientThumb.IsBusy && !_isLoadingThumb) LoadThumbnailLater(0); } } else { _postsDao = postDao; dgvList.DataSource = _postsDao.Posts; if (chkLoadPreview.Checked && !_clientThumb.IsBusy && !_isLoadingThumb) LoadThumbnailLater(0); } tsCount.Text = "| Count = " + _postsDao.Posts.Count; tsTotalCount.Text = "| Total Count = " + postDao.PostCount; _currCount = postDao.Posts.Count; _isLoadingList = false; } else { _isMorePost = false; _isLoadingList = false; ShowMessage("Main", "No Posts!"); } }
private void btnClear_Click(object sender, EventArgs e) { _loadedThumbnail = 0; _postsDao = null; dgvList.Rows.Clear(); }
private void btnList_Click(object sender, EventArgs e) { if (txtListFile.Text.Length > 0) { var option = new DanbooruPostDaoOption() { Provider = _currProvider, Url = txtListFile.Text, Referer = _currProvider.Url, Query = txtListFile.Text.Split('\\').Last(), SearchTags = "", BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked }; DanbooruPostDao newPosts = new DanbooruPostDao(option); LoadList(newPosts); } }
/// <summary> /// Get image list, return null if failed /// </summary> /// <param name="url"></param> /// <param name="searchParam"></param> /// <param name="job"></param> /// <returns></returns> private DanbooruPostDao GetBatchImageList(String url, String query, DanbooruBatchJob job) { DanbooruPostDao d = null; int currRetry = 0; int maxRetry = Convert.ToInt32(txtRetry.Text); int delay = Convert.ToInt32(txtDelay.Text); while (currRetry < maxRetry) { try { var strs = _clientBatch.DownloadString(url); using (MemoryStream ms = new MemoryStream(_clientBatch.DownloadData(url))) { var option = new DanbooruPostDaoOption() { Provider = job.Provider, Query = query, SearchTags = job.TagQuery, Referer = url, BlacklistedTags = TagBlacklist, BlacklistedTagsRegex = TagBlacklistRegex, BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked, IgnoredTags = TagIgnore, IgnoredTagsRegex = TagIgnoreRegex, IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked }; d = new DanbooruPostDao(ms, option); } break; } catch (System.Net.WebException ex) { ++currRetry; UpdateLog("DoBatchJob", "Error Getting List (" + currRetry + " of " + maxRetry + "): " + ex.Message + " Wait for " + delay + "s.", ex); for (int wait = 0; wait < delay; ++wait) { //UpdateLog("DoBatchJob", "Wait for " + wait + " of " + delay); Thread.Sleep(1000); } UpdateLog("DoBatchJob", "Retrying..."); } } return d; }