public AddJob(JobDownloadViewModel job, String messageCount) { this.NewJob = job; InitializeComponent(); Buttons = new List<Button>(); Buttons.Add(btnJobOk); Buttons.Add(btnJobCancel); this.DataContext = NewJob; txtCountMessage.DataContext = messageCount; }
private void processManga(JobDownloadViewModel job, NijieImage image) { var downloaded = -1; MainWindow.Log.Debug("Processing Manga Images:" + image.ImageId); string lastFilename = "", lastUrl = ""; for (int i = 0; i < image.ImageUrls.Count; ++i) { downloaded = -1; if (isJobCancelled(job)) return; job.PauseEvent.WaitOne(Timeout.Infinite); var filename = MainWindow.makeFilename(job, image, i, MainWindow.FilenameFormatType.Manga); job.Message = "Downloading: " + image.ImageUrls[i]; var pagefilename = filename; if (!(job.SaveFilenameFormat.Contains(MainWindow.FILENAME_FORMAT_PAGE) || job.SaveFilenameFormat.Contains(MainWindow.FILENAME_FORMAT_PAGE_ZERO))) { pagefilename += "_p" + i; } pagefilename += "." + Util.ParseExtension(image.ImageUrls[i]); pagefilename = Properties.Settings.Default.RootDirectory + Path.DirectorySeparatorChar + Util.SanitizeFilename(pagefilename); var pages = image.MangaPages as List<NijieMangaInfo>; downloaded = downloadUrl(job, image.ImageUrls[i], image.Referer, pagefilename); pages[i].SavedFilename = pagefilename; pages[i].ServerFilename = Util.ExtractFilenameFromUrl(image.ImageUrls[i]); if (downloaded == NijieException.OK) { pages[i].Filesize = new FileInfo(pagefilename).Length; if (Properties.Settings.Default.DumpDownloadedImagesToTextFile) Util.WriteTextFile(pagefilename + Environment.NewLine); } lastFilename = pagefilename; lastUrl = image.ImageUrls[i]; } image.SavedFilename = lastFilename; image.ServerFilename = Util.ExtractFilenameFromUrl(lastUrl); if (downloaded == NijieException.OK) { image.Filesize = new FileInfo(lastFilename).Length; if (Properties.Settings.Default.SaveDB) SaveImageToDB(job, image); job.DownloadCount++; } else if (downloaded == NijieException.DOWNLOAD_SKIPPED) { image.Filesize = new FileInfo(lastFilename).Length; if (Properties.Settings.Default.SaveDB) SaveImageToDB(job, image); job.SkipCount++; } }
/// <summary> /// Parse the image page /// - Illustration /// - Manga /// </summary> /// <param name="job"></param> /// <param name="memberPage"></param> /// <param name="imageTemp"></param> private void processImage(JobDownloadViewModel job, NijieMember memberPage, NijieImage imageTemp) { if (isJobCancelled(job)) return; MainWindow.Log.Debug("Processing Image:" + imageTemp.ImageId); // skip if exists in DB if (Properties.Settings.Default.SkipIfExistsInDB && !NijieDownloader.Library.Properties.Settings.Default.Overwrite) { using (var dao = new NijieContext()) { var result = NijieImage.IsDownloadedInDB(imageTemp.ImageId); if (result) { job.Message = String.Format("Image {0} already downloaded in DB", imageTemp.ImageId); job.SkipCount++; return; } } } var image = MainWindow.Bot.ParseImage(imageTemp, memberPage); if (image.IsFriendOnly) { // sample: 74587 job.Message = "Image locked!"; return; } if (image.IsGoldenMember) { job.Message = "Image only for Gold Membership"; return; } if (image.IsManga) { processManga(job, image); } else { processIllustration(job, image); } }
private void processIllustration(JobDownloadViewModel job, NijieImage image) { if (isJobCancelled(job)) return; job.PauseEvent.WaitOne(Timeout.Infinite); var filename = MainWindow.makeFilename(job, image, type: MainWindow.FilenameFormatType.Image); job.Message = "Downloading: " + image.BigImageUrl; filename = filename + "." + Util.ParseExtension(image.BigImageUrl); filename = Properties.Settings.Default.RootDirectory + Path.DirectorySeparatorChar + Util.SanitizeFilename(filename); var result = downloadUrl(job, image.BigImageUrl, image.ViewUrl, filename); image.SavedFilename = filename; image.ServerFilename = Util.ExtractFilenameFromUrl(image.BigImageUrl); if (result == NijieException.OK) { image.Filesize = new FileInfo(filename).Length; if (Properties.Settings.Default.SaveDB) SaveImageToDB(job, image); if (Properties.Settings.Default.DumpDownloadedImagesToTextFile) Util.WriteTextFile(filename + Environment.NewLine); job.DownloadCount++; } else if (result == NijieException.DOWNLOAD_SKIPPED) { image.Filesize = new FileInfo(filename).Length; if (Properties.Settings.Default.SaveDB) SaveImageToDB(job, image); job.SkipCount++; } }
private bool isJobCancelled(JobDownloadViewModel job) { lock (_lock) { if (job.CancelToken.IsCancellationRequested && job.Status != JobStatus.Completed && job.Status != JobStatus.Error) { job.Status = JobStatus.Cancelled; job.Message = "Job Cancelled."; MainWindow.Log.Debug(string.Format("Job: {0} cancelled", job.Name)); return true; } if (job.Status == JobStatus.Cancelled) { return true; } return false; } }
private void btnJobCancel_Click(object sender, RoutedEventArgs e) { NewJob = null; }
private void addJobForSearch(NijieSearchOption option) { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Tags; newJob.SearchTag = option.Query; newJob.Status = JobStatus.Added; newJob.StartPage = option.Page; newJob.Sort = option.Sort; newJob.Matching = option.Matching; newJob.SearchBy = option.SearchBy; var result = ShowAddJobDialog(newJob); if (result != null) { AddJob(result); } }
private void addJobForImage(int p) { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Image; newJob.ImageId = p; newJob.Status = JobStatus.Added; AddJob(newJob); }
/// <summary> /// Process images from member page. /// </summary> /// <param name="job"></param> private void doMemberJob(JobDownloadViewModel job) { if (isJobCancelled(job)) return; MainWindow.Log.Debug("Running Member Job: " + job.Name); try { job.Message = "Parsing member page"; job.CurrentPage = job.StartPage; NijieMember memberPage = null; do { memberPage = MainWindow.Bot.ParseMember(job.MemberId, job.MemberMode, job.CurrentPage); job.TotalImages = memberPage.TotalImages; if (Properties.Settings.Default.DownloadAvatar) { var rootPath = Properties.Settings.Default.RootDirectory; var avatarFilename = MainWindow.makeFilename(job, new NijieImage() { Member = memberPage }, type: MainWindow.FilenameFormatType.Avatar); downloadUrl(job, memberPage.AvatarUrl, memberPage.MemberUrl, rootPath + Path.DirectorySeparatorChar + avatarFilename); } foreach (var imageTemp in memberPage.Images) { if (isJobCancelled(job)) return; try { if (job.MemberMode == MemberMode.Bookmark) processImage(job, null, imageTemp); else processImage(job, memberPage, imageTemp); } catch (NijieException ne) { if (ne.ErrorCode == NijieException.DOWNLOAD_ERROR) { job.Exceptions.Add(ne); continue; } else throw; } if (job.DownloadCount > job.Limit && job.Limit != 0) { job.Message = "Image limit reached: " + job.Limit; return; } } job.CurrentPage++; MainWindow.Log.Info("Moving to next page: " + job.CurrentPage); if (job.CurrentPage > job.EndPage && job.EndPage != 0) { job.Message = "Page limit reached: " + job.EndPage; return; } } while (memberPage != null && memberPage.IsNextAvailable); } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Member Job: " + job.Name, ne); } }
/// <summary> /// Process individual image. /// </summary> /// <param name="job"></param> private void doImageJob(JobDownloadViewModel job) { if (isJobCancelled(job)) return; MainWindow.Log.Debug("Running Image Job: " + job.Name); try { NijieImage image = new NijieImage(job.ImageId); processImage(job, null, image); } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Image Job: " + job.Name, ne); } }
private void addException(JobDownloadViewModel job, NijieException nex, string url, string filename) { Application.Current.Dispatcher.BeginInvoke( new Action<BatchDownloadPage>((y) => { if (nex.ErrorCode != NijieException.DOWNLOAD_SKIPPED) { nex.Url = url; nex.Filename = filename; job.Exceptions.Add(nex); job.HasError = true; } }), new object[] { null } ); }
/// <summary> /// Run job on Task factory /// </summary> /// <param name="job"></param> public void DoJob(JobDownloadViewModel job, CancellationTokenSource cancelSource) { job.Status = JobStatus.Queued; job.CancelToken = cancelSource.Token; job.DownloadCount = 0; job.CurrentPage = 1; var taskRef = _jobFactory.StartNew(() => { job.PauseEvent.WaitOne(Timeout.Infinite); long start = DateTime.Now.Ticks; double totalSecond = 0; try { if (Properties.Settings.Default.JobDelay > 0) { MainWindow.Log.Debug(String.Format("Delay before starting job: {0}ms", Properties.Settings.Default.JobDelay)); Thread.Sleep(Properties.Settings.Default.JobDelay); } if (isJobCancelled(job)) return; job.Status = JobStatus.Running; job.Message = "Starting job..."; switch (job.JobType) { case JobType.Member: doMemberJob(job); break; case JobType.Tags: doSearchJob(job); break; case JobType.Image: doImageJob(job); break; } totalSecond = new TimeSpan(DateTime.Now.Ticks - start).TotalSeconds; } catch (Exception ex) { job.Status = JobStatus.Error; job.Message = Util.GetAllInnerExceptionMessage(ex); MainWindow.Log.Error(String.Format("Unhandled Error for {0} ==> {1}", job.Name, ex.Message), ex); } finally { if (job.Status != JobStatus.Error && job.Status != JobStatus.Cancelled) { job.Status = JobStatus.Completed; job.Message = String.Format("Job completed in {0}s", totalSecond); } } MainWindow.Log.Debug(String.Format("Job completed: {0} in {1}s", job.Name, totalSecond)); }, cancelSource.Token , TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness , jobScheduler ); job.TaskRef = taskRef; tasks.Add(taskRef); }
public bool DeleteJob(JobDownloadViewModel job) { try { job.Status = JobStatus.Cancelled; if (tasks.Remove(job.TaskRef)) { MainWindow.Log.Info("Removing Task Reference: " + job.Name); } return true; } catch (Exception exception) { MainWindow.Log.Error("Failed to delete job: " + job.Name, exception); return false; } }
/// <summary> /// Create Filename based on format on job and image information. /// </summary> /// <param name="job"></param> /// <param name="image"></param> /// <param name="currPage"></param> /// <returns></returns> public static string makeFilename(JobDownloadViewModel job, NijieImage image, int currPage = 0, FilenameFormatType type = FilenameFormatType.Image) { try { string filenameFormat = null; switch (type) { case FilenameFormatType.Image: filenameFormat = job.SaveFilenameFormat; break; case FilenameFormatType.Manga: filenameFormat = job.SaveMangaFilenameFormat; break; case FilenameFormatType.Avatar: filenameFormat = job.SaveAvatarFilenameFormat; break; } if (string.IsNullOrWhiteSpace(filenameFormat)) throw new NijieException("Empty filename format!", NijieException.INVALID_SAVE_FILENAME_FORMAT); if (image.Member != null) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_ID, image.Member.MemberId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_NAME, image.Member.UserName); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_NAME, ""); Log.Warn("No Member Information"); } if (job.JobType == JobType.Tags) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SEARCH_TAGS, job.SearchTag); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SEARCH_TAGS, ""); } if (type != FilenameFormatType.Avatar) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_ID, image.ImageId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_TITLE, image.Title); if (image.IsManga) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, "_p" + (currPage + 1).ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, "_p" + currPage.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, " of " + image.ImageUrls.Count); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.ImageUrls[currPage])); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.BigImageUrl)); } if (image.Tags != null || image.Tags.Count > 0) filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_TAGS, String.Join(" ", image.Tags)); else filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_TAGS, ""); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_TITLE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.Member.AvatarUrl)); } if (image.BookmarkedBy != null) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_ID, image.BookmarkedBy.MemberId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_NAME, image.BookmarkedBy.UserName); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_NAME, ""); } return filenameFormat; } catch (Exception ex) { Log.Error("filenameFormat=" + job.SaveFilenameFormat, ex); throw new NijieException("Failed when renaming", ex, NijieException.RENAME_ERROR); } }
private void SaveImageToDB(JobDownloadViewModel job, NijieImage image) { try { lock (_dbLock) { using (var dao = new NijieContext()) { if (Properties.Settings.Default.TraceDB) { dao.Database.Log = MainWindow.Log.Debug; } image.SaveToDb(dao); } } } catch (Exception ex) { MainWindow.Log.Error("Failed to save to DB: " + image.ImageId, ex); job.Message += ex.Message; } }
private void AddJob(JobDownloadViewModel newJob) { var ok = true; if (newJob.JobType == JobType.Tags) { if (String.IsNullOrWhiteSpace(newJob.SearchTag)) { ModernDialog.ShowMessage("Query String cannot be empty!", "Error", MessageBoxButton.OK); ok = false; } } else if (newJob.JobType == JobType.Image) { if (newJob.ImageId <= 0) { ModernDialog.ShowMessage("Image ID must be larger than 0!", "Error", MessageBoxButton.OK); ok = false; } } else if (newJob.JobType == JobType.Member) { if (newJob.MemberId <= 0) { ModernDialog.ShowMessage("Member ID must be larger than 0!", "Error", MessageBoxButton.OK); ok = false; } } if (ok) { if (ViewData.Contains(newJob, new JobDownloadViewModelComparer())) { MessageBox.Show("Job already exists", "Add Job Warning", MessageBoxButton.OK); } else { ViewData.Add(newJob); if (_jobRunner.BatchStatus == JobStatus.Running) { _jobRunner.DoJob(newJob, cancelToken); MainWindow.Log.Debug(String.Format("Add job {0} in running state.", newJob.Name)); } else if (_jobRunner.BatchStatus == JobStatus.Paused) { newJob.Pause(); _jobRunner.DoJob(newJob, cancelToken); MainWindow.Log.Debug(String.Format("Add job {0} in paused state.", newJob.Name)); } } } }
/// <summary> /// Process images from search result. /// </summary> /// <param name="job"></param> private void doSearchJob(JobDownloadViewModel job) { if (isJobCancelled(job)) return; MainWindow.Log.Debug("Running Search Job: " + job.Name); try { job.CurrentPage = job.StartPage; int endPage = job.EndPage; int limit = job.Limit; bool flag = true; job.DownloadCount = 0; while (flag) { if (isJobCancelled(job)) return; job.Message = "Parsing search page: " + job.CurrentPage; var option = new NijieSearchOption() { Query = job.SearchTag, Page = job.CurrentPage, Sort = job.Sort, Matching = job.Matching, SearchBy = job.SearchBy }; var searchPage = MainWindow.Bot.Search(option); job.TotalImages = searchPage.TotalImages; if (searchPage.Images == null || searchPage.Images.Count == 0) { job.Message = "No more images found!"; return; } foreach (var image in searchPage.Images) { if (isJobCancelled(job)) return; if (image.IsFriendOnly || image.IsGoldenMember) { job.Message = String.Format("Skipping ImageId: {0} because locked", image.ImageId); continue; } try { processImage(job, null, image); } catch (NijieException ne) { if (ne.ErrorCode == NijieException.DOWNLOAD_ERROR) { job.Exceptions.Add(ne); continue; } else throw; } if (job.DownloadCount > limit && limit != 0) { job.Message = "Image limit reached: " + limit; return; } } ++job.CurrentPage; MainWindow.Log.Info("Moving to next page: " + job.CurrentPage); if (job.CurrentPage > endPage && endPage != 0) { job.Message = "Page limit reached: " + endPage; return; } else if (job.DownloadCount > limit && limit != 0) { job.Message = "Download count reached: " + limit; return; } flag = searchPage.IsNextAvailable; } } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Search Job: " + job.Name, ne); } }
private void addJobForMember(int memberId, MemberMode mode, string message = "") { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Member; newJob.MemberId = memberId; newJob.Status = JobStatus.Added; newJob.MemberMode = mode; var result = ShowAddJobDialog(newJob, message: message); if (result != null) { AddJob(result); } }
/// <summary> /// Download the image to the specified filename from the Job. /// </summary> /// <param name="job"></param> /// <param name="url"></param> /// <param name="referer"></param> /// <param name="filename"></param> private int downloadUrl(JobDownloadViewModel job, string url, string referer, string filename) { filename = Util.SanitizeFilename(filename); url = Util.FixUrl(url, Nijie.ROOT_DOMAIN); MainWindow.Log.Debug(String.Format("Downloading url: {0} ==> {1}", url, filename)); if (isJobCancelled(job)) return NijieException.CANCELLED; try { job.Message = "Saving to: " + filename; MainWindow.Bot.Download(url, referer, filename, x => { job.Message = x; }, job.CancelToken); } catch (NijieException nex) { job.Message = Util.GetAllInnerExceptionMessage(nex); if (nex.ErrorCode == NijieException.DOWNLOAD_SKIPPED) { MainWindow.Log.Info(nex.Message); } else { MainWindow.Log.Error(nex.Message); addException(job, nex, url, filename); } return nex.ErrorCode; } return NijieException.OK; }
private JobDownloadViewModel ShowAddJobDialog(JobDownloadViewModel newJob, String title = "Add Job", String message = "") { var ctx = new NijieDownloader.UI.Main.Popup.AddJob(newJob, message); var d = new ModernDialog(); d.Buttons = ctx.Buttons; d.Content = ctx; d.Title = title; d.ShowDialog(); return ctx.NewJob; }
private void HandleJobException(JobDownloadViewModel job, NijieException ne) { job.Status = JobStatus.Error; job.Message = Util.GetAllInnerExceptionMessage(ne); }