private static void LoadTxt(string filename, ObservableCollection <JobDownloadViewModel> batchJob) { using (var reader = File.OpenText(filename)) { while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line.StartsWith("#")) { continue; } else { int memberId = 0; Int32.TryParse(line, out memberId); if (memberId > 0) { var job = new JobDownloadViewModel() { MemberId = memberId, JobType = JobType.Member, MemberMode = MemberMode.Images, Limit = 0, StartPage = 1, EndPage = 0, SaveFilenameFormat = NijieDownloader.UI.Properties.Settings.Default.FilenameFormat, SaveMangaFilenameFormat = NijieDownloader.UI.Properties.Settings.Default.MangaFilenameFormat, SaveAvatarFilenameFormat = NijieDownloader.UI.Properties.Settings.Default.AvatarFilenameFormat }; batchJob.Add(job); } } } } }
private void CheckAllJobCompleted(JobDownloadViewModel job) { // check if all other jobs are completed var isAllCompleted = true; foreach (var task in tasks) { if (task == job.TaskRef) { continue; } if (task.IsCompleted == false) { if (task.IsCanceled == true) { continue; } if (task.IsFaulted == true) { continue; } isAllCompleted = false; break; } } if (isAllCompleted) { BatchStatus = JobStatus.Completed; if (finalAction != null) { Application.Current.Dispatcher.BeginInvoke(DispatcherPriority.Background, finalAction); } } }
private void addJobForImage(int p) { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Image; newJob.ImageId = p; newJob.Status = JobStatus.Added; AddJob(newJob); }
public AddJob(JobDownloadViewModel job, String messageCount) { this.NewJob = job; InitializeComponent(); Buttons = new List <Button>(); Buttons.Add(btnJobOk); Buttons.Add(btnJobCancel); this.DataContext = NewJob; txtCountMessage.DataContext = messageCount; }
private JobDownloadViewModel ShowAddJobDialog(JobDownloadViewModel newJob, String title = "Add Job", String message = "") { var ctx = new NijieDownloader.UI.Main.Popup.AddJob(newJob, message); var d = new ModernDialog(); d.Buttons = ctx.Buttons; d.Content = ctx; d.Title = title; d.ShowDialog(); return(ctx.NewJob); }
private void AddJob(JobDownloadViewModel newJob) { var ok = true; if (newJob.JobType == JobType.Tags) { if (String.IsNullOrWhiteSpace(newJob.SearchTag)) { ModernDialog.ShowMessage("Query String cannot be empty!", "Error", MessageBoxButton.OK); ok = false; } } else if (newJob.JobType == JobType.Image) { if (newJob.ImageId <= 0) { ModernDialog.ShowMessage("Image ID must be larger than 0!", "Error", MessageBoxButton.OK); ok = false; } } else if (newJob.JobType == JobType.Member) { if (newJob.MemberId <= 0) { ModernDialog.ShowMessage("Member ID must be larger than 0!", "Error", MessageBoxButton.OK); ok = false; } } if (ok) { if (ViewData.Contains(newJob, new JobDownloadViewModelComparer())) { MessageBox.Show("Job already exists", "Add Job Warning", MessageBoxButton.OK); } else { ViewData.Add(newJob); if (_jobRunner.BatchStatus == JobStatus.Running) { _jobRunner.DoJob(newJob, cancelToken); newJob.PauseEvent.Set(); MainWindow.Log.Debug(String.Format("Add job {0} in running state.", newJob.Name)); } else if (_jobRunner.BatchStatus == JobStatus.Paused) { newJob.Pause(); _jobRunner.DoJob(newJob, cancelToken); MainWindow.Log.Debug(String.Format("Add job {0} in paused state.", newJob.Name)); } } } }
private void addJobForMember(int memberId, MemberMode mode, string message = "") { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Member; newJob.MemberId = memberId; newJob.Status = JobStatus.Added; newJob.MemberMode = mode; var result = ShowAddJobDialog(newJob, message: message); if (result != null) { AddJob(result); } }
private void addException(JobDownloadViewModel job, NijieException nex, string url, string filename) { Application.Current.Dispatcher.BeginInvoke( new Action <BatchDownloadPage>((y) => { if (nex.ErrorCode != NijieException.DOWNLOAD_SKIPPED) { nex.Url = url; nex.Filename = filename; job.Exceptions.Add(nex); job.HasError = true; } }), new object[] { null } ); }
/// <summary> /// Parse the image page /// - Illustration /// - Manga /// </summary> /// <param name="job"></param> /// <param name="memberPage"></param> /// <param name="imageTemp"></param> private void processImage(JobDownloadViewModel job, NijieMember memberPage, NijieImage imageTemp) { if (isJobCancelled(job)) { return; } MainWindow.Log.Debug("Processing Image:" + imageTemp.ImageId); // skip if exists in DB if (Properties.Settings.Default.SkipIfExistsInDB && !NijieDownloader.Library.Properties.Settings.Default.Overwrite) { using (var dao = new NijieContext()) { var result = NijieImage.IsDownloadedInDB(imageTemp.ImageId); if (result) { job.Message = String.Format("Image {0} already downloaded in DB", imageTemp.ImageId); job.SkipCount++; return; } } } var image = MainWindow.Bot.ParseImage(imageTemp, memberPage); if (image.IsFriendOnly) { // sample: 74587 job.Message = "Image locked!"; return; } if (image.IsGoldenMember) { job.Message = "Image only for Gold Membership"; return; } if (image.IsManga) { processManga(job, image); } else { processIllustration(job, image); } }
public bool DeleteJob(JobDownloadViewModel job) { try { job.Status = JobStatus.Cancelled; if (tasks.Remove(job.TaskRef)) { MainWindow.Log.Info("Removing Task Reference: " + job.Name); } return(true); } catch (Exception exception) { MainWindow.Log.Error("Failed to delete job: " + job.Name, exception); return(false); } }
private bool isJobCancelled(JobDownloadViewModel job) { lock (_lock) { if (job.CancelToken.IsCancellationRequested && job.Status != JobStatus.Completed && job.Status != JobStatus.Error) { job.Status = JobStatus.Cancelled; job.Message = "Job Cancelled."; MainWindow.Log.Debug(string.Format("Job: {0} cancelled", job.Name)); return(true); } if (job.Status == JobStatus.Cancelled) { return(true); } return(false); } }
private void addJobForSearch(NijieSearchOption option) { var newJob = new JobDownloadViewModel(); newJob.JobType = JobType.Tags; newJob.SearchTag = option.Query; newJob.Status = JobStatus.Added; newJob.StartPage = option.Page; newJob.Sort = option.Sort; newJob.Matching = option.Matching; newJob.SearchBy = option.SearchBy; var result = ShowAddJobDialog(newJob); if (result != null) { AddJob(result); } }
/// <summary> /// Process individual image. /// </summary> /// <param name="job"></param> private void doImageJob(JobDownloadViewModel job) { if (isJobCancelled(job)) { return; } MainWindow.Log.Debug("Running Image Job: " + job.Name); try { NijieImage image = new NijieImage(job.ImageId); processImage(job, null, image); } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Image Job: " + job.Name, ne); } }
private void processIllustration(JobDownloadViewModel job, NijieImage image) { if (isJobCancelled(job)) { return; } job.PauseEvent.WaitOne(Timeout.Infinite); var filename = MainWindow.makeFilename(job, image, type: MainWindow.FilenameFormatType.Image); job.Message = "Downloading: " + image.BigImageUrl; filename = filename + "." + Util.ParseExtension(image.BigImageUrl); filename = Properties.Settings.Default.RootDirectory + Path.DirectorySeparatorChar + Util.SanitizeFilename(filename); var result = downloadUrl(job, image.BigImageUrl, image.ViewUrl, filename, image.WorkDate); image.SavedFilename = filename; image.ServerFilename = Util.ExtractFilenameFromUrl(image.BigImageUrl); if (result == NijieException.OK) { image.Filesize = new FileInfo(filename).Length; if (Properties.Settings.Default.SaveDB) { SaveImageToDB(job, image); } if (Properties.Settings.Default.DumpDownloadedImagesToTextFile) { Util.WriteTextFile(filename + Environment.NewLine); } job.DownloadCount++; } else if (result == NijieException.DOWNLOAD_SKIPPED) { image.Filesize = new FileInfo(filename).Length; if (Properties.Settings.Default.SaveDB) { SaveImageToDB(job, image); } job.SkipCount++; } }
private void SaveImageToDB(JobDownloadViewModel job, NijieImage image) { try { lock (_dbLock) { using (var dao = new NijieContext()) { if (Properties.Settings.Default.TraceDB) { dao.Database.Log = MainWindow.Log.Debug; } image.SaveToDb(dao); } } } catch (Exception ex) { MainWindow.Log.Error("Failed to save to DB: " + image.ImageId, ex); job.Message += ex.Message; } }
/// <summary> /// Download the image to the specified filename from the Job. /// </summary> /// <param name="job"></param> /// <param name="url"></param> /// <param name="referer"></param> /// <param name="filename"></param> private int downloadUrl(JobDownloadViewModel job, string url, string referer, string filename, DateTime?workdate = null) { filename = Util.SanitizeFilename(filename); url = Util.FixUrl(url, Nijie.ROOT_DOMAIN); MainWindow.Log.Debug(String.Format("Downloading url: {0} ==> {1}", url, filename)); if (isJobCancelled(job)) { return(NijieException.CANCELLED); } try { job.Message = "Saving to: " + filename; MainWindow.Bot.Download(url, referer, filename, x => { job.Message = x; }, job.CancelToken); if (!NijieDownloader.UI.Properties.Settings.Default.UseServerDate) { File.SetLastWriteTime(filename, DateTime.Now); } } catch (NijieException nex) { job.Message = Util.GetAllInnerExceptionMessage(nex); if (nex.ErrorCode == NijieException.DOWNLOAD_SKIPPED) { MainWindow.Log.Info(nex.Message); } else { MainWindow.Log.Error(nex.Message); addException(job, nex, url, filename); } return(nex.ErrorCode); } return(NijieException.OK); }
/// <summary> /// Create Filename based on format on job and image information. /// </summary> /// <param name="job"></param> /// <param name="image"></param> /// <param name="currPage"></param> /// <returns></returns> public static string makeFilename(JobDownloadViewModel job, NijieImage image, int currPage = 0, FilenameFormatType type = FilenameFormatType.Image) { try { string filenameFormat = null; switch (type) { case FilenameFormatType.Image: filenameFormat = job.SaveFilenameFormat; break; case FilenameFormatType.Manga: filenameFormat = job.SaveMangaFilenameFormat; break; case FilenameFormatType.Avatar: filenameFormat = job.SaveAvatarFilenameFormat; break; } if (string.IsNullOrWhiteSpace(filenameFormat)) { throw new NijieException("Empty filename format!", NijieException.INVALID_SAVE_FILENAME_FORMAT); } if (image.Member != null) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_ID, image.Member.MemberId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_NAME, image.Member.UserName); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MEMBER_NAME, ""); Log.Warn("No Member Information"); } if (job.JobType == JobType.Tags) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SEARCH_TAGS, job.SearchTag); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SEARCH_TAGS, ""); } if (type != FilenameFormatType.Avatar) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_ID, image.ImageId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_TITLE, image.Title); if (image.IsManga) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, "_p" + (currPage + 1).ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, "_p" + currPage.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, image.ImageUrls.Count.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.ImageUrls[currPage])); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.BigImageUrl)); } if (image.Tags != null || image.Tags.Count > 0) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_TAGS, String.Join(" ", image.Tags)); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_TAGS, ""); } } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_IMAGE_TITLE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_PAGE_ZERO, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_MAX_PAGE, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_SERVER_FILENAME, Util.ExtractFilenameFromUrl(image.Member.AvatarUrl)); } if (image.BookmarkedBy != null) { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_ID, image.BookmarkedBy.MemberId.ToString()); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_NAME, image.BookmarkedBy.UserName); } else { filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_ID, ""); filenameFormat = filenameFormat.Replace(FILENAME_FORMAT_BOOKMARKED_MEMBER_NAME, ""); } // workdate if (FILENAME_FORMAT_WORKDATE.IsMatch(filenameFormat)) { var match = FILENAME_FORMAT_WORKDATE.Match(filenameFormat); var fmtDate = "yyyyMMdd"; if (!String.IsNullOrEmpty(match.Groups[1].Value)) { fmtDate = match.Groups[1].Value; } var strDate = image.WorkDate.ToString(fmtDate); filenameFormat = filenameFormat.Replace(match.Value, strDate); } return(filenameFormat); } catch (Exception ex) { Log.Error("filenameFormat=" + job.SaveFilenameFormat, ex); throw new NijieException("Failed when renaming", ex, NijieException.RENAME_ERROR); } }
/// <summary> /// Process images from search result. /// </summary> /// <param name="job"></param> private void doSearchJob(JobDownloadViewModel job) { if (isJobCancelled(job)) { return; } MainWindow.Log.Debug("Running Search Job: " + job.Name); try { job.CurrentPage = job.StartPage; int endPage = job.EndPage; int limit = job.Limit; bool flag = true; job.DownloadCount = 0; while (flag) { if (isJobCancelled(job)) { return; } job.Message = "Parsing search page: " + job.CurrentPage; MainWindow.Log.Info("Processing page: " + job.CurrentPage); var option = new NijieSearchOption() { Query = job.SearchTag, Page = job.CurrentPage, Sort = job.Sort, Matching = job.Matching, SearchBy = job.SearchBy }; var searchPage = MainWindow.Bot.Search(option); job.TotalImages = searchPage.TotalImages; if (searchPage.Images == null || searchPage.Images.Count == 0) { job.Message = "No more images found!"; return; } foreach (var image in searchPage.Images) { if (isJobCancelled(job)) { return; } if (image.IsFriendOnly || image.IsGoldenMember) { job.Message = String.Format("Skipping ImageId: {0} because locked", image.ImageId); continue; } try { #if DEBUG job.Message = "Image " + image.ImageId; job.DownloadCount++; #else processImage(job, null, image); #endif } catch (NijieException ne) { if (ne.ErrorCode == NijieException.DOWNLOAD_ERROR) { job.Exceptions.Add(ne); MainWindow.Log.Error(String.Format("Error when processing Image: {0}", image.ImageId), ne); continue; } else { throw; } } if (job.DownloadCount > limit && limit != 0) { job.Message = "Image limit reached: " + limit; MainWindow.Log.Info(job.Message + " for Job: " + job.SearchTag); return; } } ++job.CurrentPage; if (job.CurrentPage > endPage && endPage != 0) { job.Message = "Page limit reached: " + endPage; MainWindow.Log.Info(job.Message + " for Job: " + job.SearchTag); return; } else if (job.DownloadCount > limit && limit != 0) { job.Message = "Download count reached: " + limit; MainWindow.Log.Info(job.Message + " for Job: " + job.SearchTag); return; } flag = searchPage.IsNextAvailable; if (!flag) { MainWindow.Log.Info("No more image for Job: " + job.SearchTag); } } } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Search Job: " + job.Name, ne); } }
/// <summary> /// Process images from member page. /// </summary> /// <param name="job"></param> private void doMemberJob(JobDownloadViewModel job) { if (isJobCancelled(job)) { return; } MainWindow.Log.Debug("Running Member Job: " + job.Name); try { job.Message = "Parsing member page"; job.CurrentPage = job.StartPage; NijieMember memberPage = null; do { memberPage = MainWindow.Bot.ParseMember(job.MemberId, job.MemberMode, job.CurrentPage); job.TotalImages = memberPage.TotalImages; if (Properties.Settings.Default.DownloadAvatar) { var rootPath = Properties.Settings.Default.RootDirectory; var avatarFilename = MainWindow.makeFilename(job, new NijieImage() { Member = memberPage }, type: MainWindow.FilenameFormatType.Avatar); downloadUrl(job, memberPage.AvatarUrl, memberPage.MemberUrl, rootPath + Path.DirectorySeparatorChar + avatarFilename); } foreach (var imageTemp in memberPage.Images) { if (isJobCancelled(job)) { return; } try { if (job.MemberMode == MemberMode.Bookmark) { processImage(job, null, imageTemp); } else { processImage(job, memberPage, imageTemp); } } catch (NijieException ne) { if (ne.ErrorCode == NijieException.DOWNLOAD_ERROR) { job.Exceptions.Add(ne); continue; } else { throw; } } if (job.DownloadCount > job.Limit && job.Limit != 0) { job.Message = "Image limit reached: " + job.Limit; return; } } job.CurrentPage++; MainWindow.Log.Info("Moving to next page: " + job.CurrentPage); if (job.CurrentPage > job.EndPage && job.EndPage != 0) { job.Message = "Page limit reached: " + job.EndPage; return; } } while (memberPage != null && memberPage.IsNextAvailable); } catch (NijieException ne) { HandleJobException(job, ne); MainWindow.Log.Error("Error when processing Member Job: " + job.Name, ne); } }
private void processManga(JobDownloadViewModel job, NijieImage image) { var downloaded = -1; MainWindow.Log.Debug("Processing Manga Images:" + image.ImageId); string lastFilename = "", lastUrl = ""; for (int i = 0; i < image.ImageUrls.Count; ++i) { downloaded = -1; if (isJobCancelled(job)) { return; } job.PauseEvent.WaitOne(Timeout.Infinite); var filename = MainWindow.makeFilename(job, image, i, MainWindow.FilenameFormatType.Manga); job.Message = "Downloading: " + image.ImageUrls[i]; var pagefilename = filename; if (!(job.SaveMangaFilenameFormat.Contains(MainWindow.FILENAME_FORMAT_PAGE) || job.SaveMangaFilenameFormat.Contains(MainWindow.FILENAME_FORMAT_PAGE_ZERO))) { pagefilename += "_p" + i; } pagefilename += "." + Util.ParseExtension(image.ImageUrls[i]); pagefilename = Properties.Settings.Default.RootDirectory + Path.DirectorySeparatorChar + Util.SanitizeFilename(pagefilename); var pages = image.MangaPages as List <NijieMangaInfo>; downloaded = downloadUrl(job, image.ImageUrls[i], image.Referer, pagefilename, image.WorkDate); pages[i].SavedFilename = pagefilename; pages[i].ServerFilename = Util.ExtractFilenameFromUrl(image.ImageUrls[i]); if (downloaded == NijieException.OK) { pages[i].Filesize = new FileInfo(pagefilename).Length; if (Properties.Settings.Default.DumpDownloadedImagesToTextFile) { Util.WriteTextFile(pagefilename + Environment.NewLine); } } lastFilename = pagefilename; lastUrl = image.ImageUrls[i]; } image.SavedFilename = lastFilename; image.ServerFilename = Util.ExtractFilenameFromUrl(lastUrl); if (downloaded == NijieException.OK) { image.Filesize = new FileInfo(lastFilename).Length; if (Properties.Settings.Default.SaveDB) { SaveImageToDB(job, image); } job.DownloadCount++; } else if (downloaded == NijieException.DOWNLOAD_SKIPPED) { image.Filesize = new FileInfo(lastFilename).Length; if (Properties.Settings.Default.SaveDB) { SaveImageToDB(job, image); } job.SkipCount++; } }
private void HandleJobException(JobDownloadViewModel job, NijieException ne) { job.Status = JobStatus.Error; job.Message = Util.GetAllInnerExceptionMessage(ne); }
/// <summary> /// Run job on Task factory /// </summary> /// <param name="job"></param> public void DoJob(JobDownloadViewModel job, CancellationTokenSource cancelSource) { job.Status = JobStatus.Queued; job.CancelToken = cancelSource.Token; job.DownloadCount = 0; job.CurrentPage = 1; var taskRef = _jobFactory.StartNew(() => { job.PauseEvent.WaitOne(Timeout.Infinite); long start = DateTime.Now.Ticks; double totalSecond = 0; try { if (Properties.Settings.Default.JobDelay > 0) { MainWindow.Log.Debug(String.Format("Delay before starting job: {0}ms", Properties.Settings.Default.JobDelay)); Thread.Sleep(Properties.Settings.Default.JobDelay); } if (isJobCancelled(job)) { return; } job.Status = JobStatus.Running; job.Message = "Starting job..."; switch (job.JobType) { case JobType.Member: doMemberJob(job); break; case JobType.Tags: doSearchJob(job); break; case JobType.Image: doImageJob(job); break; } totalSecond = new TimeSpan(DateTime.Now.Ticks - start).TotalSeconds; } catch (Exception ex) { job.Status = JobStatus.Error; job.Message = Util.GetAllInnerExceptionMessage(ex); MainWindow.Log.Error(String.Format("Unhandled Error for {0} ==> {1}", job.Name, ex.Message), ex); } finally { if (job.Status != JobStatus.Error && job.Status != JobStatus.Cancelled) { job.Status = JobStatus.Completed; job.Message = String.Format("Job completed in {0}s", totalSecond); } CheckAllJobCompleted(job); } MainWindow.Log.Debug(String.Format("Job completed: {0} in {1}s", job.Name, totalSecond)); }, cancelSource.Token , TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness , jobScheduler ); job.TaskRef = taskRef; tasks.Add(taskRef); }