private IDictionary <int, string> GetPages(IChapterRecord chapter) { IDictionary <int, string> pages = new Dictionary <int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var chapterPages = document.SelectNodes(@"(//div[contains(@class, ""tbtitle"") and contains(@class, ""dropdown_right"")]/ul[@class=""dropdown""])/li/a"); if (chapterPages == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } foreach (var pageLink in chapterPages) { int pageNumber = 0; if (!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber)) { _log.Error("Unable to parse page number '" + pageLink.InnerText + "'"); } if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead { pageNumber = Random; } pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["href"].Value)); } return(pages); }
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord, bool prefetch) { if (chapterRecord == null) throw new ArgumentNullException("chapterRecord"); if (String.IsNullOrEmpty(chapterRecord.ChapterId)) throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord"); if (!prefetch) { return GetDownloadInfo(chapterRecord); } // this not very nice way is used to workaround this issue: http://system.data.sqlite.org/index.html/tktview/393d954be0 if (chapterRecord.MangaRecord == null) throw new ArgumentException("MangaRecord must not be null", "chapterRecord"); var mangaKey = new Tuple<Guid, string>(chapterRecord.MangaRecord.Scraper, chapterRecord.MangaRecord.MangaId); var chapters = DownloadInfoRecordsCache[mangaKey]; if (chapters == null) { chapters = _storage.GetChaptersInfo(chapterRecord.MangaRecord).ToDictionary(dci => dci.ChapterRecord.ChapterId); DownloadInfoRecordsCache[mangaKey] = chapters; } DownloadedChapterInfo downloadInfo; chapters.TryGetValue(chapterRecord.ChapterId, out downloadInfo); return downloadInfo; }
public virtual void DownloadChapterAsync(ISemaphore semaphore, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider) { if (_backgroundWorker.IsBusy) { throw new InvalidOperationException("Download is currently in progress."); } if (semaphore == null) { throw new ArgumentNullException("semaphore"); } if (chapter == null) { throw new ArgumentNullException("chapter"); } if (String.IsNullOrEmpty(outputFolder)) { throw new ArgumentException("Output folder must not be null or empty.", "outputFolder"); } if (formatProvider == null) { throw new ArgumentNullException("formatProvider"); } var workerParams = new WorkerParams() { Chapter = chapter, Semaphore = semaphore, OutputFolder = outputFolder, FormatProvider = formatProvider }; _backgroundWorker.RunWorkerAsync(workerParams); }
private IDictionary <int, string> GetPages(IChapterRecord chapter) { IDictionary <int, string> pages = new Dictionary <int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var scriptNode = document.SelectSingleNode(@"//div[@id=""image_frame""]/following-sibling::script"); if (scriptNode == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } var scriptText = scriptNode.InnerText; int page = 1; foreach (Match match in _pagesRegex.Matches(scriptText)) { pages.Add(page, GetFullUrl(match.Groups[1].Captures[0].Value)); page++; } return(pages); }
private IDictionary <int, string> GetPages(IChapterRecord chapter) { IDictionary <int, string> pages = new Dictionary <int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var chapterPages = document.SelectNodes(@"(//select[@id=""pages""])[1]/option"); if (chapterPages == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } foreach (var pageLink in chapterPages) { int pageNumber = 0; Int32.TryParse(pageLink.InnerText, out pageNumber); if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead { pageNumber = Random; } pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["value"].Value)); } return(pages); }
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path) { if (chapter == null) { throw new ArgumentNullException("chapter"); } if (downloadedFiles == null) { throw new ArgumentNullException("downloadedFiles"); } if (String.IsNullOrEmpty(outputFolder)) { throw new ArgumentException("Output path must not be null or empty.", "outputFolder"); } path = Path.Combine(outputFolder, GetNameForSave(chapter) + ".cbz"); var fileInfo = new FileInfo(path); using (var zip = new ZipFile()) { zip.AddDirectory(downloadedFiles.FullName, null); zip.Save(fileInfo.FullName); } }
public ChapterViewModel(IChapterRecord chapter, DownloadedChapterInfo downloadInfo) { if (chapter == null) throw new ArgumentNullException("chapter"); _chapterRecord = chapter; _downloadInfo = downloadInfo; }
public ChapterViewModel(IChapterRecord chapter, DownloadedChapterInfo downloadInfo) { if (chapter == null) { throw new ArgumentNullException("chapter"); } _chapterRecord = chapter; _downloadInfo = downloadInfo; }
private IDictionary <int, string> GetPages(IChapterRecord chapter) { IDictionary <int, string> pages = new Dictionary <int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); GetPagesRecursive(document, pages); return(pages); }
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord) { if (chapterRecord == null) throw new ArgumentNullException("chapterRecord"); if (String.IsNullOrEmpty(chapterRecord.ChapterId)) throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord"); return _storage.GetChapterInfo(chapterRecord.ChapterId); }
public DownloadedChapterInfo(IChapterRecord chapterRecord) { if (chapterRecord == null) { throw new ArgumentNullException("chapterRecord"); } ChapterRecord = chapterRecord; Path = String.Empty; Downloaded = DateTime.MinValue; }
public void Download(IChapterRecord chapter, string outputPath, IDownloadFormatProvider formatProvider) { var downloadViewModel = new DownloadViewModel(new DownloadedChapterInfo(chapter), _downloadsSemaphore); downloadViewModel.RemoveFromCollection += DownloadViewModel_RemoveFromCollection; downloadViewModel.DownloadStarted += DownloadViewModel_DownloadStarted; Downloads.Add(downloadViewModel); downloadViewModel.DownloadChapter(outputPath, formatProvider); OnPropertyChanged(() => HasActiveDownloads); }
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord) { if (chapterRecord == null) { throw new ArgumentNullException("chapterRecord"); } if (String.IsNullOrEmpty(chapterRecord.ChapterId)) { throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord"); } return(_storage.GetChapterInfo(chapterRecord.ChapterId)); }
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path) { if (chapter == null) throw new ArgumentNullException("chapter"); if (downloadedFiles == null) throw new ArgumentNullException("downloadedFiles"); if (String.IsNullOrEmpty(outputFolder)) throw new ArgumentException("Output path must not be null or empty.", "outputFolder"); var outputDir = new DirectoryInfo(Path.Combine(outputFolder, GetNameForSave(chapter))); path = outputDir.FullName; DirectoryCopy(downloadedFiles.FullName, path, true); }
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path) { if (chapter == null) throw new ArgumentNullException("chapter"); if (downloadedFiles == null) throw new ArgumentNullException("downloadedFiles"); if (String.IsNullOrEmpty(outputFolder)) throw new ArgumentException("Output path must not be null or empty.", "outputFolder"); path = Path.Combine(outputFolder, GetNameForSave(chapter) + ".cbz"); var fileInfo = new FileInfo(path); using (var zip = new ZipFile()) { zip.AddDirectory(downloadedFiles.FullName, null); zip.Save(fileInfo.FullName); } }
private IDictionary <int, string> GetPages(IChapterRecord chapter) { IDictionary <int, string> pages = new Dictionary <int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var commentsLink = document.SelectSingleNode(@"//a[@id=""comments""]"); if (commentsLink == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } var link = commentsLink.Attributes["href"].Value; var chapterPages = document.SelectNodes(@"//select[@class=""m""][1]/option"); if (chapterPages == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } foreach (var pageLink in chapterPages) { int pageNumber = 0; if (!Int32.TryParse(pageLink.InnerText, out pageNumber)) { continue; } if (pages.ContainsKey(pageNumber)) // if page is already in dictionary then skip adding it { continue; } pages.Add(pageNumber, link + pageNumber + ".html"); } return(pages); }
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path) { if (chapter == null) { throw new ArgumentNullException("chapter"); } if (downloadedFiles == null) { throw new ArgumentNullException("downloadedFiles"); } if (String.IsNullOrEmpty(outputFolder)) { throw new ArgumentException("Output path must not be null or empty.", "outputFolder"); } var outputDir = new DirectoryInfo(Path.Combine(outputFolder, GetNameForSave(chapter))); path = outputDir.FullName; DirectoryCopy(downloadedFiles.FullName, path, true); }
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord, bool prefetch) { if (chapterRecord == null) { throw new ArgumentNullException("chapterRecord"); } if (String.IsNullOrEmpty(chapterRecord.ChapterId)) { throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord"); } if (!prefetch) { return(GetDownloadInfo(chapterRecord)); } // this not very nice way is used to workaround this issue: http://system.data.sqlite.org/index.html/tktview/393d954be0 if (chapterRecord.MangaRecord == null) { throw new ArgumentException("MangaRecord must not be null", "chapterRecord"); } var mangaKey = new Tuple <Guid, string>(chapterRecord.MangaRecord.Scraper, chapterRecord.MangaRecord.MangaId); var chapters = DownloadInfoRecordsCache[mangaKey]; if (chapters == null) { chapters = _storage.GetChaptersInfo(chapterRecord.MangaRecord).ToDictionary(dci => dci.ChapterRecord.ChapterId); DownloadInfoRecordsCache[mangaKey] = chapters; } DownloadedChapterInfo downloadInfo; chapters.TryGetValue(chapterRecord.ChapterId, out downloadInfo); return(downloadInfo); }
private IDictionary<int, string> GetPages(IChapterRecord chapter) { IDictionary<int, string> pages = new Dictionary<int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); GetPagesRecursive(document, pages); return pages; }
protected string GetNameForSave(IChapterRecord chapter) { string fileName = String.Format("{0} - {1}", chapter.MangaName, chapter.ChapterName).Replace(" ", "_"); return(InvalidPathCharsRegex.Replace(fileName, "")); }
public virtual void DownloadChapterAsync(ISemaphore semaphore, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider) { if (_backgroundWorker.IsBusy) throw new InvalidOperationException("Download is currently in progress."); if (semaphore == null) throw new ArgumentNullException("semaphore"); if (chapter == null) throw new ArgumentNullException("chapter"); if (String.IsNullOrEmpty(outputFolder)) throw new ArgumentException("Output folder must not be null or empty.", "outputFolder"); if (formatProvider == null) throw new ArgumentNullException("formatProvider"); var workerParams = new WorkerParams() { Chapter = chapter, Semaphore = semaphore, OutputFolder = outputFolder, FormatProvider = formatProvider }; _backgroundWorker.RunWorkerAsync(workerParams); }
public void Download(IChapterRecord chapter, string outputPath, IDownloadFormatProvider formatProvider) { var downloadViewModel = new DownloadViewModel(new DownloadedChapterInfo(chapter), _downloadsSemaphore); downloadViewModel.RemoveFromCollection += DownloadViewModel_RemoveFromCollection; downloadViewModel.DownloadStarted += DownloadViewModel_DownloadStarted; Downloads.Add(downloadViewModel); downloadViewModel.DownloadChapter(outputPath, formatProvider); OnPropertyChanged(() => HasActiveDownloads); }
private void DownloadChapter(BackgroundWorker backgroundWorker, DoWorkEventArgs e, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider) { // add task -> export result AddTask(); var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "MangaScraper", Guid.NewGuid().ToString())); try { AddTask(); backgroundWorker.ReportProgress(GetPercentComplete(), "Resolving list of pages."); var pages = _pageResolver(chapter); AddTask(pages.Count); TaskDone(); backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("List of pages resolved, chapter has {0} pages.", pages.Count)); int current = 1; foreach (var page in pages) { if (backgroundWorker.CancellationPending) { e.Cancel = true; return; } backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("Downloading page {0} from {1}", current, pages.Count)); string imgUrl = _imageFinder(page.Value); string filePath = GetUniqueFileName(directory.FullName, page.Key, Path.GetExtension(imgUrl)); try { RetryHelper.Retry(() => WebHelper.DownloadImage(imgUrl, filePath)); } catch (Exception ex) { _log.Error("Unable to download image from url: '" + imgUrl + "' to '" + filePath + "'", ex); } current++; TaskDone(); } backgroundWorker.ReportProgress(GetPercentComplete(), "All pages downloaded."); backgroundWorker.ReportProgress(GetPercentComplete(), "Exporting chapter"); if (backgroundWorker.CancellationPending) { e.Cancel = true; return; } string path; formatProvider.SaveDownloadedChapter(chapter, directory, outputFolder, out path); // save result path of the downloaded file e.Result = path; TaskDone(); backgroundWorker.ReportProgress(GetPercentComplete(), "Download completed"); } finally { // remove temp dir directory.Delete(true); } }
private IDictionary<int, string> GetPages(IChapterRecord chapter) { IDictionary<int, string> pages = new Dictionary<int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var commentsLink = document.SelectSingleNode(@"//a[@id=""comments""]"); if (commentsLink == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } var link = commentsLink.Attributes["href"].Value; var chapterPages = document.SelectNodes(@"//select[@class=""m""][1]/option"); if (chapterPages == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } foreach (var pageLink in chapterPages) { int pageNumber = 0; if (!Int32.TryParse(pageLink.InnerText, out pageNumber)) continue; if (pages.ContainsKey(pageNumber)) // if page is already in dictionary then skip adding it continue; pages.Add(pageNumber, link + pageNumber + ".html"); } return pages; }
private IDictionary<int, string> GetPages(IChapterRecord chapter) { IDictionary<int, string> pages = new Dictionary<int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var chapterPages = document.SelectNodes(@"(//select[@id=""page_select""])[1]/option"); if (chapterPages == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } foreach (var pageLink in chapterPages) { int pageNumber = 0; if(!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber)) _log.Error("Unable to parse page number '" + pageLink.InnerText + "'"); if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead pageNumber = Random; pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["value"].Value)); } return pages; }
private void DownloadChapter(BackgroundWorker backgroundWorker, DoWorkEventArgs e, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider) { // add task -> export result AddTask(); var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "MangaScraper", Guid.NewGuid().ToString())); try { AddTask(); backgroundWorker.ReportProgress(GetPercentComplete(), "Resolving list of pages."); var pages = _pageResolver(chapter); AddTask(pages.Count); TaskDone(); backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("List of pages resolved, chapter has {0} pages.", pages.Count)); int current = 1; foreach (var page in pages) { if (backgroundWorker.CancellationPending) { e.Cancel = true; return; } backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("Downloading page {0} from {1}", current, pages.Count)); string imgUrl = _imageFinder(page.Value); string filePath = GetUniqueFileName(directory.FullName, page.Key, Path.GetExtension(imgUrl)); try { RetryHelper.Retry(() => WebHelper.DownloadImage(imgUrl, filePath)); } catch (Exception ex) { _log.Error("Unable to download image from url: '" + imgUrl + "' to '" + filePath + "'", ex); } current++; TaskDone(); } backgroundWorker.ReportProgress(GetPercentComplete(), "All pages downloaded."); backgroundWorker.ReportProgress(GetPercentComplete(), "Exporting chapter"); if (backgroundWorker.CancellationPending) { e.Cancel = true; return; } string path; formatProvider.SaveDownloadedChapter(chapter, directory, outputFolder, out path); // save result path of the downloaded file e.Result = path; TaskDone(); backgroundWorker.ReportProgress(GetPercentComplete(), "Download completed"); } finally { // remove temp dir directory.Delete(true); } }
private IDictionary<int, string> GetPages(IChapterRecord chapter) { IDictionary<int, string> pages = new Dictionary<int, string>(); var document = WebHelper.GetHtmlDocument(chapter.Url); var scriptNode = document.SelectSingleNode(@"//div[@id=""image_frame""]/following-sibling::script"); if (scriptNode == null) { throw new ParserException("Could not find expected elements on website.", document.InnerHtml); } var scriptText = scriptNode.InnerText; int page = 1; foreach (Match match in _pagesRegex.Matches(scriptText)) { pages.Add(page, GetFullUrl(match.Groups[1].Captures[0].Value)); page++; } return pages; }
protected string GetNameForSave(IChapterRecord chapter) { string fileName = String.Format("{0} - {1}", chapter.MangaName, chapter.ChapterName).Replace(" ", "_"); return InvalidPathCharsRegex.Replace(fileName, ""); }