private IDictionary <int, string> GetPages(IChapterRecord chapter)
{
IDictionary <int, string> pages = new Dictionary <int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var chapterPages = document.SelectNodes(@"(//div[contains(@class, ""tbtitle"") and contains(@class, ""dropdown_right"")]/ul[@class=""dropdown""])/li/a");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
if (!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber))
{
_log.Error("Unable to parse page number '" + pageLink.InnerText + "'");
}
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead
{
pageNumber = Random;
}
pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["href"].Value));
}
return(pages);
}
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord, bool prefetch)
{
if (chapterRecord == null)
throw new ArgumentNullException("chapterRecord");
if (String.IsNullOrEmpty(chapterRecord.ChapterId))
throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord");
if (!prefetch)
{
return GetDownloadInfo(chapterRecord);
}
// this not very nice way is used to workaround this issue: http://system.data.sqlite.org/index.html/tktview/393d954be0
if (chapterRecord.MangaRecord == null)
throw new ArgumentException("MangaRecord must not be null", "chapterRecord");
var mangaKey = new Tuple<Guid, string>(chapterRecord.MangaRecord.Scraper, chapterRecord.MangaRecord.MangaId);
var chapters = DownloadInfoRecordsCache[mangaKey];
if (chapters == null)
{
chapters = _storage.GetChaptersInfo(chapterRecord.MangaRecord).ToDictionary(dci => dci.ChapterRecord.ChapterId);
DownloadInfoRecordsCache[mangaKey] = chapters;
}
DownloadedChapterInfo downloadInfo;
chapters.TryGetValue(chapterRecord.ChapterId, out downloadInfo);
return downloadInfo;
}
public virtual void DownloadChapterAsync(ISemaphore semaphore, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider)
{
if (_backgroundWorker.IsBusy)
{
throw new InvalidOperationException("Download is currently in progress.");
}
if (semaphore == null)
{
throw new ArgumentNullException("semaphore");
}
if (chapter == null)
{
throw new ArgumentNullException("chapter");
}
if (String.IsNullOrEmpty(outputFolder))
{
throw new ArgumentException("Output folder must not be null or empty.", "outputFolder");
}
if (formatProvider == null)
{
throw new ArgumentNullException("formatProvider");
}
var workerParams = new WorkerParams()
{
Chapter = chapter,
Semaphore = semaphore,
OutputFolder = outputFolder,
FormatProvider = formatProvider
};
_backgroundWorker.RunWorkerAsync(workerParams);
}
private IDictionary <int, string> GetPages(IChapterRecord chapter)
{
IDictionary <int, string> pages = new Dictionary <int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var scriptNode = document.SelectSingleNode(@"//div[@id=""image_frame""]/following-sibling::script");
if (scriptNode == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
var scriptText = scriptNode.InnerText;
int page = 1;
foreach (Match match in _pagesRegex.Matches(scriptText))
{
pages.Add(page, GetFullUrl(match.Groups[1].Captures[0].Value));
page++;
}
return(pages);
}
private IDictionary <int, string> GetPages(IChapterRecord chapter)
{
IDictionary <int, string> pages = new Dictionary <int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var chapterPages = document.SelectNodes(@"(//select[@id=""pages""])[1]/option");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
Int32.TryParse(pageLink.InnerText, out pageNumber);
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead
{
pageNumber = Random;
}
pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["value"].Value));
}
return(pages);
}
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path)
{
if (chapter == null)
{
throw new ArgumentNullException("chapter");
}
if (downloadedFiles == null)
{
throw new ArgumentNullException("downloadedFiles");
}
if (String.IsNullOrEmpty(outputFolder))
{
throw new ArgumentException("Output path must not be null or empty.", "outputFolder");
}
path = Path.Combine(outputFolder, GetNameForSave(chapter) + ".cbz");
var fileInfo = new FileInfo(path);
using (var zip = new ZipFile())
{
zip.AddDirectory(downloadedFiles.FullName, null);
zip.Save(fileInfo.FullName);
}
}
public ChapterViewModel(IChapterRecord chapter, DownloadedChapterInfo downloadInfo)
{
if (chapter == null)
throw new ArgumentNullException("chapter");
_chapterRecord = chapter;
_downloadInfo = downloadInfo;
}
public ChapterViewModel(IChapterRecord chapter, DownloadedChapterInfo downloadInfo)
{
if (chapter == null)
{
throw new ArgumentNullException("chapter");
}
_chapterRecord = chapter;
_downloadInfo = downloadInfo;
}
private IDictionary <int, string> GetPages(IChapterRecord chapter)
{
IDictionary <int, string> pages = new Dictionary <int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
GetPagesRecursive(document, pages);
return(pages);
}
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord)
{
if (chapterRecord == null)
throw new ArgumentNullException("chapterRecord");
if (String.IsNullOrEmpty(chapterRecord.ChapterId))
throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord");
return _storage.GetChapterInfo(chapterRecord.ChapterId);
}
public DownloadedChapterInfo(IChapterRecord chapterRecord)
{
if (chapterRecord == null)
{
throw new ArgumentNullException("chapterRecord");
}
ChapterRecord = chapterRecord;
Path = String.Empty;
Downloaded = DateTime.MinValue;
}
public void Download(IChapterRecord chapter, string outputPath, IDownloadFormatProvider formatProvider)
{
var downloadViewModel = new DownloadViewModel(new DownloadedChapterInfo(chapter), _downloadsSemaphore);
downloadViewModel.RemoveFromCollection += DownloadViewModel_RemoveFromCollection;
downloadViewModel.DownloadStarted += DownloadViewModel_DownloadStarted;
Downloads.Add(downloadViewModel);
downloadViewModel.DownloadChapter(outputPath, formatProvider);
OnPropertyChanged(() => HasActiveDownloads);
}
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord)
{
if (chapterRecord == null)
{
throw new ArgumentNullException("chapterRecord");
}
if (String.IsNullOrEmpty(chapterRecord.ChapterId))
{
throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord");
}
return(_storage.GetChapterInfo(chapterRecord.ChapterId));
}
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path)
{
if (chapter == null)
throw new ArgumentNullException("chapter");
if (downloadedFiles == null)
throw new ArgumentNullException("downloadedFiles");
if (String.IsNullOrEmpty(outputFolder))
throw new ArgumentException("Output path must not be null or empty.", "outputFolder");
var outputDir = new DirectoryInfo(Path.Combine(outputFolder, GetNameForSave(chapter)));
path = outputDir.FullName;
DirectoryCopy(downloadedFiles.FullName, path, true);
}
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path)
{
if (chapter == null)
throw new ArgumentNullException("chapter");
if (downloadedFiles == null)
throw new ArgumentNullException("downloadedFiles");
if (String.IsNullOrEmpty(outputFolder))
throw new ArgumentException("Output path must not be null or empty.", "outputFolder");
path = Path.Combine(outputFolder, GetNameForSave(chapter) + ".cbz");
var fileInfo = new FileInfo(path);
using (var zip = new ZipFile())
{
zip.AddDirectory(downloadedFiles.FullName, null);
zip.Save(fileInfo.FullName);
}
}
private IDictionary <int, string> GetPages(IChapterRecord chapter)
{
IDictionary <int, string> pages = new Dictionary <int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var commentsLink = document.SelectSingleNode(@"//a[@id=""comments""]");
if (commentsLink == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
var link = commentsLink.Attributes["href"].Value;
var chapterPages = document.SelectNodes(@"//select[@class=""m""][1]/option");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
if (!Int32.TryParse(pageLink.InnerText, out pageNumber))
{
continue;
}
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary then skip adding it
{
continue;
}
pages.Add(pageNumber, link + pageNumber + ".html");
}
return(pages);
}
public void SaveDownloadedChapter(IChapterRecord chapter, DirectoryInfo downloadedFiles, string outputFolder, out string path)
{
if (chapter == null)
{
throw new ArgumentNullException("chapter");
}
if (downloadedFiles == null)
{
throw new ArgumentNullException("downloadedFiles");
}
if (String.IsNullOrEmpty(outputFolder))
{
throw new ArgumentException("Output path must not be null or empty.", "outputFolder");
}
var outputDir = new DirectoryInfo(Path.Combine(outputFolder, GetNameForSave(chapter)));
path = outputDir.FullName;
DirectoryCopy(downloadedFiles.FullName, path, true);
}
public DownloadedChapterInfo GetDownloadInfo(IChapterRecord chapterRecord, bool prefetch)
{
if (chapterRecord == null)
{
throw new ArgumentNullException("chapterRecord");
}
if (String.IsNullOrEmpty(chapterRecord.ChapterId))
{
throw new ArgumentException("Invalid chapter record, chapter id must not be null or empty.", "chapterRecord");
}
if (!prefetch)
{
return(GetDownloadInfo(chapterRecord));
}
// this not very nice way is used to workaround this issue: http://system.data.sqlite.org/index.html/tktview/393d954be0
if (chapterRecord.MangaRecord == null)
{
throw new ArgumentException("MangaRecord must not be null", "chapterRecord");
}
var mangaKey = new Tuple <Guid, string>(chapterRecord.MangaRecord.Scraper, chapterRecord.MangaRecord.MangaId);
var chapters = DownloadInfoRecordsCache[mangaKey];
if (chapters == null)
{
chapters = _storage.GetChaptersInfo(chapterRecord.MangaRecord).ToDictionary(dci => dci.ChapterRecord.ChapterId);
DownloadInfoRecordsCache[mangaKey] = chapters;
}
DownloadedChapterInfo downloadInfo;
chapters.TryGetValue(chapterRecord.ChapterId, out downloadInfo);
return(downloadInfo);
}
private IDictionary<int, string> GetPages(IChapterRecord chapter)
{
IDictionary<int, string> pages = new Dictionary<int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
GetPagesRecursive(document, pages);
return pages;
}
protected string GetNameForSave(IChapterRecord chapter)
{
string fileName = String.Format("{0} - {1}", chapter.MangaName, chapter.ChapterName).Replace(" ", "_");
return(InvalidPathCharsRegex.Replace(fileName, ""));
}
public virtual void DownloadChapterAsync(ISemaphore semaphore, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider)
{
if (_backgroundWorker.IsBusy)
throw new InvalidOperationException("Download is currently in progress.");
if (semaphore == null)
throw new ArgumentNullException("semaphore");
if (chapter == null)
throw new ArgumentNullException("chapter");
if (String.IsNullOrEmpty(outputFolder))
throw new ArgumentException("Output folder must not be null or empty.", "outputFolder");
if (formatProvider == null)
throw new ArgumentNullException("formatProvider");
var workerParams = new WorkerParams()
{
Chapter = chapter,
Semaphore = semaphore,
OutputFolder = outputFolder,
FormatProvider = formatProvider
};
_backgroundWorker.RunWorkerAsync(workerParams);
}
public void Download(IChapterRecord chapter, string outputPath, IDownloadFormatProvider formatProvider)
{
var downloadViewModel = new DownloadViewModel(new DownloadedChapterInfo(chapter), _downloadsSemaphore);
downloadViewModel.RemoveFromCollection += DownloadViewModel_RemoveFromCollection;
downloadViewModel.DownloadStarted += DownloadViewModel_DownloadStarted;
Downloads.Add(downloadViewModel);
downloadViewModel.DownloadChapter(outputPath, formatProvider);
OnPropertyChanged(() => HasActiveDownloads);
}
private void DownloadChapter(BackgroundWorker backgroundWorker, DoWorkEventArgs e, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider)
{
// add task -> export result
AddTask();
var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "MangaScraper", Guid.NewGuid().ToString()));
try
{
AddTask();
backgroundWorker.ReportProgress(GetPercentComplete(), "Resolving list of pages.");
var pages = _pageResolver(chapter);
AddTask(pages.Count);
TaskDone();
backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("List of pages resolved, chapter has {0} pages.", pages.Count));
int current = 1;
foreach (var page in pages)
{
if (backgroundWorker.CancellationPending)
{
e.Cancel = true;
return;
}
backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("Downloading page {0} from {1}", current, pages.Count));
string imgUrl = _imageFinder(page.Value);
string filePath = GetUniqueFileName(directory.FullName, page.Key, Path.GetExtension(imgUrl));
try
{
RetryHelper.Retry(() => WebHelper.DownloadImage(imgUrl, filePath));
}
catch (Exception ex)
{
_log.Error("Unable to download image from url: '" + imgUrl + "' to '" + filePath + "'", ex);
}
current++;
TaskDone();
}
backgroundWorker.ReportProgress(GetPercentComplete(), "All pages downloaded.");
backgroundWorker.ReportProgress(GetPercentComplete(), "Exporting chapter");
if (backgroundWorker.CancellationPending)
{
e.Cancel = true;
return;
}
string path;
formatProvider.SaveDownloadedChapter(chapter, directory, outputFolder, out path);
// save result path of the downloaded file
e.Result = path;
TaskDone();
backgroundWorker.ReportProgress(GetPercentComplete(), "Download completed");
}
finally
{
// remove temp dir
directory.Delete(true);
}
}
private IDictionary<int, string> GetPages(IChapterRecord chapter)
{
IDictionary<int, string> pages = new Dictionary<int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var commentsLink = document.SelectSingleNode(@"//a[@id=""comments""]");
if (commentsLink == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
var link = commentsLink.Attributes["href"].Value;
var chapterPages = document.SelectNodes(@"//select[@class=""m""][1]/option");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
if (!Int32.TryParse(pageLink.InnerText, out pageNumber))
continue;
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary then skip adding it
continue;
pages.Add(pageNumber, link + pageNumber + ".html");
}
return pages;
}
private IDictionary<int, string> GetPages(IChapterRecord chapter)
{
IDictionary<int, string> pages = new Dictionary<int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var chapterPages = document.SelectNodes(@"(//select[@id=""page_select""])[1]/option");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
if(!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber))
_log.Error("Unable to parse page number '" + pageLink.InnerText + "'");
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead
pageNumber = Random;
pages.Add(pageNumber, GetFullUrl(pageLink.Attributes["value"].Value));
}
return pages;
}
private void DownloadChapter(BackgroundWorker backgroundWorker, DoWorkEventArgs e, IChapterRecord chapter, string outputFolder, IDownloadFormatProvider formatProvider)
{
// add task -> export result
AddTask();
var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "MangaScraper", Guid.NewGuid().ToString()));
try
{
AddTask();
backgroundWorker.ReportProgress(GetPercentComplete(), "Resolving list of pages.");
var pages = _pageResolver(chapter);
AddTask(pages.Count);
TaskDone();
backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("List of pages resolved, chapter has {0} pages.", pages.Count));
int current = 1;
foreach (var page in pages)
{
if (backgroundWorker.CancellationPending)
{
e.Cancel = true;
return;
}
backgroundWorker.ReportProgress(GetPercentComplete(), String.Format("Downloading page {0} from {1}", current, pages.Count));
string imgUrl = _imageFinder(page.Value);
string filePath = GetUniqueFileName(directory.FullName, page.Key, Path.GetExtension(imgUrl));
try
{
RetryHelper.Retry(() => WebHelper.DownloadImage(imgUrl, filePath));
}
catch (Exception ex)
{
_log.Error("Unable to download image from url: '" + imgUrl + "' to '" + filePath + "'", ex);
}
current++;
TaskDone();
}
backgroundWorker.ReportProgress(GetPercentComplete(), "All pages downloaded.");
backgroundWorker.ReportProgress(GetPercentComplete(), "Exporting chapter");
if (backgroundWorker.CancellationPending)
{
e.Cancel = true;
return;
}
string path;
formatProvider.SaveDownloadedChapter(chapter, directory, outputFolder, out path);
// save result path of the downloaded file
e.Result = path;
TaskDone();
backgroundWorker.ReportProgress(GetPercentComplete(), "Download completed");
}
finally
{
// remove temp dir
directory.Delete(true);
}
}
private IDictionary<int, string> GetPages(IChapterRecord chapter)
{
IDictionary<int, string> pages = new Dictionary<int, string>();
var document = WebHelper.GetHtmlDocument(chapter.Url);
var scriptNode = document.SelectSingleNode(@"//div[@id=""image_frame""]/following-sibling::script");
if (scriptNode == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
var scriptText = scriptNode.InnerText;
int page = 1;
foreach (Match match in _pagesRegex.Matches(scriptText))
{
pages.Add(page, GetFullUrl(match.Groups[1].Captures[0].Value));
page++;
}
return pages;
}
protected string GetNameForSave(IChapterRecord chapter)
{
string fileName = String.Format("{0} - {1}", chapter.MangaName, chapter.ChapterName).Replace(" ", "_");
return InvalidPathCharsRegex.Replace(fileName, "");
}