public override async Task <IEnumerable <string> > FindImages(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); // find all pages in a chapter string input = await downloader.DownloadStringAsync(chapter.Url); string regExPages = "<li><a href=\"(?<Value>http://mangastream.com/r/[^\"]+)\">[^<]+</a>"; var pages = parser.Parse(regExPages, input, "Value"); // find all images in pages var pageData = await downloader.DownloadStringAsync(pages, new Progress <int>((count) => { var f = (float)count / pages.Count(); int i = Convert.ToInt32(f * 100); progress.Report(i); }), cancellationToken); var images = parser.Parse("<img id=\"manga-page\" src=\"(?<Value>[^\"]+)\"/>", pageData, "Value"); return(images); }
public override async Task <IEnumerable <string> > FindImanges(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); // find all pages in a chapter string input = await downloader.DownloadStringAsync(chapter.Url); var pages = parser.Parse(@"<option value=""(?<Value>[^""]+)"" (|selected=""selected"")>\d+</option>", input, "Value"); // transform pages link pages = pages.Select(p => { var value = new Uri(new Uri(chapter.Url), p).AbsoluteUri; return(value); }).ToList(); // find all images in pages var pageData = await downloader.DownloadStringAsync(pages, new Progress <int>((count) => { var f = (float)count / pages.Count(); int i = Convert.ToInt32(f * 100); progress.Report(i); }), cancellationToken); var images = parser.Parse("<img src=\"(?<Value>[^\"]+)\" onload=", pageData, "Value"); return(images); }
public override async Task <IEnumerable <string> > FindImanges(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { progress.Report(0); var downloader = new DownloadService { Cookies = LoginBatoto(_username, _password), Referer = "http://bato.to/reader" }; var parser = new ParserHelper(); // find all pages in a chapter var chapterUrl = TransformChapterUrl(chapter.Url); var input = await downloader.DownloadStringAsync(chapterUrl); var pages = parser.Parse(@"<option value=""(?<Value>http://bato.to/reader#[^""]+)""[^>]+>page", input, "Value"); // transform pages link var transformedPages = pages.Select(TransformChapterUrl).ToList(); // find all images in pages var pageData = await downloader.DownloadStringAsync( transformedPages, new Progress <int>((count) => { var f = (float)count / transformedPages.Count(); var i = Convert.ToInt32(f * 100); progress.Report(i); }), cancellationToken); var images = parser.Parse("img src=\"(?<Value>[^\"]+)\" style=\"z-index: 1003", pageData, "Value"); progress.Report(100); return(images); }
public override async Task<IEnumerable<string>> FindImages(Chapter chapter, IProgress<int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); // find all pages in a chapter string input = await downloader.DownloadStringAsync(chapter.Url); string regExPages = @"<select name=""pagejump"" class=""page"" onchange=""javascript:window.location='(?<Value>[^']+)'\+this\.value\+'\.html';"">"; var pageBase = parser.Parse(regExPages, input, "Value").FirstOrDefault(); var pagesExtend = parser.Parse(@"<option value=""(?<FileName>\d+)""(| selected=""selected"")>Page \d+</option>", input, "FileName"); // transform pages link pagesExtend = pagesExtend.Select(p => { string baseLink = pageBase + p + ".html"; var value = new Uri(new Uri(chapter.Url), baseLink).AbsoluteUri; return value; }).ToList(); // find all images in pages var pageData = await downloader.DownloadStringAsync(pagesExtend, new Progress<int>((count) => { var f = (float)count / pagesExtend.Count(); int i = Convert.ToInt32(f * 100); progress.Report(i); }), cancellationToken); var images = parser.Parse(@"<img src=""(?<Value>[^""]+)"" border=""0"" alt=""[^""]+"" />\n", pageData, "Value"); return images; }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { progress.Report(0); var downloader = new DownloadService { Cookies = LoginBatoto(_username, _password), Referrer = "http://bato.to/reader" }; var parser = new ParserHelper(); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga, cancellationToken); var allLanguagesRegEx = "<a href=\"(?<Value>https://bato.to/reader#[^\"]+)\" title=\"(?<Name>[^|]+)"; // Choose only specific languages if it set so in config if (!string.IsNullOrEmpty(_languagesRegEx)) { allLanguagesRegEx = _languagesRegEx + allLanguagesRegEx; } var chaps = parser.ParseGroup(allLanguagesRegEx, input, "Name", "Value"); progress.Report(100); return(chaps); }
public async Task Batoto_Test() { string url = "http://bato.to/comic/_/comics/21st-century-boys-r1591"; var service = FrameworkProvider.GetService(url); service.Configuration(new[] { new KeyValuePair <string, object>("Username", "gufrohepra"), new KeyValuePair <string, object>("Password", "123"), new KeyValuePair <string, object>("Languages", "") }); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("Vol.01 Ch.01 Read Online", chapter.Name); Assert.AreEqual("https://bato.to/reader#900d11d96d1466f2", chapter.Url); var images = await service.FindImages(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(31, images.Count()); Assert.IsTrue(images.ToArray()[0].StartsWith("http://img.bato.to/comics/2014/10/08/2/read54357eb5e1ca9/img000001.jpg")); Assert.IsTrue(images.ToArray()[1].StartsWith("http://img.bato.to/comics/2014/10/08/2/read54357eb5e1ca9/img000002.jpg")); Assert.IsTrue(images.ToArray()[2].StartsWith("http://img.bato.to/comics/2014/10/08/2/read54357eb5e1ca9/img000003.jpg")); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0], _source.Token); Assert.IsNotNull(imageString, "Cannot download image!"); }
public override async Task <IEnumerable <string> > FindImages(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { progress.Report(0); var downloader = new DownloadService(); var parser = new ParserHelper(); string html, chapterUrl; try { Logger.Info("> FindImages()"); chapterUrl = string.Concat(chapter.Url, "/full"); html = await downloader.DownloadStringAsync(chapterUrl, cancellationToken); if (!string.IsNullOrWhiteSpace(html)) { string pattern = "<img\\s+class=\"chapter_img\".*src=\"(?<Value>.[^\"]*)"; var pages = parser.Parse(pattern, html, "Value"); return(pages); } } finally { downloader = null; parser = null; html = null; chapterUrl = null; } throw new Core.CustomException.MangaRipperException("Can't find pages."); }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { Logger.Info($@"> FindChapters(): {manga}"); progress.Report(0); var downloader = new DownloadService(); var parser = new ParserHelper(); try { string html = await downloader.DownloadStringAsync(manga, cancellationToken); progress.Report(50); string pattern = "<a\\s+class=\"ch-name\"\\s+href=\"(?<Value>.[^\"]*)\">(?<Name>.*)(?=\\<)"; var chapters = parser.ParseGroup(pattern, html, "Name", "Value"); progress.Report(90); ((List <Chapter>)chapters).Reverse(); progress.Report(100); return(chapters); } finally { downloader = null; parser = null; } }
public async Task MangaReader_Test() { string url = "http://www.mangareader.net/naruto"; var service = FrameworkProvider.GetService(url); // Test service can find chapters var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); // Test chapters are in correct order. var chapter = chapters.Last(); Assert.AreEqual("Naruto 1", chapter.Name); Assert.AreEqual("http://www.mangareader.net/naruto/1", chapter.Url); // Test there're no duplicated chapters. var anyDuplicated = chapters.GroupBy(x => x.Url).Any(g => g.Count() > 1); Assert.IsFalse(anyDuplicated, "There're duplicated chapters."); // Test service can find images. var images = await service.FindImanges(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(53, images.Count()); Assert.AreEqual("http://i10.mangareader.net/naruto/1/naruto-1564773.jpg", images.ToArray()[0]); Assert.AreEqual("http://i4.mangareader.net/naruto/1/naruto-1564774.jpg", images.ToArray()[1]); Assert.AreEqual("http://i1.mangareader.net/naruto/1/naruto-1564825.jpg", images.ToArray()[52]); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0]); Assert.IsNotNull(imageString, "Cannot download image!"); }
public override async Task <IEnumerable <string> > FindImages(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { progress.Report(0); var downloader = new DownloadService(); var parser = new ParserHelper(); var pages = (await FindPagesInChapter(chapter.Url, cancellationToken)).ToList(); var transformedPages = TransformPagesUrl(chapter.Url, pages).ToArray(); // find all images in pages var pageData = await downloader.DownloadStringAsync( transformedPages, new Progress <int>(count => { var f = (float)count / transformedPages.Count(); var i = Convert.ToInt32(f * 100); progress.Report(i); }), cancellationToken); var images = parser.Parse("<img src=\"(?<Value>[^\"]+)\"[ ]+width=", pageData, "Value"); progress.Report(100); return(images); }
public async Task MangaFox_Test() { // Test with unlicensed manga. Appveyor CI is US based and cannot access licensed manga in the US. // If we test with a licensed manga, this test will failed. string url = "http://mangafox.me/manga/tian_jiang_xian_shu_nan/"; var service = FrameworkProvider.GetService(url); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("Tian Jiang Xian Shu Nan 1", chapter.Name); Assert.AreEqual("http://mangafox.me/manga/tian_jiang_xian_shu_nan/c001/1.html", chapter.Url); var images = await service.FindImanges(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(15, images.Count()); Assert.IsTrue(images.ToArray()[0].StartsWith("http://h.mfcdn.net/store/manga/19803/001.0/compressed/q001.jpg")); Assert.IsTrue(images.ToArray()[1].StartsWith("http://h.mfcdn.net/store/manga/19803/001.0/compressed/q002.jpg")); Assert.IsTrue(images.ToArray()[2].StartsWith("http://h.mfcdn.net/store/manga/19803/001.0/compressed/q003.jpg")); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0]); Assert.IsNotNull(imageString, "Cannot download image!"); }
private async Task <IEnumerable <string> > FindPagesInChapter(string chapterUrl, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); var input = await downloader.DownloadStringAsync(chapterUrl, cancellationToken); return(parser.Parse(@"<option value=""(?<Value>[^""]+)"" (|selected=""selected"")>\d+</option>", input, "Value")); }
public override async Task<IEnumerable<Chapter>> FindChapters(string manga, IProgress<int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); progress.Report(0); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga); string regEx = "<td class=\"datarow-0\"><a href=\"(?<Value>[^\"]+)\"><img src=\"http://read.mangashare.com/static/images/dlmanga.gif\" class=\"inlineimg\" border=\"0\" alt=\"(?<Name>[^\"]+)\" /></a></td>"; var chaps = parser.ParseGroup(regEx, input, "Name", "Value"); progress.Report(100); return chaps; }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); progress.Report(0); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga); var chaps = parser.ParseGroup("<a class=\"color_0077\" href=\"(?<Value>http://[^\"]+)\"[^<]+>(?<Name>[^<]+)</a>", input, "Name", "Value"); progress.Report(100); return(chaps); }
public override async Task<IEnumerable<Chapter>> FindChapters(string manga, IProgress<int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); progress.Report(0); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga, cancellationToken); var chaps = parser.ParseGroup("<a href=\"(?<Value>[^\"]+)\">(?<Name>[^<]+)</a> :", input, "Name", "Value"); // reverse chapters order and remove duplicated chapters in latest section chaps = chaps.Reverse().GroupBy(x => x.Url).Select(g => g.First()).ToList(); // transform pages link chaps = chaps.Select(c => new Chapter(c.Name, new Uri(new Uri(manga), c.Url).AbsoluteUri)).ToList(); progress.Report(100); return chaps; }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); progress.Report(0); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga); var chaps = parser.ParseGroup("<td>\n<a href=\"(?=/Manga/)(?<Value>.[^\"]*)\" title=\"(?<Name>.[^\"]*)\"", input, "Name", "Value"); chaps = chaps.Select(c => NameResolver(c.Name, c.Url, new Uri(manga))); progress.Report(100); return(chaps); }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { Logger.Info($@"> FindChapters(): {manga}"); progress.Report(0); var downloader = new DownloadService(); var parser = new ParserHelper(); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga, cancellationToken); var chaps = parser.ParseGroup("<a href=\"(?<Value>[^\"]+)\" title=\"(|[^\"]+)\" class=\"tips\">(?<Name>[^<]+)</a>", input, "Name", "Value"); progress.Report(100); return(chaps); }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); progress.Report(0); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga, cancellationToken); string regEx = "<td><a href=\"(?<Value>http://readms.net/r/[^\"]+)\">(?<Name>[^<]+)</a>"; var chaps = parser.ParseGroup(regEx, input, "Name", "Value"); progress.Report(100); return(chaps); }
public override async Task <IEnumerable <string> > FindImages(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); string input = await downloader.DownloadStringAsync(chapter.Url); var encryptPages = parser.Parse("lstImages.push\\(wrapKA\\(\"(?<Value>.[^\"]*)\"\\)\\)", input, "Value"); var pages = encryptPages.Select(e => _decryptor.DecryptFromBase64(e)); // transform pages link pages = pages.Select(p => { var value = new Uri(new Uri(chapter.Url), p).AbsoluteUri; return(value); }).ToList(); return(pages); }
public override async Task <IEnumerable <Chapter> > FindChapters(string manga, IProgress <int> progress, CancellationToken cancellationToken) { progress.Report(0); var downloader = new DownloadService { Cookies = LoginBatoto(_username, _password), Referer = "http://bato.to/reader" }; var parser = new ParserHelper(); // find all chapters in a manga string input = await downloader.DownloadStringAsync(manga); var chaps = parser.ParseGroup("<a href=\"(?<Value>http://bato.to/reader#[^\"]+)\" title=\"(?<Name>[^|]+)", input, "Name", "Value"); progress.Report(100); return(chaps); }
public async Task MangaStream_Test() { string url = "http://mangastream.com/manga/dragon_ball_super"; var service = FrameworkProvider.GetService(url); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("001 - The God of Destruction's Prophetic Dream", chapter.Name); Assert.AreEqual("http://readms.net/r/dragon_ball_super/001/2831/1", chapter.Url); var images = await service.FindImages(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(17, images.Count()); Assert.IsTrue(images.ToArray()[0].StartsWith("http://img.readms.net/cdn/manga/107/2831/001.jpg")); Assert.IsTrue(images.ToArray()[1].StartsWith("http://img.readms.net/cdn/manga/107/2831/001a.jpg")); Assert.IsTrue(images.ToArray()[2].StartsWith("http://img.readms.net/cdn/manga/107/2831/002.png")); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0], _source.Token); Assert.IsNotNull(imageString, "Cannot download image!"); }
public async Task MangaHere_Test() { string url = "http://www.mangahere.co/manga/the_god_of_high_school/"; var service = FrameworkProvider.GetService(url); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("The God Of High School 1", chapter.Name); Assert.AreEqual("http://www.mangahere.co/manga/the_god_of_high_school/c001/", chapter.Url); var images = await service.FindImanges(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(55, images.Count()); Assert.IsTrue(images.ToArray()[0].StartsWith("http://h.mhcdn.net/store/manga/9275/001.0/compressed/m001.01.jpg")); Assert.IsTrue(images.ToArray()[1].StartsWith("http://h.mhcdn.net/store/manga/9275/001.0/compressed/m001.02.jpg")); Assert.IsTrue(images.ToArray()[54].StartsWith("http://h.mhcdn.net/store/manga/9275/001.0/compressed/m001.55.jpg")); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0]); Assert.IsNotNull(imageString, "Cannot download image!"); }
public async Task KissManga_Test() { string url = "http://kissmanga.com/Manga/Onepunch-Man"; var service = FrameworkProvider.GetService(url); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("Onepunch-Man _vol.001 ch.001", chapter.Name); Assert.AreEqual("http://kissmanga.com/Manga/Onepunch-Man/vol-001-ch-001?id=313725", chapter.Url); var images = await service.FindImanges(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(19, images.Count()); Assert.IsTrue(images.ToArray()[0].StartsWith("https://images1-focus-opensocial.googleusercontent.com/gadgets/proxy?container=focus&gadget=a&no_expand=1&resize_h=0&rewriteMime=image%2F*&url=http%3a%2f%2f2.p.mpcdn.net%2f50%2f531513%2f1.jpg&imgmax=30000")); Assert.IsTrue(images.ToArray()[1].StartsWith("https://images1-focus-opensocial.googleusercontent.com/gadgets/proxy?container=focus&gadget=a&no_expand=1&resize_h=0&rewriteMime=image%2F*&url=http%3a%2f%2f2.p.mpcdn.net%2f50%2f531513%2f2.jpg&imgmax=30000")); Assert.IsTrue(images.ToArray()[2].StartsWith("https://images1-focus-opensocial.googleusercontent.com/gadgets/proxy?container=focus&gadget=a&no_expand=1&resize_h=0&rewriteMime=image%2F*&url=http%3a%2f%2f2.p.mpcdn.net%2f50%2f531513%2f3.jpg&imgmax=30000")); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0]); Assert.IsNotNull(imageString, "Cannot download image!"); }
public async Task MangaShare_Test() { string url = "http://read.mangashare.com/Gantz"; var service = FrameworkProvider.GetService(url); var chapters = await service.FindChapters(url, new Progress <int>(), _source.Token); Assert.IsTrue(chapters.Any(), "Cannot find chapters."); var chapter = chapters.Last(); Assert.AreEqual("Gantz 1", chapter.Name); Assert.AreEqual("http://read.mangashare.com/Gantz/chapter-001/page001.html", chapter.Url); var images = await service.FindImanges(chapter, new Progress <int>(), _source.Token); Assert.AreEqual(43, images.Count()); Assert.AreEqual("http://dl01.mangashare.com/manga/Gantz/001/001.jpg", images.ToArray()[0]); Assert.AreEqual("http://dl01.mangashare.com/manga/Gantz/001/002.jpg", images.ToArray()[1]); Assert.AreEqual("http://dl01.mangashare.com/manga/Gantz/001/043.jpg", images.ToArray()[42]); var downloader = new DownloadService(); string imageString = await downloader.DownloadStringAsync(images.ToArray()[0]); Assert.IsNotNull(imageString, "Cannot download image!"); }
public override async Task <IEnumerable <string> > FindImages(Chapter chapter, IProgress <int> progress, CancellationToken cancellationToken) { var downloader = new DownloadService(); var parser = new ParserHelper(); string input = await downloader.DownloadStringAsync(chapter.Url, cancellationToken); /// Could be secured against changes by capturing the script's path as it exists in the live document instead of assuming the location. string pattern = "<script\\s+(type=[\"']text/javascript[\"'])?\\s+(src=[\"']/Scripts/{0}[\"'])>"; string concatedPattern = string.Concat(string.Format(pattern, "ca.js"), "|", string.Format(pattern, "lo.js")); if (Regex.IsMatch(input, concatedPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)) { string funcUri = "http://kissmanga.com/Scripts/lo.js"; string decryptFunc = await downloader.DownloadStringAsync(funcUri, cancellationToken); /// Execute CryptoJS from saved resources to reduce HTTP requests. _engine.Execute(Properties.Resources.CryptoJs); /// Execute the decryption function to allow it to be called later. _engine.Execute(decryptFunc); var keysPattern = "<script type=\"text/javascript\">[\\s]*(?<Value>.*)(?!</script>)"; var regex = new Regex(keysPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); var keys = string.Empty; foreach (Match match in regex.Matches(input)) { if (match.Value.Contains("CryptoJS")) { keys = match.Groups["Value"].Value; break; } } if (string.IsNullOrWhiteSpace(keys)) { throw new ArgumentException("Cannot decrypt image URIs."); } else { _engine.Execute(keys); } /// As with the script locations, to avoid unnecessary breaking the application, the function name could be captured and invoked /// in the event it changes. var encryptPages = parser.Parse("lstImages.push\\(wrapKA\\(\"(?<Value>.[^\"]*)\"\\)\\)", input, "Value"); var pages = encryptPages.Select(e => { string value = string.Empty; try { value = _engine.CallGlobalFunction <string>("wrapKA", e); } catch (Exception ex) { _logger.Fatal(ex); throw; } return(value); }); pages = pages.Select(p => { var value = new Uri(new Uri(chapter.Url), p).AbsoluteUri; return(value); }).ToList(); return(pages); } return(null); }