public override async Task <WebNovelChapter> GetChapterAsync(ChapterLink link,
ChapterRetrievalOptions options = default(ChapterRetrievalOptions),
CancellationToken token = default(CancellationToken))
{
string content = await GetWebPageAsync(link.Url, token);
IHtmlDocument doc = await Parser.ParseAsync(content, token);
var paged = GetPagedChapterUrls(doc.DocumentElement);
WebNovelChapter chapter = ParseChapter(doc.DocumentElement, token);
if (chapter == null)
{
return(null);
}
chapter.Url = link.Url;
chapter.NextChapterUrl = UrlHelper.ToAbsoluteUrl(link.Url, chapter.NextChapterUrl);
foreach (var page in paged)
{
string pageContent = await GetWebPageAsync(page, token);
IHtmlDocument pageDoc = await Parser.ParseAsync(pageContent, token);
chapter.Content += ParseChapter(pageDoc.DocumentElement, token).Content;
}
return(chapter);
}
public override async Task <WebNovelChapter> GetChapterAsync(
ChapterLink link,
ChapterRetrievalOptions options = default(ChapterRetrievalOptions),
CancellationToken token = default(CancellationToken))
{
string baseContent = await GetWebPageAsync(link.Url, token);
IHtmlDocument doc = await Parser.ParseAsync(baseContent, token);
IElement titleElement = doc.DocumentElement.FirstWhereHasClass(TitleClasses);
WebNovelChapter chapter = ParseChapter(doc, link.Url, doc.DocumentElement, token);
chapter.Url = link.Url;
if (titleElement != null)
{
chapter.ChapterName = titleElement.Text().Trim();
}
return(chapter);
}
private WebNovelChapter ParseChapter(IDocument doc, string baseUrl, IElement rootElement, CancellationToken token = default(CancellationToken))
{
WebNovelChapter chapter = new WebNovelChapter();
IElement element = rootElement.WhereHasClass(PostClasses).LastOrDefault()
?? rootElement.Descendents <IElement>().FirstOrDefault(p => p.LocalName == "article");
if (element != null)
{
RemoveBloat(element);
}
IElement nextChapterElement = (from e in rootElement?.Descendents <IElement>() ?? new List <IElement>()
where e.LocalName == "a"
let text = e.Text()
let a = NextChapterNames.FirstOrDefault(p => text.IndexOf(p, StringComparison.OrdinalIgnoreCase) >= 0)
where a != null
orderby NextChapterNames.IndexOf(a)
select e).FirstOrDefault();
if (nextChapterElement != null)
{
chapter.NextChapterUrl = nextChapterElement.GetAttribute("href");
}
if (element != null)
{
chapter.Content = new ContentCleanup(baseUrl).Execute(doc, element);
}
else
{
chapter.Content = "No Content";
}
return(chapter);
}
protected virtual WebNovelChapter ParseChapter(IElement rootElement, CancellationToken token = default(CancellationToken))
{
IElement articleElement = rootElement.Descendents <IElement>().FirstOrDefault(p => p.LocalName == "article");
IElement element = rootElement.FirstWhereHasClass(PostClasses) ?? articleElement;
if (element != null)
{
RemoveBloat(element);
}
IElement chapterNameElement = rootElement.FirstWhereHasClass(TitleClasses);
if (element != null && chapterNameElement == null)
{
chapterNameElement = (from e in element.Descendents <IElement>()
where e.LocalName == "h1" || e.LocalName == "h2" ||
e.LocalName == "h3" || e.LocalName == "h4"
select e).FirstOrDefault();
}
else
{
IElement chNameLinkElement = (from e in chapterNameElement.Descendents <IElement>()
where e.LocalName == "a"
select e).FirstOrDefault();
if (chNameLinkElement != null)
{
chapterNameElement = chNameLinkElement;
}
}
IElement nextChapterElement = (from e in articleElement?.Descendents <IElement>() ?? rootElement.Descendents <IElement>()
where e.LocalName == "a"
let text = e.Text()
let a = NextChapterNames.FirstOrDefault(p => text.IndexOf(p, StringComparison.OrdinalIgnoreCase) >= 0)
where a != null || (e.HasAttribute("rel") && e.GetAttribute("rel") == "next")
let index = NextChapterNames.IndexOf(a)
let o = index >= 0 ? index : int.MaxValue
orderby o
select e).FirstOrDefault();
WebNovelChapter chapter = new WebNovelChapter();
if (nextChapterElement != null)
{
chapter.NextChapterUrl = nextChapterElement.GetAttribute("href");
}
if (element != null)
{
RemoveNavigation(element);
RemoveScriptStyleElements(element);
chapter.ChapterName = chapterNameElement?.Text()?.Trim();
chapter.Content = element.InnerHtml;
}
else
{
chapter.Content = "No Content";
}
return(chapter);
}
private async void convertBackgroundWorker_DoWork(object sender, DoWorkEventArgs e)
{
var items = new List <object>();
string type = string.Empty;
string mode = string.Empty;
Invoke((MethodInvoker) delegate
{
type = ((string)websiteTypeComboBox.SelectedItem).ToLower();
mode = ((string)modeComboBox.SelectedItem).ToLower();
items.AddRange(chaptersListBox.Items.Cast <object>());
});
EBook book = new EBook
{
Title = titleTextBox.Text,
CoverImage = coverTextBox.Text
};
foreach (object obj in items)
{
if (obj is ChapterLink)
{
ChapterLink link = (ChapterLink)obj;
try
{
WebNovelSource source = GetSource(link.Url, type);
WebNovelChapter chapter = await source.GetChapterAsync(link);
if (chapter == null)
{
WriteText($"Failed to process {link.Name}!", Color.Red);
}
else
{
book.Chapters.Add(new Chapter {
Name = link.Name, Content = chapter.Content
});
WriteText($"{link.Name} has been processed.", Color.Green);
}
}
catch (Exception ex)
{
WriteText($"Failed to process {link.Name}!", Color.Red);
WriteText($"ERROR: {ex}", Color.Red);
}
if (Settings.Default.DelayPerChapter > 0)
{
await Task.Delay(TimeSpan.FromSeconds(Settings.Default.DelayPerChapter));
}
}
else if (obj is WebNovelChapter)
{
WebNovelChapter wn = (WebNovelChapter)obj;
book.Chapters.Add(new Chapter {
Name = wn.ChapterName, Content = wn.Content
});
}
}
WriteText("Generating epub...");
try
{
await book.GenerateEpubAsync(e.Argument.ToString());
}
catch (Exception ex)
{
WriteText("Error generating Epub", Color.Red);
WriteText($"ERROR: {ex}", Color.Red);
}
WriteText("Done!", Color.Green);
Invoke((MethodInvoker) delegate
{
progressBar.Visible = false;
convertButton.Enabled = true;
});
}
