public void GetListChapters(string url)
{
if (string.IsNullOrEmpty(url))
{
return;
}
if (File.Exists(Path.Combine(Cfg.StoryDirectory, "chapters.json")))
{
return;
}
List <LinkPage> linkPages = new List <LinkPage>();
linkPages.Add(new LinkPage()
{
Link = url
});
while (linkPages.Any(o => !o.Processed))
{
var page = linkPages.FirstOrDefault(o => !o.Processed);
//ClearCurrentConsoleLine();
//using (var wc = new WebClient())
if (page != null)
{
Cfg.LogQueue.Enqueue($"{page.Link}");
try
{
//wc.Headers.Add("User-Agent", Cfg.StrUserAgent);
//if (url != null)
{
//var bytes = wc.DownloadData(url);
//var html = Encoding.UTF8.GetString(bytes);
var html = Util.DownloadUrl(page.Link);
var parser = new HtmlParser();
var doc = parser.Parse(html);
var chaps = doc.QuerySelectorAll(".list-chapter")
.Where(o =>
{
var f = o.QuerySelector("h2");
if (f != null)
{
return(Regex.Match(f.TextContent, "Danh sách chương", RegexOptions.IgnoreCase)
.Success);
}
return(false);
})
.Select(o => o.QuerySelector("ul"))
.SelectMany(o => o.QuerySelectorAll("a")
.Select(p => new ChapterInfo()
{
Title = p.Text(),
Link = p.Attributes["href"]?.Value
}).ToList())
.Where(o => !string.IsNullOrEmpty(o.Link))
.ToList();
AllStories.AddRange(chaps);
// get chapters link
// get next list chapters
var foundUrls = doc
.QuerySelectorAll(".pagination > ul > li > a")
.Where(o =>
{
int.TryParse(o.Text(), out var i);
return(i > 0);
})
.Select(o => o.Attributes["href"]?.Value)
.Where(o => !string.IsNullOrEmpty(o))
.ToList()
.Except(linkPages.Select(o => o.Link).ToList())
.Select(o => new LinkPage()
{
Link = o
})
.ToList();
linkPages.AddRange(foundUrls);
}
page.Processed = true;
}
catch (Exception ex)
{
Cfg.LogQueue.Enqueue($"download error:{ex.StackTrace}");
Task.Delay(1000).Wait();
}
}
}
}
public void GetListChapters(string url)
{
if (string.IsNullOrEmpty(url))
{
return;
}
if (File.Exists(Path.Combine(Cfg.StoryDirectory, "chapters.json")))
{
return;
}
var found = true;
while (found)
{
//ClearCurrentConsoleLine();
var url2 = url;
Cfg.LogQueue.Enqueue($"{url2}");
using (var wc = new WebClient())
{
try
{
wc.Headers.Add("User-Agent", Cfg.StrUserAgent);
if (url != null)
{
var bytes = wc.DownloadData(url);
var html = Encoding.UTF8.GetString(bytes);
var parser = new HtmlParser();
var doc = parser.Parse(html);
var chaps = doc.QuerySelectorAll("#list-chapter")
.SelectMany(o => o.QuerySelectorAll("ul")
.Where(p => p.Attributes["class"]?.Value?.ToString()
.Contains("list-chapter") ??
false)
.ToList()
)
.Where(o => o != null)
.SelectMany(o => o.QuerySelectorAll("a")
.Select(p => new ChapterInfo()
{
Title = p.Text(),
Link = p.Attributes["href"]?.Value
}).ToList())
.Where(o => !string.IsNullOrEmpty(o.Link))
.ToList();
AllStories.AddRange(chaps);
// get chapters link
// get next list chapters
url = doc
.QuerySelectorAll(".pagination > li")
.ToList()
.Where(o => o.QuerySelectorAll(".glyphicon-menu-right").Length > 0)
.Select(o => o.QuerySelector("a"))
.Select(o => o.Attributes["href"]?.Value.Split('#').FirstOrDefault())
.FirstOrDefault(o => o != null);
}
found = !string.IsNullOrEmpty(url);
}
catch (Exception ex)
{
Cfg.LogQueue.Enqueue($"Error {url} -{ex.StackTrace}");
//Task.Delay(1000).Wait();
}
}
}
}