public async Task <List <ImageModel> > FindImageUrls(ImagePageModel imagePageModel)
{
string html = await HttpDownloader.DownloadHtmlPage(imagePageModel.ImagePageUrl);
IBrowsingContext context = BrowsingContext.New();
IDocument document = await context.OpenAsync(response => response.Content(html));
List <ImageModel> imageModels = new List <ImageModel>();
IElement titleElement = document.QuerySelector("title");
string albumTitle = titleElement?.InnerHtml;
IElement divElement = document.QuerySelector("div.picture");
var imageElements = divElement.QuerySelectorAll("img");
if (imageElements == null || imageElements.Count() == 0)
{
Debug.WriteLine("在图片页面hmtl中没有匹配到图片url元素: " + imagePageModel.ImagePageUrl);
throw new Exception("在图片页面hmtl中没有匹配到图片url元素");
}
string pageNum = "";
IElement pageNumElement = document.QuerySelector("div.paging");
pageNum = pageNumElement?.QuerySelector("b")?.InnerHtml;
int index = 0;
foreach (IElement imgElement in imageElements)
{
string imageName;
string url = imgElement.GetAttribute("src");
if (index > 0)
{
imageName = albumTitle + "_" + pageNum + "_" + index;
}
else
{
imageName = albumTitle + "_" + pageNum;
}
ImageModel imageModel = new ImageModel(imagePageModel, imageName, url);
index++;
Debug.WriteLine("成功:找到图片" + imageModel.ImageUrl + imageModel.ImageName + imageModel.ImagePage.ImagePageUrl);
imageModels.Add(imageModel);
}
return(imageModels);
}
//static string albumTitle;
public async Task <List <ImagePageModel> > FindImagePageUrl(TaskItem taskItem)
{
//List<string> urls = new List<string>();
string html;
try
{
html = await HttpDownloader.DownloadHtmlPage(taskItem.Url);
}
catch
{
throw;
}
IBrowsingContext context = BrowsingContext.New();
IDocument document = await context.OpenAsync(response => response.Content(html));
IElement pageNumElement = document.QuerySelector("div.paging");
if (pageNumElement == null)
{
throw new TargetNotFindException("无法在HTML中找到div.paging");
}
string pageNumString;
try
{
pageNumString = pageNumElement.QuerySelector("a").InnerHtml;
}
catch
{
throw new TargetNotFindException("无法在HTML中找到a标签");
}
Match matchNum = Regex.Match(pageNumString, "[0-9]+");
int maxPageNum = Int32.Parse(matchNum.Value);
string generalUrl = taskItem.Url.Remove(taskItem.Url.Length - 6);
List <ImagePageModel> imagePageModels = new List <ImagePageModel>();
for (int i = 0; i < maxPageNum; i++)
{
string url = generalUrl + i + ".html";
imagePageModels.Add(new ImagePageModel(url, taskItem));
}
return(imagePageModels);
}
public async Task <List <EbookPageModel> > FindEbookPage(string url)
{
string html;
try
{
html = await HttpDownloader.DownloadHtmlPage(url);
}
catch
{
throw;
}
IBrowsingContext context = BrowsingContext.New();
IDocument document = await context.OpenAsync(response => response.Content(html));
IEnumerable <IElement> urlElements = document.QuerySelectorAll("h2.entry-title");
if (urlElements.Count() == 0)
{
throw new TargetNotFindException("无法在HTML中找到h2.entry-title");
}
List <EbookPageModel> webPageModels = new List <EbookPageModel>();
try
{
foreach (IElement element in urlElements)
{
string bookPageUrl = element.QuerySelector("a").GetAttribute("href");
webPageModels.Add(new EbookPageModel(bookPageUrl));
Debug.WriteLine("成功:找到pageurl=" + bookPageUrl);
}
}
catch
{
throw new TargetNotFindException("无法在HTML中找到a");
}
return(webPageModels);
}
public async Task <EbookPageModel> FindEbookUrl(EbookPageModel webPageModel)
{
string html;
try
{
html = await HttpDownloader.DownloadHtmlPage(webPageModel.Url);
}
catch
{
throw;
}
IBrowsingContext context = BrowsingContext.New();
IDocument document = await context.OpenAsync(response => response.Content(html));
IEnumerable <IElement> downloadLinkElements = document.QuerySelectorAll("span.download-links");
if (downloadLinkElements.Count() == 0)
{
throw new TargetNotFindException("无法在HTML中找到下载链接");
}
IElement detail = document.QuerySelector("div.book-detail");
IEnumerable <IElement> terms = detail.QuerySelectorAll("dd");
IEnumerable <IElement> authorsElements = terms.ElementAt(0).QuerySelectorAll("a");
webPageModel.Author = string.Join(";", authorsElements.Select(x => x.InnerHtml));
webPageModel.ISBN = terms.ElementAt(1).InnerHtml;
webPageModel.Year = Int32.Parse(terms.ElementAt(2).InnerHtml);
webPageModel.Pages = Int32.Parse(terms.ElementAt(3).InnerHtml);
webPageModel.Language = terms.ElementAt(4).InnerHtml;
webPageModel.FileSize = terms.ElementAt(5).InnerHtml;
//webPageModel.FileFormat = terms.ElementAt(6).InnerHtml;
IEnumerable <IElement> tagElements = terms.ElementAt(7).QuerySelectorAll("a");
webPageModel.Category = string.Join(";", tagElements.Select(x => x.InnerHtml));
webPageModel.Description = document.QuerySelector("div.entry-content").InnerHtml;
webPageModel.Title = document.QuerySelector("h1.single-title").InnerHtml;
IElement subTitleElement = document.QuerySelector("h4");
if (subTitleElement != null)
{
webPageModel.SubTitle = subTitleElement.InnerHtml;
}
IElement imageElement = document.QuerySelector("header.entry-header");
IElement imageElement1 = imageElement.QuerySelector("img");
string imageUrl = imageElement1.GetAttribute("src");
string fileUniqueName = TimeHelper.GetTimeStamp();
webPageModel.Image = new EbookImageModel(fileUniqueName, imageUrl);
List <EbookFileModel> ebooks = new List <EbookFileModel>();
int i = 1;
foreach (Element downloadLinkElement in downloadLinkElements)
{
string url = downloadLinkElement.QuerySelector("a").GetAttribute("href");
string fileName = fileUniqueName + "_" + i;
ebooks.Add(new EbookFileModel(webPageModel, url, fileName));
i++;
//Debug.WriteLine("fileName=" + fileName);
}
webPageModel.EBooks = ebooks;
return(webPageModel);
}