public void Process(Page page) { foreach (ExtractRule extractRule in _extractRules) { if (extractRule.IsMulti) { IList<string> results = page.GetHtml().SelectDocumentForList(extractRule.Selector); if (extractRule.IsNotNull && results.Count == 0) { page.SetSkip(true); } else { page.GetResultItems().Put(extractRule.FieldName, results); } } else { string result = page.GetHtml().SelectDocument(extractRule.Selector); if (extractRule.IsNotNull && result == null) { page.SetSkip(true); } else { page.GetResultItems().Put(extractRule.FieldName, result); } } } }
public void Process(Page page) { IList<String> links = page.GetHtml().Links().Regex("http://my\\.oschina\\.net/flashsword/blog/\\d+").GetAll(); page.AddTargetRequests(links); page.PutField("title", page.GetHtml().XPath("//div[@class='BlogEntity']/div[@class='BlogTitle']/h1/text()").ToString()); page.PutField("content", page.GetHtml().XPath("//div[@class='BlogContent']/tidyText()").ToString()); page.PutField("tags", page.GetHtml().XPath("//div[@class='BlogTags']/a/text()").GetAll()); page.PutField("artical", page.GetHtml().XPath("//*[@Class='Blog']/div[1]/div/h2/a").ToString()); }
/// <summary> /// ����Ҳ�������URL, ��Ȼ���ص�URL̫�� /// </summary> /// <param name="page"></param> /// <param name="urlRegionSelector"></param> /// <param name="urlPatterns"></param> private void ExtractLinks(Page page, ISelector urlRegionSelector, IList<Regex> urlPatterns) { var links = urlRegionSelector == null ? new List<string>() : page.GetHtml().SelectList(urlRegionSelector).Links().GetAll(); if (urlPatterns == null || urlPatterns.Count == 0) { page.AddTargetRequests(links); return; } foreach (Regex targetUrlPattern in urlPatterns) { foreach (string link in links) { if (targetUrlPattern.IsMatch(link)) { page.AddTargetRequest(new Request(link, page.GetRequest().Extras)); } } } }