/// <summary>
/// 构造方法
/// </summary>
/// <param name="partterns">匹配目标链接的正则表达式</param>
/// <param name="excludeParterns">排除目标链接的正则表达式</param>
public DefaultPageProcessor(string[] partterns = null, string[] excludeParterns = null)
{
var targetUrlsExtractor = new RegionAndPatternTargetRequestExtractor();
if (partterns != null && partterns.Length > 0)
{
targetUrlsExtractor.AddTargetUrlExtractor(".", partterns);
}
if (excludeParterns != null && excludeParterns.Length > 0)
{
targetUrlsExtractor.AddExcludeTargetUrlPatterns(excludeParterns);
}
TargetUrlsExtractor = targetUrlsExtractor;
}
public void RegionAndPatternTargetUrlsExtractor()
{
HttpClient client = new HttpClient();
var html = client.GetStringAsync("http://www.cnblogs.com").Result;
var extracotr = new RegionAndPatternTargetRequestExtractor(".//div[@class='pager']", "/sitehome/p/\\d+", "^http://www\\.cnblogs\\.com/$");
var page = new Page(new Request("http://cnblogs.com"));
page.Content = html;
page.ContentType = ContentType.Html;
var requets = Enumerable.ToList(extracotr.ExtractRequests(page));
Assert.Equal(11, requets.Count);
Assert.Contains(requets, r => r.Url == "http://cnblogs.com/sitehome/p/2");
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="model">数据模型</param>
/// <param name="extractor">模型解析器</param>
/// <param name="targetRequestExtractor">目标链接的解析器</param>
/// <param name="dataHandlers">数据处理器</param>
public ModelProcessor(IModel model, IModelExtractor extractor = null, ITargetRequestExtractor targetRequestExtractor = null,
params IDataHandler[] dataHandlers)
{
Model = model ?? throw new ArgumentNullException(nameof(model));
Extractor = extractor ?? new ModelExtractor();
if (targetRequestExtractor != null)
{
TargetUrlsExtractor = targetRequestExtractor;
}
RegionAndPatternTargetRequestExtractor regionAndPatternTargetUrlsExtractor;
if (TargetUrlsExtractor == null)
{
regionAndPatternTargetUrlsExtractor = new RegionAndPatternTargetRequestExtractor();
TargetUrlsExtractor = regionAndPatternTargetUrlsExtractor;
}
else
{
regionAndPatternTargetUrlsExtractor = TargetUrlsExtractor as RegionAndPatternTargetRequestExtractor;
}
if (regionAndPatternTargetUrlsExtractor == null)
{
return;
}
if (Model.TargetRequestSelectors != null && Model.TargetRequestSelectors.Any())
{
foreach (var targetUrlsSelector in Model.TargetRequestSelectors)
{
var patterns = targetUrlsSelector.Patterns?.Select(x => x?.Trim()).Distinct().ToArray();
var xpaths = targetUrlsSelector.XPaths?.Select(x => x?.Trim()).Distinct().ToList();
if (xpaths == null && patterns == null)
{
throw new SpiderException("Region xpath and patterns should not be null both");
}
if (xpaths != null && xpaths.Count > 0)
{
foreach (var xpath in xpaths)
{
regionAndPatternTargetUrlsExtractor.AddTargetUrlExtractor(xpath, patterns);
}
}
else
{
if (patterns != null && patterns.Length > 0)
{
regionAndPatternTargetUrlsExtractor.AddTargetUrlExtractor(null, patterns);
}
}
if (targetUrlsSelector.ExcludePatterns != null)
{
foreach (var p in targetUrlsSelector.ExcludePatterns)
{
regionAndPatternTargetUrlsExtractor.ExcludeTargetUrlPatterns.Add(new Regex(p));
}
}
}
}
if (dataHandlers != null)
{
foreach (var datahandler in dataHandlers)
{
if (datahandler != null)
{
_dataHandlers.Add(datahandler);
}
}
}
}
public NewsProcessor()
{
TargetUrlsExtractor = new RegionAndPatternTargetRequestExtractor(null, "^http://www\\.cnblogs\\.com/$", "^http://www\\.cnblogs\\.com/news/$", "www\\.cnblogs\\.com/news/\\d+");
}
public BlogSumaryProcessor()
{
// 定义目标页的筛选
TargetUrlsExtractor = new RegionAndPatternTargetRequestExtractor(".", "^http://www\\.cnblogs\\.com/$", "http://www\\.cnblogs\\.com/sitehome/p/\\d+");
}
