protected override Task Parse(DataContext context)
{
var newsList = context.Selectable.SelectList(Selectors.XPath(".//div[@class='news_block']"));
foreach (var news in newsList)
{
var title = news.Select(Selectors.XPath(".//h2[@class='news_entry']"))?.Value;
var url = news.Select(Selectors.XPath(".//h2[@class='news_entry']/a/@href"))?.Value;
var summary = news.Select(Selectors.XPath(".//div[@class='entry_summary']"))?.Value;
var views = news.Select(Selectors.XPath(".//span[@class='view']"))?.Value.Replace(" 人浏览", "");
if (!string.IsNullOrWhiteSpace(url))
{
var request = context.CreateNewRequest(url);
request.SetProperty("title", title);
request.SetProperty("url", url);
request.SetProperty("summary", summary);
request.SetProperty("views", views);
context.AddFollowRequests(request);
}
}
return(Task.CompletedTask);
}
/// <summary>
/// 数据解析
/// </summary>
/// <param name="context">处理上下文</param>
/// <returns></returns>
public override async Task HandleAsync(DataContext context)
{
context.NotNull(nameof(context));
context.Response.NotNull(nameof(context.Response));
if (!IsValidRequest(context.Request))
{
Logger.LogInformation($"{GetType().Name} ignore request {context.Request.RequestUri}");
return;
}
var request = context.Request;
if (context.Selectable == null)
{
if (SelectableBuilder != null)
{
context.Selectable = SelectableBuilder(context);
}
else
{
var text = context.Response.ReadAsString().TrimStart();
if (text.StartsWith("<!DOCTYPE html>") || text.StartsWith("<html>"))
{
context.Selectable = GetHtmlSelectable(context, text);
}
else
{
try
{
var token = (JObject)JsonConvert.DeserializeObject(text);
context.Selectable = new JsonSelectable(token);
}
catch
{
context.Selectable = new TextSelectable(text);
}
}
}
}
await Parse(context);
var requests = new List <Request>();
if (_followRequestQueriers != null)
{
foreach (var followRequestQuerier in _followRequestQueriers)
{
var followRequests = followRequestQuerier(context);
if (followRequests != null)
{
requests.AddRange(followRequests);
}
}
}
foreach (var followRequest in requests)
{
if (IsValidRequest(followRequest))
{
// 在此强制设制 Owner, 防止用户忘记导致出错
followRequest.Owner = request.Owner;
followRequest.Agent = context.Response.Agent;
context.AddFollowRequests(followRequest);
}
}
}
