public object FeedJob(string pages) { try { var node = ServerManager.Get(Request.RequestUri.Authority); if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY) { var ps = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray(); var feeds = FeedLiteDb.GetFeedModels(ps, 50); feeds.RemoveAll(m => m.Status == FeedStatus.OFF); var compile = new CompileFeedAddress(); foreach (var feed in feeds) { feed.Address = compile.Compile(feed.Address); } return(feeds); } } catch { } return(new { }); }
public object Run(object t, ParallelTask task) { var model = t as CrawlTaskModel; var results = new List <ExtractResult>(); var reporter = task.Progress as IProgress <string>; reporter.Report("正在读取Feed记录"); var feed = FeedLiteDb.GetFeed(model.FeedId); reporter.Report("正在下载 Feed"); var compile = new CompileFeedAddress(); feed.Address = compile.Compile(feed.Address); var job = new FeedJob(); var snap = job.DoTask(feed, false); reporter.Report("Feed 下载完成"); var block = RuiJiExpression.ParserBlock(feed.RuiJiExpression); var feedResult = RuiJiExtracter.Extract(snap.Content, block); results.Add(feedResult); reporter.Report("正在提取Feed地址"); var j = new FeedExtractJob(); var urls = j.ExtractAddress(snap); reporter.Report("Feed地址提取完成"); foreach (var url in urls) { reporter.Report("正在提取地址 " + url); var r = ContentQueue.Instance.Extract(url); results.AddRange(r); } reporter.Report("计算完成"); if (!model.IncludeContent) { results.ForEach((m) => { ClearContent(m); }); } return(results); }