public bool FeedStatusChange(string ids, string status)
{
var changeIds = ids.Split(',').Select(i => Convert.ToInt32(i)).ToArray();
var statusEnum = (Status)Enum.Parse(typeof(Status), status.ToUpper());
return(FeedLiteDb.StatusChange(changeIds, statusEnum));
}
public bool ChangeFeedStatus(string ids, string status)
{
var changeIds = ids.Split(',').Select(i => Convert.ToInt32(i)).ToArray();
var statusEnum = (Status)Enum.Parse(typeof(Status), status.ToUpper());
if (statusEnum == Status.ON)
{
var feeds = FeedLiteDb.GetFeed(changeIds);
foreach (var feed in feeds)
{
var @event = new BroadcastEvent()
{
Event = BroadcastEventEnum.UPDATE,
Args = feed
};
Broadcast(@event);
}
}
else
{
var @event = new BroadcastEvent()
{
Event = BroadcastEventEnum.REMOVE,
Args = changeIds
};
Broadcast(@event);
}
return(FeedLiteDb.ChangeStatus(changeIds, statusEnum));
}
public object FeedJob(string pages)
{
try
{
var node = ServerManager.Get(Request.RequestUri.Authority);
if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
{
var ps = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
var feeds = FeedLiteDb.GetFeedModels(ps, 50);
feeds.RemoveAll(m => m.Status == FeedStatus.OFF);
var compile = new CompileFeedAddress();
foreach (var feed in feeds)
{
feed.Address = compile.Compile(feed.Address);
}
return(feeds);
}
}
catch { }
return(new { });
}
public void TestHistoryUpdate()
{
var feed = FeedLiteDb.GetFeed(1);
var job = new FeedExtractJob();
job.DoTask(@"D:\云同步\vcoded\RuiJi.Net\RuiJi.Cmd\bin\Debug\snapshot\1_636635303414097356.json");
Assert.True(true);
}
public object Run(object t, ParallelTask task)
{
var model = t as CrawlTaskModel;
var results = new List <ExtractResult>();
var reporter = task.Progress as IProgress <string>;
reporter.Report("正在读取Feed记录");
var feed = FeedLiteDb.GetFeed(model.FeedId);
reporter.Report("正在下载 Feed");
var compile = new CompileFeedAddress();
feed.Address = compile.Compile(feed.Address);
var job = new FeedJob();
var snap = job.DoTask(feed, false);
reporter.Report("Feed 下载完成");
var block = RuiJiExpression.ParserBlock(feed.RuiJiExpression);
var feedResult = RuiJiExtracter.Extract(snap.Content, block);
results.Add(feedResult);
reporter.Report("正在提取Feed地址");
var j = new FeedExtractJob();
var urls = j.ExtractAddress(snap);
reporter.Report("Feed地址提取完成");
foreach (var url in urls)
{
reporter.Report("正在提取地址 " + url);
var r = ContentQueue.Instance.Extract(url);
results.AddRange(r);
}
reporter.Report("计算完成");
if (!model.IncludeContent)
{
results.ForEach((m) =>
{
ClearContent(m);
});
}
return(results);
}
public void UpdateFeed([FromBody] FeedModel feed)
{
FeedLiteDb.AddOrUpdate(feed);
var @event = new BroadcastEvent()
{
Event = BroadcastEventEnum.UPDATE,
Args = feed
};
Broadcast(@event);
}
private List <FeedModel> GetFeedJobs(string baseUrl, string proxyUrl, FeedNode node)
{
Logger.GetLogger(baseUrl).Info("start get feed");
try
{
if (NodeConfigurationSection.Standalone)
{
var paging = new Paging();
paging.CurrentPage = _page;
paging.PageSize = 50;
var feeds = FeedLiteDb.GetAvailableFeeds(paging);
if (_page > 0 && feeds.Count == 0)
{
_page = 0;
paging.CurrentPage = _page;
feeds = FeedLiteDb.GetAvailableFeeds(paging);
}
_page++;
Logger.GetLogger(baseUrl).Info("get feed jobs:" + feeds.Count);
return(feeds);
}
else
{
var d = node.GetData("/config/feed/" + node.BaseUrl);
var config = JsonConvert.DeserializeObject <NodeConfig>(d.Data);
var pages = config.Pages == null ? "" : string.Join(",", config.Pages);
var client = new RestClient("http://" + proxyUrl);
var restRequest = new RestRequest("api/feed/job?pages=" + pages);
restRequest.Method = Method.GET;
var restResponse = client.Execute(restRequest);
var feeds = JsonConvert.DeserializeObject <List <FeedModel> >(restResponse.Content);
Logger.GetLogger(baseUrl).Info("get feed jobs:" + feeds.Count);
return(feeds);
}
}
catch (Exception ex)
{
Logger.GetLogger(baseUrl).Info("get feed error " + ex.Message);
return(new List <FeedModel>());
}
}
public object GetFeedJob(string pages)
{
try
{
var ps = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
var feeds = FeedLiteDb.GetFeedModels(ps, 50);
return(feeds);
}
catch { }
return(new { });
}
public object Feeds(int offset, int limit, string key, string method, string type, string status)
{
var paging = new Paging();
paging.CurrentPage = (offset / limit) + 1;
paging.PageSize = limit;
return(new
{
rows = FeedLiteDb.GetFeedModels(paging, key, method, type, status),
total = paging.Count
});
}
public bool RemoveFeed(string ids)
{
var removes = ids.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
var @event = new BroadcastEvent()
{
Event = BroadcastEventEnum.REMOVE,
Args = removes
};
Broadcast(@event);
return(FeedLiteDb.Remove(removes));
}
public object FeedJob(string pages)
{
try
{
var node = ServerManager.Get(Request.RequestUri.Authority);
if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
{
var ps = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
var feeds = FeedLiteDb.GetFeedModels(ps, 50);
feeds.RemoveAll(m => m.Status == FeedStatus.OFF);
return(PreProcessUrl(feeds));
}
}
catch { }
return(new { });
}
public object Feeds(int offset, int limit)
{
var node = ServerManager.Get(Request.RequestUri.Authority);
if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
{
var paging = new Paging();
paging.CurrentPage = (offset / limit) + 1;
paging.PageSize = limit;
return(new
{
rows = FeedLiteDb.GetFeedModels(paging),
total = paging.Count
});
}
return(new { });
}
public Task SyncFeed()
{
return(Task.Run(() =>
{
try
{
scheduler.Clear();
if (NodeConfigurationSection.Standalone)
{
var page = 1;
var paging = new Paging();
paging.CurrentPage = page;
paging.PageSize = 5000;
var feeds = FeedLiteDb.GetAvailableFeeds(paging);
while (feeds.Count != 0)
{
foreach (var feed in feeds)
{
AddJob(feed);
}
Logger.GetLogger(baseUrl).Info(baseUrl + " sync feed and add feed jobs:" + feeds.Count);
paging.CurrentPage = ++page;
feeds = FeedLiteDb.GetAvailableFeeds(paging);
}
}
else
{
var data = feedNode.GetData("/config/feed/" + baseUrl);
var config = JsonConvert.DeserializeObject <NodeConfig>(data.Data);
if (config.Pages == null || config.Pages.Length == 0)
{
return;
}
feedPages = config.Pages;
var feedsResponse = NodeVisitor.Feeder.GetFeedJobs(string.Join(",", config.Pages));
if (string.IsNullOrEmpty(feedsResponse))
{
throw new Exception("feedproxy can't connect");
}
var feeds = JsonConvert.DeserializeObject <List <FeedModel> >(feedsResponse);
foreach (var feed in feeds)
{
AddJob(feed);
}
Logger.GetLogger(baseUrl).Info(baseUrl + " sync feed and add feed jobs:" + feeds.Count);
}
}
catch (Exception ex)
{
Logger.GetLogger(baseUrl).Error(baseUrl + " sync feed error " + ex.Message);
}
}));
}
public object GetFeed(int id)
{
var feed = FeedLiteDb.GetFeed(id);
return(feed);
}
public void UpdateFeed([FromBody] FeedModel feed)
{
FeedLiteDb.AddOrUpdate(feed);
}
public void UpdateFeed(FeedModel feed)
{
FeedLiteDb.AddOrUpdate(feed);
}
public object Run(object t, ParallelTask task)
{
var model = t as CrawlTaskModel;
var results = new List <object>();
var reporter = task.Progress as IProgress <string>;
reporter.Report("正在读取Feed记录");
var feed = FeedLiteDb.GetFeed(model.FeedId);
reporter.Report("正在下载 Feed");
var compile = new UrlCompile();
var addrs = compile.GetResult(feed.Address);
foreach (var addr in addrs)
{
feed.Address = addr.ToString();
var job = new FeedJob();
var snap = job.DoTask(feed, false);
reporter.Report("Feed 下载完成");
var block = RuiJiBlockParser.ParserBlock(feed.RuiJiExpression);
var feedResult = RuiJiExtractor.Extract(snap.Content, block);
results.Add(feedResult);
reporter.Report("正在提取Feed地址");
var j = new FeedExtractJob();
var urls = j.ExtractAddress(snap);
reporter.Report("Feed地址提取完成");
if (!string.IsNullOrEmpty(snap.RuiJiExpression))
{
foreach (var url in urls)
{
reporter.Report("正在提取地址 " + url);
var result = Cooperater.GetResult(url);
if (result != null)
{
var cm = new ContentModel();
cm.Id = model.FeedId;
cm.Url = url;
cm.Metas = result.Metas;
cm.CDate = DateTime.Now;
results.Add(cm);
}
}
}
reporter.Report("计算完成");
if (!model.IncludeContent)
{
results.ForEach((m) =>
{
ClearContent(m);
});
}
}
return(results);
}
public bool RemoveFeed(string ids)
{
var removes = ids.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
return(FeedLiteDb.Remove(removes));
}
