public bool FeedStatusChange(string ids, string status)
        {
            var changeIds  = ids.Split(',').Select(i => Convert.ToInt32(i)).ToArray();
            var statusEnum = (Status)Enum.Parse(typeof(Status), status.ToUpper());

            return(FeedLiteDb.StatusChange(changeIds, statusEnum));
        }
        public bool ChangeFeedStatus(string ids, string status)
        {
            var changeIds  = ids.Split(',').Select(i => Convert.ToInt32(i)).ToArray();
            var statusEnum = (Status)Enum.Parse(typeof(Status), status.ToUpper());

            if (statusEnum == Status.ON)
            {
                var feeds = FeedLiteDb.GetFeed(changeIds);

                foreach (var feed in feeds)
                {
                    var @event = new BroadcastEvent()
                    {
                        Event = BroadcastEventEnum.UPDATE,
                        Args  = feed
                    };

                    Broadcast(@event);
                }
            }
            else
            {
                var @event = new BroadcastEvent()
                {
                    Event = BroadcastEventEnum.REMOVE,
                    Args  = changeIds
                };

                Broadcast(@event);
            }

            return(FeedLiteDb.ChangeStatus(changeIds, statusEnum));
        }
        public object FeedJob(string pages)
        {
            try
            {
                var node = ServerManager.Get(Request.RequestUri.Authority);

                if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
                {
                    var ps    = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
                    var feeds = FeedLiteDb.GetFeedModels(ps, 50);
                    feeds.RemoveAll(m => m.Status == FeedStatus.OFF);

                    var compile = new CompileFeedAddress();
                    foreach (var feed in feeds)
                    {
                        feed.Address = compile.Compile(feed.Address);
                    }

                    return(feeds);
                }
            }
            catch { }

            return(new { });
        }
Example #4
0
        public void TestHistoryUpdate()
        {
            var feed = FeedLiteDb.GetFeed(1);
            var job  = new FeedExtractJob();

            job.DoTask(@"D:\云同步\vcoded\RuiJi.Net\RuiJi.Cmd\bin\Debug\snapshot\1_636635303414097356.json");

            Assert.True(true);
        }
        public object Run(object t, ParallelTask task)
        {
            var model = t as CrawlTaskModel;

            var results  = new List <ExtractResult>();
            var reporter = task.Progress as IProgress <string>;

            reporter.Report("正在读取Feed记录");
            var feed = FeedLiteDb.GetFeed(model.FeedId);

            reporter.Report("正在下载 Feed");

            var compile = new CompileFeedAddress();

            feed.Address = compile.Compile(feed.Address);

            var job  = new FeedJob();
            var snap = job.DoTask(feed, false);

            reporter.Report("Feed 下载完成");

            var block = RuiJiExpression.ParserBlock(feed.RuiJiExpression);

            var feedResult = RuiJiExtracter.Extract(snap.Content, block);

            results.Add(feedResult);

            reporter.Report("正在提取Feed地址");
            var j    = new FeedExtractJob();
            var urls = j.ExtractAddress(snap);

            reporter.Report("Feed地址提取完成");

            foreach (var url in urls)
            {
                reporter.Report("正在提取地址 " + url);
                var r = ContentQueue.Instance.Extract(url);

                results.AddRange(r);
            }

            reporter.Report("计算完成");

            if (!model.IncludeContent)
            {
                results.ForEach((m) =>
                {
                    ClearContent(m);
                });
            }

            return(results);
        }
        public void UpdateFeed([FromBody] FeedModel feed)
        {
            FeedLiteDb.AddOrUpdate(feed);

            var @event = new BroadcastEvent()
            {
                Event = BroadcastEventEnum.UPDATE,
                Args  = feed
            };

            Broadcast(@event);
        }
Example #7
0
        private List <FeedModel> GetFeedJobs(string baseUrl, string proxyUrl, FeedNode node)
        {
            Logger.GetLogger(baseUrl).Info("start get feed");

            try
            {
                if (NodeConfigurationSection.Standalone)
                {
                    var paging = new Paging();
                    paging.CurrentPage = _page;
                    paging.PageSize    = 50;

                    var feeds = FeedLiteDb.GetAvailableFeeds(paging);
                    if (_page > 0 && feeds.Count == 0)
                    {
                        _page = 0;
                        paging.CurrentPage = _page;
                        feeds = FeedLiteDb.GetAvailableFeeds(paging);
                    }

                    _page++;

                    Logger.GetLogger(baseUrl).Info("get feed jobs:" + feeds.Count);

                    return(feeds);
                }
                else
                {
                    var d      = node.GetData("/config/feed/" + node.BaseUrl);
                    var config = JsonConvert.DeserializeObject <NodeConfig>(d.Data);
                    var pages  = config.Pages == null ? "" : string.Join(",", config.Pages);

                    var client      = new RestClient("http://" + proxyUrl);
                    var restRequest = new RestRequest("api/feed/job?pages=" + pages);
                    restRequest.Method = Method.GET;

                    var restResponse = client.Execute(restRequest);

                    var feeds = JsonConvert.DeserializeObject <List <FeedModel> >(restResponse.Content);

                    Logger.GetLogger(baseUrl).Info("get feed jobs:" + feeds.Count);

                    return(feeds);
                }
            }
            catch (Exception ex)
            {
                Logger.GetLogger(baseUrl).Info("get feed error " + ex.Message);

                return(new List <FeedModel>());
            }
        }
Example #8
0
        public object GetFeedJob(string pages)
        {
            try
            {
                var ps    = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
                var feeds = FeedLiteDb.GetFeedModels(ps, 50);

                return(feeds);
            }
            catch { }

            return(new { });
        }
        public object Feeds(int offset, int limit, string key, string method, string type, string status)
        {
            var paging = new Paging();

            paging.CurrentPage = (offset / limit) + 1;
            paging.PageSize    = limit;

            return(new
            {
                rows = FeedLiteDb.GetFeedModels(paging, key, method, type, status),
                total = paging.Count
            });
        }
        public bool RemoveFeed(string ids)
        {
            var removes = ids.Split(',').Select(m => Convert.ToInt32(m)).ToArray();

            var @event = new BroadcastEvent()
            {
                Event = BroadcastEventEnum.REMOVE,
                Args  = removes
            };

            Broadcast(@event);

            return(FeedLiteDb.Remove(removes));
        }
Example #11
0
        public object FeedJob(string pages)
        {
            try
            {
                var node = ServerManager.Get(Request.RequestUri.Authority);

                if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
                {
                    var ps    = pages.Split(',').Select(m => Convert.ToInt32(m)).ToArray();
                    var feeds = FeedLiteDb.GetFeedModels(ps, 50);
                    feeds.RemoveAll(m => m.Status == FeedStatus.OFF);

                    return(PreProcessUrl(feeds));
                }
            }
            catch { }

            return(new { });
        }
Example #12
0
        public object Feeds(int offset, int limit)
        {
            var node = ServerManager.Get(Request.RequestUri.Authority);

            if (node.NodeType == Node.NodeTypeEnum.FEEDPROXY)
            {
                var paging = new Paging();
                paging.CurrentPage = (offset / limit) + 1;
                paging.PageSize    = limit;

                return(new
                {
                    rows = FeedLiteDb.GetFeedModels(paging),
                    total = paging.Count
                });
            }

            return(new { });
        }
Example #13
0
        public Task SyncFeed()
        {
            return(Task.Run(() =>
            {
                try
                {
                    scheduler.Clear();

                    if (NodeConfigurationSection.Standalone)
                    {
                        var page = 1;

                        var paging = new Paging();
                        paging.CurrentPage = page;
                        paging.PageSize = 5000;

                        var feeds = FeedLiteDb.GetAvailableFeeds(paging);
                        while (feeds.Count != 0)
                        {
                            foreach (var feed in feeds)
                            {
                                AddJob(feed);
                            }

                            Logger.GetLogger(baseUrl).Info(baseUrl + " sync feed and add feed jobs:" + feeds.Count);

                            paging.CurrentPage = ++page;
                            feeds = FeedLiteDb.GetAvailableFeeds(paging);
                        }
                    }
                    else
                    {
                        var data = feedNode.GetData("/config/feed/" + baseUrl);
                        var config = JsonConvert.DeserializeObject <NodeConfig>(data.Data);
                        if (config.Pages == null || config.Pages.Length == 0)
                        {
                            return;
                        }
                        feedPages = config.Pages;

                        var feedsResponse = NodeVisitor.Feeder.GetFeedJobs(string.Join(",", config.Pages));
                        if (string.IsNullOrEmpty(feedsResponse))
                        {
                            throw new Exception("feedproxy can't connect");
                        }

                        var feeds = JsonConvert.DeserializeObject <List <FeedModel> >(feedsResponse);

                        foreach (var feed in feeds)
                        {
                            AddJob(feed);
                        }

                        Logger.GetLogger(baseUrl).Info(baseUrl + " sync feed and add feed jobs:" + feeds.Count);
                    }
                }
                catch (Exception ex)
                {
                    Logger.GetLogger(baseUrl).Error(baseUrl + " sync feed error " + ex.Message);
                }
            }));
        }
        public object GetFeed(int id)
        {
            var feed = FeedLiteDb.GetFeed(id);

            return(feed);
        }
Example #15
0
 public void UpdateFeed([FromBody] FeedModel feed)
 {
     FeedLiteDb.AddOrUpdate(feed);
 }
Example #16
0
 public void UpdateFeed(FeedModel feed)
 {
     FeedLiteDb.AddOrUpdate(feed);
 }
Example #17
0
        public object Run(object t, ParallelTask task)
        {
            var model = t as CrawlTaskModel;

            var results  = new List <object>();
            var reporter = task.Progress as IProgress <string>;

            reporter.Report("正在读取Feed记录");
            var feed = FeedLiteDb.GetFeed(model.FeedId);

            reporter.Report("正在下载 Feed");

            var compile = new UrlCompile();
            var addrs   = compile.GetResult(feed.Address);

            foreach (var addr in addrs)
            {
                feed.Address = addr.ToString();

                var job  = new FeedJob();
                var snap = job.DoTask(feed, false);
                reporter.Report("Feed 下载完成");

                var block = RuiJiBlockParser.ParserBlock(feed.RuiJiExpression);

                var feedResult = RuiJiExtractor.Extract(snap.Content, block);
                results.Add(feedResult);

                reporter.Report("正在提取Feed地址");
                var j    = new FeedExtractJob();
                var urls = j.ExtractAddress(snap);
                reporter.Report("Feed地址提取完成");

                if (!string.IsNullOrEmpty(snap.RuiJiExpression))
                {
                    foreach (var url in urls)
                    {
                        reporter.Report("正在提取地址 " + url);
                        var result = Cooperater.GetResult(url);

                        if (result != null)
                        {
                            var cm = new ContentModel();
                            cm.Id    = model.FeedId;
                            cm.Url   = url;
                            cm.Metas = result.Metas;
                            cm.CDate = DateTime.Now;

                            results.Add(cm);
                        }
                    }
                }

                reporter.Report("计算完成");

                if (!model.IncludeContent)
                {
                    results.ForEach((m) =>
                    {
                        ClearContent(m);
                    });
                }
            }

            return(results);
        }
Example #18
0
        public bool RemoveFeed(string ids)
        {
            var removes = ids.Split(',').Select(m => Convert.ToInt32(m)).ToArray();

            return(FeedLiteDb.Remove(removes));
        }