/// <summary> /// Connect to geekstream db and get list of feeds to be collected /// </summary> private void CollectContinously(string minutes) { if (!Regex.IsMatch(minutes, @"^\d+$")) { Console.WriteLine("not a number: {0}", minutes); return; } var pollIntervall = TimeSpan.FromMinutes(int.Parse(minutes)); while (true) { DateTime collectedBefore = DateTime.Now.Add(-pollIntervall); FeedView[] feedsToCollect = _geekStreamDb.Execute(new GetFeedsToCollectQuery(collectedBefore, 0, 20)); if (feedsToCollect.Length == 0) { Console.WriteLine("\nNo feeds to collect, waiting 1 minute"); Thread.Sleep(TimeSpan.FromMinutes(1)); } else { List <int> collectedFeedIds = new List <int>(); var collector = new FeedCollector <FeedView>(feedsToCollect, feedView => feedView.Url); collector.ItemCollected += ItemCollected; collector.SourceCollected += (sender, args) => collectedFeedIds.Add(args.Source.Id); collector.Collect(); _geekStreamDb.Execute(new SetFeedsLastCollectedCommand(collectedFeedIds.ToArray(), DateTime.Now)); } } }
private void ItemCollected(object sender, FeedCollector <FeedView> .SyndicationItemEventArgs e) { try { FeedView feedView = e.Source; string[] indexKeys; FeedItem item = CreateFeedItem(e.SyndicationItem, out indexKeys); //Include feed title in search terms var set = new HashSet <string>(indexKeys, StringComparer.InvariantCultureIgnoreCase); set.UnionWith(Regex.Split(feedView.Title, @"\W+")); indexKeys = set.ToArray(); if (!feedView.ItemUrls.Contains(item.Url)) { Console.WriteLine(" + " + item.Title); var command = new AddFeedItemCommand(item, indexKeys, feedView.Id, DateTime.Now); _geekStreamDb.Execute(command); } } catch (Exception ex) { Console.WriteLine("\nSkipping item, error:" + ex.Message); } }
private void AddFeed(string url) { try { SyndicationFeed syndicationFeed = FeedCollector <FeedView> .GetFeed(url); CreateFeed(syndicationFeed, url); Console.WriteLine("[+] " + url); } catch (Exception ex) { Console.WriteLine("[!] " + url); Console.WriteLine(ex.Message); } }