/// <summary>
/// Connect to geekstream db and get list of feeds to be collected
/// </summary>
private void CollectContinously(string minutes)
{
if (!Regex.IsMatch(minutes, @"^\d+$"))
{
Console.WriteLine("not a number: {0}", minutes);
return;
}
var pollIntervall = TimeSpan.FromMinutes(int.Parse(minutes));
while (true)
{
DateTime collectedBefore = DateTime.Now.Add(-pollIntervall);
FeedView[] feedsToCollect = _geekStreamDb.Execute(new GetFeedsToCollectQuery(collectedBefore, 0, 20));
if (feedsToCollect.Length == 0)
{
Console.WriteLine("\nNo feeds to collect, waiting 1 minute");
Thread.Sleep(TimeSpan.FromMinutes(1));
}
else
{
List <int> collectedFeedIds = new List <int>();
var collector = new FeedCollector <FeedView>(feedsToCollect, feedView => feedView.Url);
collector.ItemCollected += ItemCollected;
collector.SourceCollected += (sender, args) => collectedFeedIds.Add(args.Source.Id);
collector.Collect();
_geekStreamDb.Execute(new SetFeedsLastCollectedCommand(collectedFeedIds.ToArray(), DateTime.Now));
}
}
}
private void ItemCollected(object sender, FeedCollector <FeedView> .SyndicationItemEventArgs e)
{
try
{
FeedView feedView = e.Source;
string[] indexKeys;
FeedItem item = CreateFeedItem(e.SyndicationItem, out indexKeys);
//Include feed title in search terms
var set = new HashSet <string>(indexKeys, StringComparer.InvariantCultureIgnoreCase);
set.UnionWith(Regex.Split(feedView.Title, @"\W+"));
indexKeys = set.ToArray();
if (!feedView.ItemUrls.Contains(item.Url))
{
Console.WriteLine(" + " + item.Title);
var command = new AddFeedItemCommand(item, indexKeys, feedView.Id, DateTime.Now);
_geekStreamDb.Execute(command);
}
}
catch (Exception ex)
{
Console.WriteLine("\nSkipping item, error:" + ex.Message);
}
}
private void AddFeed(string url)
{
try
{
SyndicationFeed syndicationFeed = FeedCollector <FeedView> .GetFeed(url);
CreateFeed(syndicationFeed, url);
Console.WriteLine("[+] " + url);
}
catch (Exception ex)
{
Console.WriteLine("[!] " + url);
Console.WriteLine(ex.Message);
}
}
