Beispiel #1
0
        public void AddPotentialTweets(List <Tweet> tweets, bool retweets = false)
        {
            GetPotentialTweets(retweets); //Loads them up if they are not already loaded up

            if (!retweets)
            {
                potentialTweets.AddRange(tweets);
                tweetRepo.Save(POTENTIAL_TWEETS + SettingsGuid, tweets);
            }
            else
            {
                potentialReTweets.AddRange(tweets);
                tweetRepo.Save(POTENTIAL_RETWEETS + SettingsGuid, tweets);
            }
        }
Beispiel #2
0
        public JsonResult Refresh()
        {
            var command = new BotCommand()
            {
                Value = DateTime.Now.ToString(), Command = BotCommand.CommandType.Refresh
            };

            commandRepo.Save(RepoKey, command);
            return(Json(command, JsonRequestBehavior.AllowGet));
        }
Beispiel #3
0
        static void Main(string[] args)
        {
            var connectionString = ConfigurationManager.AppSettings["AzureStorageConnectionString"];

            if (string.IsNullOrEmpty(connectionString))
            {
                throw new Exception("Config Section 'appSettings' missing AzureStorageConnectionString value!");
            }

            var storageAccount = CloudStorageAccount.Parse(connectionString);
            var blobClient     = storageAccount.CreateCloudBlobClient();

            var deleteTasks = new List <Task>(10000000);

            Action <Task> delete = t =>
            {
                lock (deleteTasks)
                {
                    deleteTasks.Add(t);
                }
                Console.Clear();
                Console.WriteLine("Deleting " + deleteTasks.Count + " items");
            };

            var version = TwitterModel.VERSION;

            bool content      = args.Any(a => a.ToLower() == "content");
            bool cleanVersion = args.Any(a => a.ToLower() == "version");
            bool shrink       = args.Any(a => a.ToLower().StartsWith("shrink"));
            int  newSize      = args.Where(a => a.ToLower().StartsWith("shrink")).Select(x => x == "shrink" ? 500 : int.Parse(x.Replace("shrink", ""))).FirstOrDefault();

            blobClient.ListContainers()
            //.Skip(1) //Short circuit for testing
            //.Take(1) //Short circuit for testing
            .ToList().AsParallel().ForAll(c =>
            {
                #region Index Cleanup
                var index = c.GetDirectoryReference("Index");
                foreach (var b in index.ListBlobs().Where(x => x is CloudBlockBlob).Cast <CloudBlockBlob>())
                {
                    if (cleanVersion && !b.Name.Contains(version))
                    {
                        //Delete Index
                        var i = c.GetBlockBlobReference(b.Name);
                        delete(i.DeleteIfExistsAsync());

                        //Delete all Tweets
                        var d = c.GetDirectoryReference(b.Name.Split('/').Last());
                        foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast <CloudBlockBlob>())
                        {
                            delete(t.DeleteIfExistsAsync());
                        }
                    }

                    if (shrink && newSize > 0 && b.Name.Contains(version))
                    {
                        //Get Storage Index
                        var i            = c.GetBlockBlobReference(b.Name);
                        var storageIndex = Newtonsoft.Json.JsonConvert.DeserializeObject <StorageEntityIndex>(DownloadBlob(i));

                        if (storageIndex.EntityKeys.Count > newSize)
                        {
                            //Delete extra Tweets
                            var d = c.GetDirectoryReference(b.Name.Split('/').Last());
                            foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast <CloudBlockBlob>().OrderByDescending(x => x.Properties.LastModified).Skip(newSize))
                            {
                                storageIndex.EntityKeys.Remove(t.Name.Split('/').Last());
                                delete(t.DeleteIfExistsAsync());
                            }

                            //Update Storage Index
                            UploadBlob(i, storageIndex);
                        }
                    }
                }
                #endregion

                #region Content Cleanup
                if (content)
                {
                    try
                    {
                        var repoIndex = new SimpleRepository <ArticleStubIndex>(c.Name);
                        var repoPage  = new SimpleRepository <ArticleStubPage>(c.Name);
                        var stubIndex = repoIndex.Query(TwitterModel.Instance(c.Name).CONTENT_INDEX).FirstOrDefault();
                        if (stubIndex != null)
                        {
                            var remove = new List <KeyValuePair <long, string> >();
                            for (var i = stubIndex.ArticleStubPages.Count - 31; i > -1 && i < stubIndex.ArticleStubPages.Count; i++) // Only the last month(ish)
                            {
                                var si   = stubIndex.ArticleStubPages[i];
                                var page = repoPage.Query(TwitterModel.Instance(c.Name).CONTENT + "_" + si.Value).FirstOrDefault();
                                if (page.ArticleStubs == null || page.ArticleStubs.Count == 0)
                                {
                                    repoPage.Delete(TwitterModel.Instance(c.Name).CONTENT + "_" + si.Value);
                                    remove.Add(si);
                                }
                            }
                            remove.ForEach(x => stubIndex.ArticleStubPages.Remove(x));
                            repoIndex.Save(TwitterModel.Instance(c.Name).CONTENT_INDEX, stubIndex);
                        }
                    }
                    catch { }
                }
                #endregion
            });

            Console.WriteLine("Waiting on all tasks to complete");
            Task.WaitAll(deleteTasks.ToArray());
        }
Beispiel #4
0
        protected override void StoreInRepository(IEnumerable <Twitter.Tweet> tweets)
        {
            var start  = DateTime.Now.AddHours(-48);
            var dayTag = "_" + DateTime.Now.ToShortDateString();

            Func <Tweet, bool> where = t =>
                                       t != null &&
                                       //Should everything be displayed or do you only want content
                                       (User.OnlyTweetsWithLinks == false || (t.Links != null && t.Links.Count > 0)) &&
                                       //Minumum threshold applied so we get results worth seeing (if it is your own tweet it gets a pass on this step)
                                       ((t.RetweetCount > User.RetweetThreshold || t.User.ScreenName.ToLower() == User.TwitterScreenName.ToLower()) &&
                                       //Apply Date Range
                                        (t.CreatedAt >= start));

            Tweets = Tweets.Union(tweets.Where(where)).OrderByDescending(x => x.TweetRank).Take(MAX_CONTENT).ToList();

            var groups = Tweets
                         //Group similar tweets
                         .GroupSimilar2()
                         //Convert groups into something we can display
                         .Select(g => new TweetGroup(g)
            {
                RepositoryKey = TwitterModel.Instance(User.TwitterScreenName).CONTENT
            })
                         //Order by TweetRank
                         .OrderByDescending(g => g.TweetRank)
                         //Only the top content
                         .Take(MAX_CONTENT);

            Task <List <ArticleStub> > contentTask = null;
            Task continueTask = null;

            if (groups != null && groups.Count() > 0)
            {
                //Get Standard Deviation
                double stdev  = 0;
                var    values = groups.Select(x => x.TweetRank);
                double avg    = values.Average();
                stdev = Math.Sqrt(values.Sum(d => (d - avg) * (d - avg)) / values.Count());

                //Filter groups that are way high...
                //groups = groups.Where(x => x.TweetRank < (avg + stdev));

                var results = groups.OrderByDescending(x => x.TweetRank).ToList();
                contentTask  = CreateContent(results, Page);
                continueTask = contentTask.ContinueWith(task => {
                    if (task.Result.Count >= 25)
                    {
                        var key = TwitterModel.Instance(screenName).CONTENT.ToLower();
                        Page    = new ArticleStubPage(1, task.Result.Take(100));

                        repoPage.Delete(key);
                        repoPage.Save(key, Page);

                        repoPage.Delete(key + dayTag);
                        repoPage.Save(key + dayTag, Page);

                        var articleStubIndex = repoIndex.Query(TwitterModel.Instance(screenName).CONTENT_INDEX).FirstOrDefault() ?? new ArticleStubIndex();
                        var day = DateTime.Now.StartOfDay();
                        if (articleStubIndex.ArticleStubPages.Where(x => x.Key == day.ToFileTimeUtc()).Count() == 0)
                        {
                            articleStubIndex.ArticleStubPages.Add(new KeyValuePair <long, string>(day.ToFileTimeUtc(), day.ToShortDateString()));
                            repoIndex.Save(TwitterModel.Instance(screenName).CONTENT_INDEX, articleStubIndex);
                        }
                    }
                });
            }

            base.StoreInRepository(tweets);

            if (contentTask != null && contentTask != null)
            {
                Task.WaitAll(contentTask, continueTask);
            }
        }
 public int Save(Post entity)
 {
     return(_simpleRepository.Save(entity));
 }
Beispiel #6
0
        static void Main(string[] args)
        {
            var connectionString = ConfigurationManager.AppSettings["AzureStorageConnectionString"];
            if (string.IsNullOrEmpty(connectionString))
                throw new Exception("Config Section 'appSettings' missing AzureStorageConnectionString value!");

            var storageAccount = CloudStorageAccount.Parse(connectionString);
            var blobClient = storageAccount.CreateCloudBlobClient();

            var deleteTasks = new List<Task>(10000000);

            Action<Task> delete = t =>
            {
                lock (deleteTasks)
                {
                    deleteTasks.Add(t);
                }
                Console.Clear();
                Console.WriteLine("Deleting " + deleteTasks.Count + " items");
            };

            var version = TwitterModel.VERSION;

            bool content = args.Any(a => a.ToLower() == "content");
            bool cleanVersion = args.Any(a => a.ToLower() == "version");
            bool shrink = args.Any(a => a.ToLower().StartsWith("shrink"));
            int newSize = args.Where(a => a.ToLower().StartsWith("shrink")).Select(x => x == "shrink" ? 500 : int.Parse(x.Replace("shrink", ""))).FirstOrDefault();

            blobClient.ListContainers()
                //.Skip(1) //Short circuit for testing
                //.Take(1) //Short circuit for testing
                .ToList().AsParallel().ForAll(c =>
            {
                #region Index Cleanup
                var index = c.GetDirectoryReference("Index");
                foreach (var b in index.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>())
                {
                    if (cleanVersion && !b.Name.Contains(version))
                    {
                        //Delete Index
                        var i = c.GetBlockBlobReference(b.Name);
                        delete(i.DeleteIfExistsAsync());

                        //Delete all Tweets
                        var d = c.GetDirectoryReference(b.Name.Split('/').Last());
                        foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>())
                        {
                            delete(t.DeleteIfExistsAsync());
                        }
                    }

                    if (shrink && newSize > 0 && b.Name.Contains(version))
                    {
                        //Get Storage Index
                        var i = c.GetBlockBlobReference(b.Name);
                        var storageIndex =  Newtonsoft.Json.JsonConvert.DeserializeObject<StorageEntityIndex>(DownloadBlob(i));

                        if (storageIndex.EntityKeys.Count > newSize)
                        {
                            //Delete extra Tweets
                            var d = c.GetDirectoryReference(b.Name.Split('/').Last());
                            foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>().OrderByDescending(x => x.Properties.LastModified).Skip(newSize))
                            {
                                storageIndex.EntityKeys.Remove(t.Name.Split('/').Last());
                                delete(t.DeleteIfExistsAsync());
                            }

                            //Update Storage Index
                            UploadBlob(i, storageIndex);
                        }
                    }
                }
                #endregion

                #region Content Cleanup
                if (content)
                {
                    try
                    {
                        var repoIndex = new SimpleRepository<ArticleStubIndex>(c.Name);
                        var repoPage = new SimpleRepository<ArticleStubPage>(c.Name);
                        var stubIndex = repoIndex.Query(TwitterModel.Instance(c.Name).CONTENT_INDEX).FirstOrDefault();
                        if (stubIndex != null)
                        {
                            var remove = new List<KeyValuePair<long, string>>();
                            for (var i = stubIndex.ArticleStubPages.Count - 31; i > -1 && i < stubIndex.ArticleStubPages.Count; i++) // Only the last month(ish)
                            {
                                var si = stubIndex.ArticleStubPages[i];
                                var page = repoPage.Query(TwitterModel.Instance(c.Name).CONTENT + "_" + si.Value).FirstOrDefault();
                                if (page.ArticleStubs == null || page.ArticleStubs.Count == 0)
                                {
                                    repoPage.Delete(TwitterModel.Instance(c.Name).CONTENT + "_" + si.Value);
                                    remove.Add(si);
                                }
                            }
                            remove.ForEach(x => stubIndex.ArticleStubPages.Remove(x));
                            repoIndex.Save(TwitterModel.Instance(c.Name).CONTENT_INDEX, stubIndex);
                        }
                    }
                    catch { }
                }
                #endregion
            });

            Console.WriteLine("Waiting on all tasks to complete");
            Task.WaitAll(deleteTasks.ToArray());
        }
Beispiel #7
0
        static void Main(string[] args)
        {
            if (!EnsureSingleLoad())
            {
                Console.WriteLine("{0}: Another Instance Currently Running", DateTime.Now);
                return;
            }

            var start = DateTime.Now;

            Console.WriteLine("{0}: Started", start);

            var users = UsersCollection.PrimaryUsers() ?? new List <PostworthyUser>();

            var tasks = new List <Task>();

            users.AsParallel().ForAll(u =>
            {
                var tweet     = "";
                var repoIndex = new SimpleRepository <ArticleStubIndex>(u.TwitterScreenName);
                var repoPage  = new SimpleRepository <ArticleStubPage>(u.TwitterScreenName);
                ArticleStubIndex articleStubIndex = null;
                string dayTag = "";
                DateTime day  = DateTime.MinValue;
                if (args.Length > 0)
                {
                    if (DateTime.TryParse(args[0], out day))
                    {
                        day = day.StartOfDay();

                        dayTag           = "_" + day.ToShortDateString();
                        articleStubIndex = repoIndex.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX).FirstOrDefault() ?? new ArticleStubIndex();
                        if (articleStubIndex.ArticleStubPages.Where(x => x.Key == day.ToFileTimeUtc()).Count() == 0)
                        {
                            articleStubIndex.ArticleStubPages.Add(new KeyValuePair <long, string>(day.ToFileTimeUtc(), day.ToShortDateString()));
                        }
                        else
                        {
                            articleStubIndex = null;
                        }
                    }
                }
                else
                {
                    articleStubIndex = repoIndex.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX).FirstOrDefault() ?? new ArticleStubIndex();
                    day = DateTime.Now.AddDays(-1);
                    day = day.StartOfDay();
                    if (articleStubIndex.ArticleStubPages.Where(x => x.Key == day.ToFileTimeUtc()).Count() == 0)
                    {
                        dayTag = "_" + day.ToShortDateString();
                        articleStubIndex.ArticleStubPages.Add(new KeyValuePair <long, string>(day.ToFileTimeUtc(), day.ToShortDateString()));
                        var domain = u.PrimaryDomains.OrderBy(x => x.Length).FirstOrDefault();
                        if (!string.IsNullOrEmpty(domain) && !domain.StartsWith("beta"))
                        {
                            tweet = "Here are the top articles from " + day.ToShortDateString().Replace('/', '-') + " http://" + domain + "/" + day.ToShortDateString().Replace('/', '-');
                        }
                    }
                    else
                    {
                        articleStubIndex = null;
                        day    = DateTime.MinValue;
                        dayTag = "";
                    }
                }


                var groupingResults = CreateGroups(u, day == DateTime.MinValue ? null : (DateTime?)day);
                var existing        = repoPage.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag).FirstOrDefault();
                var contentTask     = CreateContent(u, groupingResults, existing);
                Console.WriteLine("{0}: Waiting on content for {1}", DateTime.Now, u.TwitterScreenName);
                var continueTask = contentTask.ContinueWith(task =>
                {
                    Console.WriteLine("{0}: Content completed for {1}", DateTime.Now, u.TwitterScreenName);
                    var stubs = task.Result.Take(MAX_CONTENT);
                    if (stubs.Count() > 0 || !string.IsNullOrEmpty(dayTag))
                    {
                        var articleStubPage = new ArticleStubPage(1, stubs);

                        if (existing != null && existing.ExcludedArticleStubs.Count > 0)
                        {
                            articleStubPage.ExcludedArticleStubs = existing.ExcludedArticleStubs.Where(e => articleStubPage.ArticleStubs.Contains(e)).ToList();
                        }

                        Console.WriteLine("{0}: Deleting old data from files from storage for {1}", DateTime.Now, u.TwitterScreenName);
                        repoPage.Delete(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag);

                        Console.WriteLine("{0}: Storing data in repository for {1}", DateTime.Now, u.TwitterScreenName);
                        repoPage.Save(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag, articleStubPage);

                        if (articleStubIndex != null)
                        {
                            repoIndex.Save(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX, articleStubIndex);
                        }

                        if (!string.IsNullOrEmpty(tweet))
                        {
                            try
                            {
                                TwitterModel.Instance(u.TwitterScreenName).UpdateStatus(tweet, processStatus: false);
                            }
                            catch (Exception ex) { Console.WriteLine("{0}: Could not tweet message: {1}" + Environment.NewLine + "The following exception was thrown: {2}", DateTime.Now, tweet, ex.ToString()); }
                        }
                    }
                    else
                    {
                        Console.WriteLine("{0}: No articles found for {1}", DateTime.Now, u.TwitterScreenName);
                    }
                });
                tasks.Add(contentTask);
                tasks.Add(continueTask);
            });

            Task.WaitAll(tasks.ToArray());

            var end = DateTime.Now;

            Console.WriteLine("{0}: Ending and it took {1} minutes to complete", end, (end - start).TotalMinutes);
        }
Beispiel #8
0
        private void ExecutePendingCommands()
        {
            var unexecutedCommands = commandRepo.Query(CommandRepoKey, where : x => !x.HasBeenExecuted);

            if (unexecutedCommands != null)
            {
                foreach (var command in unexecutedCommands)
                {
                    switch (command.Command)
                    {
                    case BotCommand.CommandType.Refresh:
                        //We dont have to do anything since the data is saved to repo below...
                        break;

                    case BotCommand.CommandType.AddKeyword:
                        if (!RuntimeSettings.KeywordsManuallyAdded.Contains(command.Value))
                        {
                            RuntimeSettings.KeywordsManuallyAdded.Add(command.Value);
                            RuntimeSettings.Keywords.Add(new CountableItem(command.Value, 0));
                            RuntimeSettings.KeywordsManuallyIgnored.Remove(command.Value);
                            hasNewKeywordSuggestions = true;
                        }
                        break;

                    case BotCommand.CommandType.IgnoreKeyword:
                        if (!RuntimeSettings.KeywordsManuallyIgnored.Contains(command.Value))
                        {
                            RuntimeSettings.KeywordsManuallyIgnored.Add(command.Value);

                            RuntimeSettings.Keywords.Remove(RuntimeSettings.Keywords.Where(x => x.Key == command.Value).FirstOrDefault());

                            var shouldResetKeywords = RuntimeSettings.KeywordsManuallyAdded.Remove(command.Value);
                            //|| RuntimeSettings.KeywordSuggestions.Remove(RuntimeSettings.KeywordSuggestions.Where(x => x.Key == command.Value).FirstOrDefault());

                            if (shouldResetKeywords)
                            {
                                hasNewKeywordSuggestions = true;
                            }
                        }
                        break;

                    case BotCommand.CommandType.IgnoreTweep:
                        var tweepIgnore = RuntimeSettings.PotentialFriendRequests.Where(x => x.Key.UniqueKey == command.Value).FirstOrDefault();
                        if (tweepIgnore != null)
                        {
                            tweepIgnore.Key.Type = Tweep.TweepType.IgnoreAlways;
                        }
                        break;

                    case BotCommand.CommandType.TargetTweep:
                        var tweepTarget = RuntimeSettings.PotentialFriendRequests.Where(x => x.Key.UniqueKey == command.Value).FirstOrDefault();
                        if (tweepTarget != null)
                        {
                            tweepTarget.Key.Type = Tweep.TweepType.Target;
                        }
                        break;

                    case BotCommand.CommandType.RemovePotentialRetweet:
                        var retweet = RuntimeSettings.GetPotentialTweets(true).Where(x => x.UniqueKey == command.Value).FirstOrDefault();
                        RuntimeSettings.RemovePotentialTweet(retweet, true);
                        break;

                    case BotCommand.CommandType.RemovePotentialTweet:
                        var tweet = RuntimeSettings.GetPotentialTweets().Where(x => x.UniqueKey == command.Value).FirstOrDefault();
                        RuntimeSettings.RemovePotentialTweet(tweet);
                        break;
                    }

                    command.HasBeenExecuted = true;
                }

                commandRepo.Save(CommandRepoKey, unexecutedCommands);
                settingsRepo.Save(RuntimeRepoKey, RuntimeSettings);
            }
        }
Beispiel #9
0
        static void Main(string[] args)
        {
            if (!EnsureSingleLoad())
            {
                Console.WriteLine("{0}: Another Instance Currently Running", DateTime.Now);
                return;
            }

            var start = DateTime.Now;
            Console.WriteLine("{0}: Started", start);

            var users = UsersCollection.PrimaryUsers() ?? new List<PostworthyUser>();

            var tasks = new List<Task>();

            users.AsParallel().ForAll(u =>
            {
                var tweet = "";
                var repoIndex = new SimpleRepository<ArticleStubIndex>(u.TwitterScreenName);
                var repoPage = new SimpleRepository<ArticleStubPage>(u.TwitterScreenName);
                ArticleStubIndex articleStubIndex = null;
                string dayTag = "";
                DateTime day = DateTime.MinValue;
                if (args.Length > 0)
                {
                    if (DateTime.TryParse(args[0], out day))
                    {
                        day = day.StartOfDay();

                        dayTag = "_" + day.ToShortDateString();
                        articleStubIndex = repoIndex.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX).FirstOrDefault() ?? new ArticleStubIndex();
                        if (articleStubIndex.ArticleStubPages.Where(x => x.Key == day.ToFileTimeUtc()).Count() == 0)
                            articleStubIndex.ArticleStubPages.Add(new KeyValuePair<long, string>(day.ToFileTimeUtc(), day.ToShortDateString()));
                        else
                            articleStubIndex = null;
                    }
                }
                else
                {
                    articleStubIndex = repoIndex.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX).FirstOrDefault() ?? new ArticleStubIndex();
                    day = DateTime.Now.AddDays(-1);
                    day = day.StartOfDay();
                    if (articleStubIndex.ArticleStubPages.Where(x => x.Key == day.ToFileTimeUtc()).Count() == 0)
                    {
                        dayTag = "_" + day.ToShortDateString();
                        articleStubIndex.ArticleStubPages.Add(new KeyValuePair<long, string>(day.ToFileTimeUtc(), day.ToShortDateString()));
                        var domain = u.PrimaryDomains.OrderBy(x => x.Length).FirstOrDefault();
                        if (!string.IsNullOrEmpty(domain) && !domain.StartsWith("beta"))
                            tweet = "Here are the top articles from " + day.ToShortDateString().Replace('/', '-') + " http://" + domain + "/" + day.ToShortDateString().Replace('/', '-');
                    }
                    else
                    {
                        articleStubIndex = null;
                        day = DateTime.MinValue;
                        dayTag = "";
                    }
                }


                var groupingResults = CreateGroups(u, day == DateTime.MinValue ? null : (DateTime?)day);
                var existing = repoPage.Query(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag).FirstOrDefault();
                var contentTask = CreateContent(u, groupingResults, existing);
                Console.WriteLine("{0}: Waiting on content for {1}", DateTime.Now, u.TwitterScreenName);
                var continueTask = contentTask.ContinueWith(task =>
                {
                    Console.WriteLine("{0}: Content completed for {1}", DateTime.Now, u.TwitterScreenName);
                    var stubs = task.Result.Take(MAX_CONTENT);
                    if (stubs.Count() > 0 || !string.IsNullOrEmpty(dayTag))
                    {
                        var articleStubPage = new ArticleStubPage(1, stubs);

                        if (existing != null && existing.ExcludedArticleStubs.Count > 0)
                        {
                            articleStubPage.ExcludedArticleStubs = existing.ExcludedArticleStubs.Where(e => articleStubPage.ArticleStubs.Contains(e)).ToList();
                        }

                        Console.WriteLine("{0}: Deleting old data from files from storage for {1}", DateTime.Now, u.TwitterScreenName);
                        repoPage.Delete(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag);

                        Console.WriteLine("{0}: Storing data in repository for {1}", DateTime.Now, u.TwitterScreenName);
                        repoPage.Save(TwitterModel.Instance(u.TwitterScreenName).CONTENT + dayTag, articleStubPage);

                        if (articleStubIndex != null)
                            repoIndex.Save(TwitterModel.Instance(u.TwitterScreenName).CONTENT_INDEX, articleStubIndex);

                        if (!string.IsNullOrEmpty(tweet))
                        {
                            try
                            {
                                TwitterModel.Instance(u.TwitterScreenName).UpdateStatus(tweet, processStatus: false);
                            }
                            catch(Exception ex) { Console.WriteLine("{0}: Could not tweet message: {1}" + Environment.NewLine + "The following exception was thrown: {2}", DateTime.Now, tweet, ex.ToString()); }
                        }
                    }
                    else
                        Console.WriteLine("{0}: No articles found for {1}", DateTime.Now, u.TwitterScreenName);
                });
                tasks.Add(contentTask);
                tasks.Add(continueTask);
            });

            Task.WaitAll(tasks.ToArray());

            var end = DateTime.Now;
            Console.WriteLine("{0}: Ending and it took {1} minutes to complete", end, (end - start).TotalMinutes);
        }