public ArticleStubPage GetArticleStubPage(DateTime date) { var dayTag = date.ToShortDateString() == DateTime.Now.ToShortDateString() ? "" : "_" + date.ToShortDateString(); return(CachedRepository <ArticleStubPage> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).CONTENT + dayTag).FirstOrDefault() ?? new ArticleStubPage()); }
public ActionResult FrontPage() { ViewBag.ArticleStubIndex = CachedRepository <ArticleStubIndex> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).CONTENT_INDEX) .FirstOrDefault(); var page = CachedRepository <ArticleStubPage> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).CONTENT) .FirstOrDefault() ?? new ArticleStubPage(); var index = CachedRepository <ArticleIndex> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE_INDEX).FirstOrDefault() ?? new ArticleIndex(); var items = index.Articles.OrderByDescending(i => i.Published).Take(25); var fullArticles = new List <Article>(); foreach (var item in items) { var articles = CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + item.DayTag).ToList(); var article = articles.Where(x => x.UniqueKey == item.Key).FirstOrDefault(); if (article != null) { fullArticles.Add(article); } } return(View("FrontPage", new FrontPageModel(fullArticles, page.ArticleStubs))); }
public void UpdateStatus(string statusText, string screenname = null, bool processStatus = true) { if (string.IsNullOrEmpty(screenname)) { screenname = PrimaryUser.TwitterScreenName; } var tweetTask = GetAuthorizedTwitterContext(screenname).TweetAsync(statusText); tweetTask.Wait(); var status = tweetTask.Result; if (processStatus) { status = TwitterModel.Instance(PrimaryUser.TwitterScreenName).GetAuthorizedTwitterContext(screenname) .Status .Where(s => s.StatusID == status.StatusID && s.ScreenName == screenname && s.IncludeEntities == true && s.Type == StatusType.User && s.Count == 1) .ToList().FirstOrDefault(); if (status != null) { var tweet = new Tweet(status); tweet.PopulateExtendedData(); CachedRepository <Tweet> .Instance(PrimaryUser.TwitterScreenName).Save(screenname + TWEETS, tweet); } } }
public void UpdateFriendsForPrimaryUser() { string screenname = PrimaryUser.TwitterScreenName; var user = UsersCollection.Single(screenname); if (user != null && user.CanAuthorize) { //try //{ var friends = GetFollowers(screenname); if (friends != null && CachedRepository <Tweep> .Instance(PrimaryUser.TwitterScreenName).ContainsKey(screenname + FRIENDS)) { //var repoFriends = CachedRepository<Tweep>.Instance.Query(screenname + FRIENDS); //friends = friends.Except(repoFriends).ToList(); } if (friends != null) { CachedRepository <Tweep> .Instance(PrimaryUser.TwitterScreenName).Save(screenname + FRIENDS, friends); //CachedRepository<Tweep>.Instance.FlushChanges(); } //} //catch { } } }
protected override void StoreInRepository(IEnumerable <Tweet> tweets) { var tracker = TwitterModel.Instance(screenName).TRACKER; CachedRepository <Tweet> .Instance(screenName).Save(tracker, tweets.OrderBy(t => t.CreatedAt).Select(t => t).ToList()); log.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, tweets.Count(), tracker); //CachedRepository<Tweet>.Instance.FlushChanges(); }
protected virtual void StoreInRepository(IEnumerable <Tweet> tweets) { tweets .GroupBy(t => t.User.ScreenName) .ToList() .ForEach(g => { CachedRepository <Tweet> .Instance(screenName).Save(g.Key + TwitterModel.Instance(screenName).TWEETS, g.OrderBy(t => t.CreatedAt).Select(t => t).ToList()); log.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key); }); }
public List <Tweep> Friends(string screenname) { var friends = CachedRepository <Tweep> .Instance(PrimaryUser.TwitterScreenName).Query(screenname + FRIENDS, pageSize: 1000).ToList(); /* * if (friends == null) * { * LoadFriendCache(); * friends = CachedRepository<Tweep>.Instance.Find(screenname + FRIENDS); * } * */ return(friends ?? new List <Tweep>()); }
public Article GetArticle(uint id) { var articleIndex = GetArticleIndex().Articles.Where(x => x.ID() == id).FirstOrDefault(); if (articleIndex != null) { var articles = CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + articleIndex.DayTag).ToList(); return(articles.Where(x => x.UniqueKey == articleIndex.Key).FirstOrDefault()); } return(null); }
public void ExcludeArticleStub(DateTime date, string slug) { var dayTag = "_" + date.ToShortDateString(); var model = new PostworthyArticleModel(PrimaryUser); var page = model.GetArticleStubPage(date); var article = page.ArticleStubs.Where(s => s.GetSlug() == slug).FirstOrDefault(); page.ExcludedArticleStubs.Add(article); page.ExcludedArticleStubs = page.ExcludedArticleStubs.Distinct().ToList(); CachedRepository <ArticleStubPage> .Instance(PrimaryUser.TwitterScreenName).Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).CONTENT + dayTag, page); }
public void SaveArticle(Article article) { string dayTag = "_" + article.PublishedDate.ToShortDateString(); var index = GetArticleIndex(); CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + dayTag, article); index.Articles.Add(new ArticleIndex.Index(article.PublishedDate.ToFileTimeUtc(), article.Title, article.UniqueKey, article.Tags)); CachedRepository <ArticleIndex> .Instance(PrimaryUser.TwitterScreenName) .Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE_INDEX, index); }
public void Init(string screenname, TextWriter log) { this.log = log; PrimaryTweep = new Tweep(UsersCollection.Single(screenname), Tweep.TweepType.None); settingsRepo = CachedRepository <LanguageAnalyticsResults> .Instance(screenname); LanguageAnalyticsResults = (settingsRepo.Query(RuntimeRepoKey) ?? new List <LanguageAnalyticsResults> { new LanguageAnalyticsResults() }).FirstOrDefault() ?? new LanguageAnalyticsResults(); NoTweetList.Add(screenname); }
public void EditArticle(Article article) { string dayTag = "_" + article.PublishedDate.ToShortDateString(); CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + dayTag, article); var index = GetArticleIndex(); var articleIndex = index.Articles.Where(x => x.Key == article.UniqueKey).FirstOrDefault(); articleIndex.Title = article.Title; articleIndex.Tags = article.Tags; CachedRepository <ArticleIndex> .Instance(PrimaryUser.TwitterScreenName) .Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE_INDEX, index); }
private List <ITweet> GetTweets(string screenname, bool includeRelevantScreenNames, List <ulong> excludeStatusIDs = null) { List <string> screenNames = null; var user = UsersCollection.Single(screenname); if (includeRelevantScreenNames) { screenNames = GetRelevantScreenNames(screenname); } else { screenNames = new List <string> { screenname.ToLower() } }; int RetweetThreshold = PrimaryUser.RetweetThreshold; Func <Tweet, bool> where = t => //If there are any IDs we want to filter out (excludeStatusIDs == null || !excludeStatusIDs.Contains(t.StatusID)) && //Should everything be displayed or do you only want content (user.OnlyTweetsWithLinks == false || (t.Links != null && t.Links.Count > 0)) && //Minumum threshold applied so we get results worth seeing (if it is your own tweet it gets a pass on this step) ((t.RetweetCount >= RetweetThreshold /*&& t.CreatedAt > DateTime.Now.AddHours(-48)*/) || t.User.ScreenName.ToLower() == screenname.ToLower()); var tweets = screenNames //For each screen name (i.e. - you and your friends if included) select the most recent tweets .SelectMany(x => CachedRepository <Tweet> .Instance(PrimaryUser.TwitterScreenName).Query(x + TWEETS, where : where) ?? new List <Tweet>()) //Order all tweets based on rank .OrderByDescending(t => t.TweetRank) .Distinct(Tweet.GetTweetTextComparer()) .ToList(); if (!string.IsNullOrEmpty(PrimaryUser.Track)) { tweets.AddRange(CachedRepository <Tweet> .Instance(PrimaryUser.TwitterScreenName).Query(TRACKER, pageSize: 1000, where : where) ?? new List <Tweet>()); } return(tweets.Cast <ITweet>().ToList()); }
public void DeleteArticle(uint id) { var index = GetArticleIndex(); var articleIndex = index.Articles.Where(x => x.ID() == id).FirstOrDefault(); if (articleIndex != null) { var article = CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + articleIndex.DayTag, where : x => x.UniqueKey == articleIndex.Key).FirstOrDefault(); if (article != null) { CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Delete(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + articleIndex.DayTag, article); index.Articles.Remove(articleIndex); CachedRepository <ArticleIndex> .Instance(PrimaryUser.TwitterScreenName) .Save(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE_INDEX, index); } } }
public IEnumerable <Article> Articles(Func <Article, bool> where = null) { var articleIndexGroups = GetArticleIndex().Articles.GroupBy(x => x.DayTag).OrderByDescending(x => x.First().Published); foreach (var group in articleIndexGroups) { var articles = CachedRepository <Article> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE + group.Key).ToList(); var filtered = where != null?articles .Where(where) : articles; if (articles != null) { foreach (var article in filtered) { yield return(article); } } } yield return(null); }
public void Init(string screenName, TextWriter log) { this.log = log; PrimaryTweep = new Tweep(UsersCollection.Single(screenName), Tweep.TweepType.None); settingsRepo = CachedRepository <TweetBotRuntimeSettings> .Instance(screenName); commandRepo = new SimpleRepository <BotCommand>(screenName); RuntimeSettings = (settingsRepo.Query(RuntimeRepoKey) ?? new List <TweetBotRuntimeSettings> { new TweetBotRuntimeSettings() }).FirstOrDefault() ?? new TweetBotRuntimeSettings(); NoTweetList.Add(screenName.ToLower()); Messages = TweetBotSettings.Get.Messages.Count == 0 ? null : Enumerable.Range(0, TweetBotSettings.Get.Messages.Count - 1) .Select(i => TweetBotSettings.Get.Messages[i].Value).ToArray(); OnlyWithMentions = TweetBotSettings.Get.Filters["OnlyWithMentions"] != null ? TweetBotSettings.Get.Filters["OnlyWithMentions"].Value : false; ForceSimulationMode = TweetBotSettings.Get.Settings["IsSimulationMode"] != null ? TweetBotSettings.Get.Settings["IsSimulationMode"].Value : false; if (ForceSimulationMode) { log.WriteLine("{0}: Running in forced simulation mode. No actions will be taken.", DateTime.Now); } else if (SimulationMode) { log.WriteLine("{0}: Running in automatic simulation mode to aquire a baseline. No actions will be taken for {1}hrs.", DateTime.Now, TweetBotRuntimeSettings.SIMULATION_MODE_HOURS); } if (Messages == null) { log.WriteLine("{0}: 'TweetBotSettings' configuration section is missing Messages. No responses will be sent.", DateTime.Now); } else { log.WriteLine("{0}: TweetBot will respond with: {1}", DateTime.Now, Environment.NewLine + string.Join(Environment.NewLine, Messages)); } try { StopWords = File.OpenText("Resources/stopwords.txt") .ReadToEnd() .Split('\n').Select(x => x.Replace("\r", "").ToLower()); if (StopWords.Count() > 0) { StopWordsRegex = new Regex("(" + string.Join("|", StopWords.Select(sw => "\\b" + sw + "\\b")) + ")", RegexOptions.Compiled | RegexOptions.IgnoreCase); } log.WriteLine("{0}: Stop Words: {1}", DateTime.Now, string.Join(",", StopWords)); } catch { StopWords = new List <string>(); } }
static void Main(string[] args) { if (!EnsureSingleLoad()) { Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now); return; } var start = DateTime.Now; Console.WriteLine("{0}: Started", start); var users = UsersCollection.PrimaryUsers() ?? new List <PostworthyUser>(); users.AsParallel().ForAll(user => { var primaryUserName = user.TwitterScreenName; Console.WriteLine("{0}: Deleting old groups from files from storage", DateTime.Now); CachedRepository <TweetGroup> .Instance(primaryUserName).Delete(TwitterModel.Instance(primaryUserName).GROUPING); List <string> screenNames = null; screenNames = TwitterModel.Instance(primaryUserName).GetRelevantScreenNames(user.TwitterScreenName); int RetweetThreshold = user.RetweetThreshold; Func <Tweet, bool> where = t => //Should everything be displayed or do you only want content (user.OnlyTweetsWithLinks == false || (t.Links != null && t.Links.Count > 0)) && //Minumum threshold applied so we get results worth seeing (if it is your own tweet it gets a pass on this step) ((t.RetweetCount > RetweetThreshold /*&& t.CreatedAt > DateTime.Now.AddHours(-48)*/) || t.User.ScreenName.ToLower() == user.TwitterScreenName.ToLower()); var startGrouping = DateTime.Now; Console.WriteLine("{0}: Starting grouping procedure", startGrouping); var tweets = screenNames //For each screen name (i.e. - you and your friends if included) select the most recent tweets .SelectMany(x => CachedRepository <Tweet> .Instance(primaryUserName).Query(x + TwitterModel.Instance(primaryUserName).TWEETS, where : where) ?? new List <Tweet>()) //Order all tweets based on rank (TweetRank takes into acount many important factors, i.e. - time, mentions, hotness, ect.) .OrderByDescending(t => t.TweetRank) //Just to make sure we are not trying to group a very very large number of items .Take(1000); var groups = tweets //Group similar tweets (the ordering is done first so that the earliest tweet gets credit) .GroupSimilar(log: Console.Out) //Convert groups into something we can display .Select(g => new TweetGroup(g) { RepositoryKey = TwitterModel.Instance(primaryUserName).GROUPING }) //For the sake of space we only want to store groups that have more than 1 item .Where(g => g.GroupStatusIDs.Count > 1); var results = groups.ToList(); var endGrouping = DateTime.Now; Console.WriteLine("{0}: Grouping procedure completed and it took {1} minutes to complete", endGrouping, (endGrouping - startGrouping).TotalMinutes); Console.WriteLine("Storing data in repository"); if (results != null && results.Count > 0) { CachedRepository <TweetGroup> .Instance(primaryUserName).Save(TwitterModel.Instance(primaryUserName).GROUPING, results); } }); var end = DateTime.Now; Console.WriteLine("{0}: Ending and it took {1} minutes to complete", end, (end - start).TotalMinutes); }
public List <ITweet> Tweets(string screenname, bool includeRelevantScreenNames = true) { List <ITweet> returnTweets; //Since the grouping can be a somewhat expensive task we will cache the results to gain a speed up. var cachedResponse = HttpRuntime.Cache[screenname + "_" + _CACHED_TWEETS] as List <ITweet>; if (cachedResponse != null && cachedResponse.Count > 0) { returnTweets = cachedResponse; } else { var sharedResult = CachedRepository <TweetGroup> .Instance(PrimaryUser.TwitterScreenName).Query(GROUPING, pageSize: 1000); //Check to see if we have a recent grouping available (by recent it must be within the last 30 minutes) //This is what the Grouping task does for us in the background var useShared = sharedResult != null && sharedResult.Count() > 0 && sharedResult.First().CreatedOn.AddMinutes(30) > DateTime.Now; if (useShared) { lock (tweets_lock) { cachedResponse = HttpRuntime.Cache[screenname + "_" + _CACHED_TWEETS] as List <ITweet>; if (cachedResponse != null && cachedResponse.Count > 0) { returnTweets = cachedResponse; } else { var results = sharedResult.Cast <ITweet>().ToList(); results.AddRange(GetTweets(screenname, includeRelevantScreenNames, sharedResult.SelectMany(g => g.GroupStatusIDs).ToList())); HttpRuntime.Cache.Add(screenname + "_" + _CACHED_TWEETS, results, null, DateTime.Now.AddMinutes(15), Cache.NoSlidingExpiration, CacheItemPriority.Normal, null); returnTweets = results; } } } //If there is nothing local and nothing in shared cache then pull something from the repo //These results are raw and ungrouped since they are pulled on the fly and grouping can be time consuming else { lock (tweets_lock) { cachedResponse = HttpRuntime.Cache[screenname + "_" + _CACHED_TWEETS] as List <ITweet>; if (cachedResponse != null && cachedResponse.Count > 0) { returnTweets = cachedResponse; } else { var results = GetTweets(screenname, includeRelevantScreenNames); if (results != null && results.Count > 0) { HttpRuntime.Cache.Add(screenname + "_" + _CACHED_TWEETS, results, null, DateTime.Now.AddMinutes(5), Cache.NoSlidingExpiration, CacheItemPriority.Normal, null); } returnTweets = results; } } } } //Before we return the results we should order the results rank return(returnTweets.OrderByTweetRank().ToList()); }
private void LoadFromRepository() { var me = new Tweep(User, Tweep.TweepType.None); CachedRepository <TweetBotRuntimeSettings> settingsRepo = CachedRepository <TweetBotRuntimeSettings> .Instance(User.TwitterScreenName); SimpleRepository <BotCommand> commandRepo = new SimpleRepository <BotCommand>(User.TwitterScreenName); //var runtimeSettings = Newtonsoft.Json.JsonConvert.DeserializeObject<TweetBotRuntimeSettings>(System.IO.File.OpenText("c:\\temp\\runtimesettings.demo.json.txt").ReadToEnd()); var runtimeSettings = (settingsRepo.Query(RuntimeRepoKey) ?? new List <TweetBotRuntimeSettings> { new TweetBotRuntimeSettings() }).FirstOrDefault(); if (runtimeSettings != null) { IsSimulationMode = runtimeSettings.IsSimulationMode; BotStartupTime = runtimeSettings.BotFirstStart; LastTweetTime = runtimeSettings.LastTweetTime; TweetsSentSinceLastFriendRequest = runtimeSettings.TweetsSentSinceLastFriendRequest; TweetsPerHour = runtimeSettings.GetPastTweets().Count() > 1 ? runtimeSettings.GetPastTweets() .GroupBy(x => x.CreatedAt.ToShortDateString()) .SelectMany(y => y.GroupBy(z => z.CreatedAt.Hour)) .Select(x => x.Count()) .Average() : 0; TweetsPerHourMax = runtimeSettings.GetPastTweets().Count() > 2 ? runtimeSettings.GetPastTweets() .GroupBy(x => x.CreatedAt.ToShortDateString()) .SelectMany(y => y.GroupBy(z => z.CreatedAt.Hour)) .Select(x => x.Count()) .Max() : 0; MinimumRetweetLevel = (int)Math.Ceiling(runtimeSettings.MinimumRetweetLevel); CurrentClout = me.User.FollowersCount; FollowerCount = me.User.FollowersCount; FollowingCount = me.User.FriendsCount; TwitterStreamVolume = runtimeSettings.TotalTweetsProcessed / (1.0 * Runtime.TotalMinutes); TwitterFollowSuggestions = runtimeSettings.TwitterFollowSuggestions; PotentialTweets = runtimeSettings.GetPotentialTweets().OrderByDescending(t => t.TweetRank).ToList(); PotentialReTweets = runtimeSettings.GetPotentialTweets(true).OrderByDescending(t => t.TweetRank).ToList(); Tweeted = runtimeSettings.GetPastTweets().ToList(); PotentialFriendRequests = runtimeSettings.PotentialFriendRequests .Select(x => new KeyValuePair <Tweep, int>(x.Key, x.Count)).ToList(); KeywordSuggestions = runtimeSettings.KeywordSuggestions .Select(x => new KeyValuePair <string, int>(x.Key, x.Count)).ToList(); runtimeSettings.GetPastTweets() .Where(t => t.CreatedAt.AddDays(30) >= DateTime.Now) .GroupBy(t => t.CreatedAt.Day) .Select(g => new { i = g.FirstOrDefault().CreatedAt.Day - 1, date = g.FirstOrDefault().CreatedAt, count = g.Count() }) .ToList() .ForEach(x => TweetsLastThirtyDays[x.i] = x.count); TopFriendTweetCounts = runtimeSettings.GetPastTweets() .Where(t => me.Followers().Select(f => f.ID).Contains(t.User.UserID)) .GroupBy(t => t.User.UserID) .Select(g => new KeyValuePair <Tweep, int>(new Tweep(g.FirstOrDefault().User, Tweep.TweepType.None), g.Count())) .ToList(); SeededKeywords = runtimeSettings.KeywordsToIgnore; KeywordsWithOccurrenceCount = runtimeSettings.Keywords //.Concat(runtimeSettings.KeywordSuggestions.Where(x => x.Count >= TweetBotProcessingStep.MINIMUM_KEYWORD_COUNT)) .OrderByDescending(x => x.Count) .ThenByDescending(x => x.Key) .Select(x => new KeyValuePair <string, int>(x.Key, x.Count)) .ToList(); PotentialKeywordsWithOccurrenceCount = runtimeSettings.KeywordSuggestions //.Where(x => x.Count < TweetBotProcessingStep.MINIMUM_KEYWORD_COUNT) .Select(x => new KeyValuePair <string, int>(x.Key, x.Count)).ToList(); } var commands = commandRepo.Query(CommandRepoKey, where : x => !x.HasBeenExecuted); if (commands != null) { PendingKeywordAdd = commands.Where(c => c.Command == BotCommand.CommandType.AddKeyword && !c.HasBeenExecuted).Select(c => c.Value).Distinct().ToList(); PendingKeywordIgnore = commands.Where(c => c.Command == BotCommand.CommandType.IgnoreKeyword && !c.HasBeenExecuted).Select(c => c.Value).Distinct().ToList(); PendingTweetRemoval = commands.Where(c => (c.Command == BotCommand.CommandType.RemovePotentialTweet || c.Command == BotCommand.CommandType.RemovePotentialRetweet) && !c.HasBeenExecuted).Select(c => c.Value).Distinct().ToList(); } }
protected void RemoveOldTweeps() { CachedRepository <Tweep> .Instance(screenName).Delete(screenName + TwitterModel.Instance(screenName).FRIENDS); log.WriteLine("{0}: Removed Old Friends for {1}", DateTime.Now, screenName); }
protected virtual void StoreInRepository(IEnumerable <Tweep> tweeps) { CachedRepository <Tweep> .Instance(screenName).Save(screenName + TwitterModel.Instance(screenName).FRIENDS, tweeps); log.WriteLine("{0}: {1} Freinds Added ({3}) for {2}", DateTime.Now, tweeps.Count(), screenName, string.Join(",", tweeps.Select(x => x.ScreenName))); }
public ArticleIndex GetArticleIndex() { return(CachedRepository <ArticleIndex> .Instance(PrimaryUser.TwitterScreenName) .Query(TwitterModel.Instance(PrimaryUser.TwitterScreenName).ARTICLE_INDEX).FirstOrDefault() ?? new ArticleIndex()); }