private static async void ProcessPosts(object s) { var sub = Client.GetSubreddit(Subreddit); var newPosts = new List <Post>(); var hotPosts = new List <Post>(); var risingPosts = new List <Post>(); //avoid getting unnecessary posts to keep requests lower if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.New))) { newPosts = sub.New.Take(100).ToList(); } if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Hot))) { hotPosts = sub.Hot.Take(50).ToList(); } if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Rising))) { risingPosts = sub.Rising.Take(50).ToList(); } var postComparer = new PostIdEqualityComparer(); var allPosts = new HashSet <Post>(postComparer); allPosts.UnionWith(newPosts); allPosts.UnionWith(hotPosts); allPosts.UnionWith(risingPosts); //Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful) var alreadyProcessed = ProcessedPost.GetProcessed(allPosts.Select(p => p.Id).ToList()); var removedPreviously = new List <ProcessedPost>(); //select posts that have already been removed once and add them to list removedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "remove")); var reportedPreviously = new List <ProcessedPost>(); //select posts that have already been removed once and add them to list reportedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "report")); var postTasks = new List <Task <Dictionary <string, PostAnalysisResults> > >(); foreach (var module in ActiveModules) { //hashset to prevent duplicates being passed. var posts = new HashSet <Post>(postComparer); if (module.Settings.PostTypes.HasFlag(PostType.New)) { posts.UnionWith(newPosts); } if (module.Settings.PostTypes.HasFlag(PostType.Hot)) { posts.UnionWith(hotPosts); } if (module.Settings.PostTypes.HasFlag(PostType.Rising)) { posts.UnionWith(risingPosts); } List <Post> postsList = new List <Post>(); if (!module.MultiScan) { //only add unseen posts postsList.AddRange(posts.Where(ph => alreadyProcessed.Count(ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag(module.ModuleEnum)) == 0)); } else { postsList = posts.ToList(); } if (postsList.Count > 0) { postTasks.Add(Task.Run(() => module.Analyze(postsList))); } } var results = new Dictionary <string, PostAnalysisResults>(); while (postTasks.Count > 0) { var finishedTask = await Task.WhenAny(postTasks); postTasks.Remove(finishedTask); var result = await finishedTask; foreach (var key in result.Keys) { if (results.Keys.Contains(key)) { results[key].Scores.AddRange(result[key].Scores); results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule; } else { results.Add(key, result[key]); } } } int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0; foreach (var result in results) { var combinedAnalysis = result.Value; string action = "None"; //change to Enum at some point bool unseen = false; ProcessedPost original = alreadyProcessed.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id); if (original == null) { original = new ProcessedPost(Settings.Subreddit, combinedAnalysis.Post.Id, "invalid"); unseen = true; } else { var prevScores = original.AnalysisResults.Scores.Where(os => combinedAnalysis.Scores.Count(cs => cs.ModuleName == os.ModuleName) == 0).ToList(); combinedAnalysis.Scores.AddRange(prevScores); combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule; } if (combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0) { ProcessedPost removed = removedPreviously.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id); if (removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore) { //only remove the post if it wasn't previously removed by the bot, OR if the score has increased combinedAnalysis.Post.Remove(); if (combinedAnalysis.HasFlair) { combinedAnalysis.Post.SetFlair(combinedAnalysis.FlairText, combinedAnalysis.FlairClass); } removedCounter++; } else { ignoredCounter++; } action = "Remove"; } else if (combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0) { if (reportedPreviously.Count(p => p.PostID == combinedAnalysis.Post.Id) == 0) { //can't change report text or report an item again. Thanks Obama... err... Reddit... combinedAnalysis.Post.Report(VotableThing.ReportType.Other, combinedAnalysis.ReportReason); reportedCounter++; } action = "Report"; } if (combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule) { if (combinedAnalysis.TotalScore > 0) { original.AnalysisResults = combinedAnalysis; } else { original.AnalysisResults = null; } original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule; original.Action = action; //processed post needs updated in if (unseen) { try { ProcessedPost.AddProcessedPost(original); } catch (Exception ex) { Console.WriteLine("Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message); } } else { try { ProcessedPost.UpdateProcessedPost(original); } catch (Exception ex) { Console.WriteLine("Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + (ex.InnerException != null ? ex.InnerException.Message : "null")); } } } } Console.WriteLine($"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}"); }
private static async void ProcessPosts( object s ) { var sub = Client.GetSubreddit( Subreddit ); var newPosts = new List<Post>(); var hotPosts = new List<Post>(); var risingPosts = new List<Post>(); //avoid getting unnecessary posts to keep requests lower if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.New ) ) ) { newPosts = sub.New.Take( 100 ).ToList(); } if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Hot ) ) ) { hotPosts = sub.Hot.Take( 50 ).ToList(); } if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Rising ) ) ) { risingPosts = sub.Rising.Take( 50 ).ToList(); } var postComparer = new PostIdEqualityComparer(); var allPosts = new HashSet<Post>( postComparer ); allPosts.UnionWith( newPosts ); allPosts.UnionWith( hotPosts ); allPosts.UnionWith( risingPosts ); //Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful) var alreadyProcessed = ProcessedPost.GetProcessed( allPosts.Select( p => p.Id ).ToList() ); var removedPreviously = new List<ProcessedPost>(); //select posts that have already been removed once and add them to list removedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "remove" ) ); var reportedPreviously = new List<ProcessedPost>(); //select posts that have already been removed once and add them to list reportedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "report" ) ); var postTasks = new List<Task<Dictionary<string, PostAnalysisResults>>>(); foreach ( var module in ActiveModules ) { //hashset to prevent duplicates being passed. var posts = new HashSet<Post>( postComparer ); if ( module.Settings.PostTypes.HasFlag( PostType.New ) ) { posts.UnionWith( newPosts ); } if ( module.Settings.PostTypes.HasFlag( PostType.Hot ) ) { posts.UnionWith( hotPosts ); } if ( module.Settings.PostTypes.HasFlag( PostType.Rising ) ) { posts.UnionWith( risingPosts ); } List<Post> postsList = new List<Post>(); if ( !module.MultiScan ) { //only add unseen posts postsList.AddRange( posts.Where( ph => alreadyProcessed.Count( ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag( module.ModuleEnum ) ) == 0 ) ); } else { postsList = posts.ToList(); } if ( postsList.Count > 0 ) postTasks.Add( Task.Run( () => module.Analyze( postsList ) ) ); } var results = new Dictionary<string, PostAnalysisResults>(); while ( postTasks.Count > 0 ) { var finishedTask = await Task.WhenAny( postTasks ); postTasks.Remove( finishedTask ); var result = await finishedTask; foreach ( var key in result.Keys ) { if ( results.Keys.Contains( key ) ) { results[key].Scores.AddRange( result[key].Scores ); results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule; } else { results.Add( key, result[key] ); } } } int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0; foreach ( var result in results ) { var combinedAnalysis = result.Value; string action = "None"; //change to Enum at some point bool unseen = false; ProcessedPost original = alreadyProcessed.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id ); if ( original == null ) { original = new ProcessedPost( Settings.Subreddit, combinedAnalysis.Post.Id, "invalid" ); unseen = true; } else { var prevScores = original.AnalysisResults.Scores.Where( os => combinedAnalysis.Scores.Count( cs => cs.ModuleName == os.ModuleName ) == 0 ).ToList(); combinedAnalysis.Scores.AddRange( prevScores ); combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule; } if ( combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0 ) { ProcessedPost removed = removedPreviously.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id ); if ( removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore ) { //only remove the post if it wasn't previously removed by the bot, OR if the score has increased combinedAnalysis.Post.Remove(); if ( combinedAnalysis.HasFlair ) { combinedAnalysis.Post.SetFlair( combinedAnalysis.FlairText, combinedAnalysis.FlairClass ); } removedCounter++; } else { ignoredCounter++; } action = "Remove"; } else if ( combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0 ) { if ( reportedPreviously.Count( p => p.PostID == combinedAnalysis.Post.Id ) == 0 ) { //can't change report text or report an item again. Thanks Obama... err... Reddit... combinedAnalysis.Post.Report( VotableThing.ReportType.Other, combinedAnalysis.ReportReason ); reportedCounter++; } action = "Report"; } if ( combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule ) { if ( combinedAnalysis.TotalScore > 0 ) { original.AnalysisResults = combinedAnalysis; } else original.AnalysisResults = null; original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule; original.Action = action; //processed post needs updated in if ( unseen ) { try { ProcessedPost.AddProcessedPost( original ); } catch ( Exception ex ) { Console.WriteLine( "Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message ); } } else { try { ProcessedPost.UpdateProcessedPost( original ); } catch ( Exception ex ) { Console.WriteLine( "Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + ( ex.InnerException != null ? ex.InnerException.Message : "null" ) ); } } } } Console.WriteLine( $"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}" ); }