private static async void ProcessPosts(object s)
{
var sub = Client.GetSubreddit(Subreddit);
var newPosts = new List <Post>();
var hotPosts = new List <Post>();
var risingPosts = new List <Post>();
//avoid getting unnecessary posts to keep requests lower
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.New)))
{
newPosts = sub.New.Take(100).ToList();
}
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Hot)))
{
hotPosts = sub.Hot.Take(50).ToList();
}
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Rising)))
{
risingPosts = sub.Rising.Take(50).ToList();
}
var postComparer = new PostIdEqualityComparer();
var allPosts = new HashSet <Post>(postComparer);
allPosts.UnionWith(newPosts);
allPosts.UnionWith(hotPosts);
allPosts.UnionWith(risingPosts);
//Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful)
var alreadyProcessed = ProcessedPost.GetProcessed(allPosts.Select(p => p.Id).ToList());
var removedPreviously = new List <ProcessedPost>();
//select posts that have already been removed once and add them to list
removedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "remove"));
var reportedPreviously = new List <ProcessedPost>();
//select posts that have already been removed once and add them to list
reportedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "report"));
var postTasks = new List <Task <Dictionary <string, PostAnalysisResults> > >();
foreach (var module in ActiveModules)
{
//hashset to prevent duplicates being passed.
var posts = new HashSet <Post>(postComparer);
if (module.Settings.PostTypes.HasFlag(PostType.New))
{
posts.UnionWith(newPosts);
}
if (module.Settings.PostTypes.HasFlag(PostType.Hot))
{
posts.UnionWith(hotPosts);
}
if (module.Settings.PostTypes.HasFlag(PostType.Rising))
{
posts.UnionWith(risingPosts);
}
List <Post> postsList = new List <Post>();
if (!module.MultiScan)
{
//only add unseen posts
postsList.AddRange(posts.Where(ph => alreadyProcessed.Count(ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag(module.ModuleEnum)) == 0));
}
else
{
postsList = posts.ToList();
}
if (postsList.Count > 0)
{
postTasks.Add(Task.Run(() => module.Analyze(postsList)));
}
}
var results = new Dictionary <string, PostAnalysisResults>();
while (postTasks.Count > 0)
{
var finishedTask = await Task.WhenAny(postTasks);
postTasks.Remove(finishedTask);
var result = await finishedTask;
foreach (var key in result.Keys)
{
if (results.Keys.Contains(key))
{
results[key].Scores.AddRange(result[key].Scores);
results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule;
}
else
{
results.Add(key, result[key]);
}
}
}
int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0;
foreach (var result in results)
{
var combinedAnalysis = result.Value;
string action = "None"; //change to Enum at some point
bool unseen = false;
ProcessedPost original = alreadyProcessed.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id);
if (original == null)
{
original = new ProcessedPost(Settings.Subreddit, combinedAnalysis.Post.Id, "invalid");
unseen = true;
}
else
{
var prevScores = original.AnalysisResults.Scores.Where(os => combinedAnalysis.Scores.Count(cs => cs.ModuleName == os.ModuleName) == 0).ToList();
combinedAnalysis.Scores.AddRange(prevScores);
combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule;
}
if (combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0)
{
ProcessedPost removed = removedPreviously.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id);
if (removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore)
{
//only remove the post if it wasn't previously removed by the bot, OR if the score has increased
combinedAnalysis.Post.Remove();
if (combinedAnalysis.HasFlair)
{
combinedAnalysis.Post.SetFlair(combinedAnalysis.FlairText, combinedAnalysis.FlairClass);
}
removedCounter++;
}
else
{
ignoredCounter++;
}
action = "Remove";
}
else if (combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0)
{
if (reportedPreviously.Count(p => p.PostID == combinedAnalysis.Post.Id) == 0)
{
//can't change report text or report an item again. Thanks Obama... err... Reddit...
combinedAnalysis.Post.Report(VotableThing.ReportType.Other, combinedAnalysis.ReportReason);
reportedCounter++;
}
action = "Report";
}
if (combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule)
{
if (combinedAnalysis.TotalScore > 0)
{
original.AnalysisResults = combinedAnalysis;
}
else
{
original.AnalysisResults = null;
}
original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule;
original.Action = action;
//processed post needs updated in
if (unseen)
{
try {
ProcessedPost.AddProcessedPost(original);
}
catch (Exception ex) {
Console.WriteLine("Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message);
}
}
else
{
try {
ProcessedPost.UpdateProcessedPost(original);
}
catch (Exception ex) {
Console.WriteLine("Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + (ex.InnerException != null ? ex.InnerException.Message : "null"));
}
}
}
}
Console.WriteLine($"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}");
}
private static async void ProcessPosts( object s ) {
var sub = Client.GetSubreddit( Subreddit );
var newPosts = new List<Post>();
var hotPosts = new List<Post>();
var risingPosts = new List<Post>();
//avoid getting unnecessary posts to keep requests lower
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.New ) ) ) {
newPosts = sub.New.Take( 100 ).ToList();
}
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Hot ) ) ) {
hotPosts = sub.Hot.Take( 50 ).ToList();
}
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Rising ) ) ) {
risingPosts = sub.Rising.Take( 50 ).ToList();
}
var postComparer = new PostIdEqualityComparer();
var allPosts = new HashSet<Post>( postComparer );
allPosts.UnionWith( newPosts );
allPosts.UnionWith( hotPosts );
allPosts.UnionWith( risingPosts );
//Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful)
var alreadyProcessed = ProcessedPost.GetProcessed( allPosts.Select( p => p.Id ).ToList() );
var removedPreviously = new List<ProcessedPost>();
//select posts that have already been removed once and add them to list
removedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "remove" ) );
var reportedPreviously = new List<ProcessedPost>();
//select posts that have already been removed once and add them to list
reportedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "report" ) );
var postTasks = new List<Task<Dictionary<string, PostAnalysisResults>>>();
foreach ( var module in ActiveModules ) {
//hashset to prevent duplicates being passed.
var posts = new HashSet<Post>( postComparer );
if ( module.Settings.PostTypes.HasFlag( PostType.New ) ) {
posts.UnionWith( newPosts );
}
if ( module.Settings.PostTypes.HasFlag( PostType.Hot ) ) {
posts.UnionWith( hotPosts );
}
if ( module.Settings.PostTypes.HasFlag( PostType.Rising ) ) {
posts.UnionWith( risingPosts );
}
List<Post> postsList = new List<Post>();
if ( !module.MultiScan ) {
//only add unseen posts
postsList.AddRange( posts.Where( ph => alreadyProcessed.Count( ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag( module.ModuleEnum ) ) == 0 ) );
}
else {
postsList = posts.ToList();
}
if ( postsList.Count > 0 ) postTasks.Add( Task.Run( () => module.Analyze( postsList ) ) );
}
var results = new Dictionary<string, PostAnalysisResults>();
while ( postTasks.Count > 0 ) {
var finishedTask = await Task.WhenAny( postTasks );
postTasks.Remove( finishedTask );
var result = await finishedTask;
foreach ( var key in result.Keys ) {
if ( results.Keys.Contains( key ) ) {
results[key].Scores.AddRange( result[key].Scores );
results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule;
}
else {
results.Add( key, result[key] );
}
}
}
int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0;
foreach ( var result in results ) {
var combinedAnalysis = result.Value;
string action = "None"; //change to Enum at some point
bool unseen = false;
ProcessedPost original = alreadyProcessed.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id );
if ( original == null ) {
original = new ProcessedPost( Settings.Subreddit, combinedAnalysis.Post.Id, "invalid" );
unseen = true;
}
else {
var prevScores = original.AnalysisResults.Scores.Where( os => combinedAnalysis.Scores.Count( cs => cs.ModuleName == os.ModuleName ) == 0 ).ToList();
combinedAnalysis.Scores.AddRange( prevScores );
combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule;
}
if ( combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0 ) {
ProcessedPost removed = removedPreviously.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id );
if ( removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore ) {
//only remove the post if it wasn't previously removed by the bot, OR if the score has increased
combinedAnalysis.Post.Remove();
if ( combinedAnalysis.HasFlair ) {
combinedAnalysis.Post.SetFlair( combinedAnalysis.FlairText, combinedAnalysis.FlairClass );
}
removedCounter++;
}
else {
ignoredCounter++;
}
action = "Remove";
}
else if ( combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0 ) {
if ( reportedPreviously.Count( p => p.PostID == combinedAnalysis.Post.Id ) == 0 ) {
//can't change report text or report an item again. Thanks Obama... err... Reddit...
combinedAnalysis.Post.Report( VotableThing.ReportType.Other, combinedAnalysis.ReportReason );
reportedCounter++;
}
action = "Report";
}
if ( combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule ) {
if ( combinedAnalysis.TotalScore > 0 ) {
original.AnalysisResults = combinedAnalysis;
}
else original.AnalysisResults = null;
original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule;
original.Action = action;
//processed post needs updated in
if ( unseen ) {
try {
ProcessedPost.AddProcessedPost( original );
}
catch ( Exception ex ) {
Console.WriteLine( "Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message );
}
}
else {
try {
ProcessedPost.UpdateProcessedPost( original );
}
catch ( Exception ex ) {
Console.WriteLine( "Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + ( ex.InnerException != null ? ex.InnerException.Message : "null" ) );
}
}
}
}
Console.WriteLine( $"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}" );
}
