public void AddProcessedPostTest()
{
ProcessedPost p = new ProcessedPost( "videos", "3Znqhs", "remove" );
p.AnalysisResults = new Modules.PostAnalysisResults();
p.AnalysisResults.Scores.Add( new Modules.AnalysisScore( 1, "thisasds totally notasdfadt. nope.. noasdft all", "testing1, or so I'm told", "TestModuleName", new Flair( "flurrr", "red", 1 ) ) );
p.AnalysisResults.Scores.Add( new Modules.AnalysisScore( 1, "this is totally not a test. nope.. not at all", "testing2, or so I'm told", "TestModuleName", new Flair( "flurrr", "red", 1 ) ) );
p.AnalysisResults.Scores.Add( new Modules.AnalysisScore( 1, "this asdfy not a test. nope.. not fsda", "testing3, or so I'm told", "TestModuleName", new Flair( "flurrr", "red", 1 ) ) );
p.AnalysisResults.Scores.Add( new Modules.AnalysisScore( 1, "this is totally not a test. nope.. not at all", "testing4, or so I'm told", "TestModuleName", new Flair( "flurrr", "red", 1 ) ) );
p.AnalysisResults.Scores.Add( new Modules.AnalysisScore( 1, "thasdfff is totallyvxzst. nope.. not fsfall", "testing5, or so I'm told", "TestModuleName", new Flair( "flurrr", "red", 1 ) ) );
Logging.ProcessedPost.AddProcessedPost( p );
Assert.Fail();
}
public static void AddProcessedPost( ProcessedPost post )
{
//string jsonAnalysisResults = JsonConvert.SerializeObject( post.AnalysisResults,Formatting.None );
byte[] serialized = Helpers.ProcessedPostHelpers.SerializeAndCompressResults( post );//System.Text.Encoding.ASCII.GetBytes( jsonAnalysisResults ); //This would need to change to support unicode report reasons & full explanations
var query = "" +
"insert into ProcessedPosts (SubredditID,PostID,ActionID,SeenByModules,AnalysisResults) " +
"select sub.ID, @PostID, act.ID, @SeenByModules, @AnalysisResults " +
"from Subreddits sub " +
"inner join Actions act on act.ActionName = @Action " +
"where sub.SubName like @SubName" +
";";
using ( var conn = DirtBagConnection.GetConn() ) {
conn.Execute( query, new { post.SubName, post.PostID, post.Action,post.SeenByModules, AnalysisResults = serialized } );
}
}
public static void UpdateProcessedPost(ProcessedPost post)
{
byte[] serialized = Helpers.ProcessedPostHelpers.SerializeAndCompressResults(post);
var query = "" +
"Update ProcessedPosts " +
"Set ActionID = (select ID from Actions where ActionName = @Action), AnalysisResults = @AnalysisResults, SeenByModules = @SeenByModules " +
//"FROM ProcessedPosts pp " +
//"inner join Actions act on act.ActionName = @Action " +
"where PostID like @PostID " +
";";
using (var conn = DirtBagConnection.GetConn()) {
conn.Execute(query, new { AnalysisResults = serialized, post.Action, post.PostID, post.SeenByModules });
}
}
public static void AddProcessedPost(ProcessedPost post)
{
//string jsonAnalysisResults = JsonConvert.SerializeObject( post.AnalysisResults,Formatting.None );
byte[] serialized = Helpers.ProcessedPostHelpers.SerializeAndCompressResults(post); //System.Text.Encoding.ASCII.GetBytes( jsonAnalysisResults ); //This would need to change to support unicode report reasons & full explanations
var query = "" +
"insert into ProcessedPosts (SubredditID,PostID,ActionID,SeenByModules,AnalysisResults) " +
"select sub.ID, @PostID, act.ID, @SeenByModules, @AnalysisResults " +
"from Subreddits sub " +
"inner join Actions act on act.ActionName = @Action " +
"where sub.SubName like @SubName" +
";";
using (var conn = DirtBagConnection.GetConn()) {
conn.Execute(query, new { post.SubName, post.PostID, post.Action, post.SeenByModules, AnalysisResults = serialized });
}
}
public static void UpdateProcessedPost( ProcessedPost post )
{
byte[] serialized = Helpers.ProcessedPostHelpers.SerializeAndCompressResults( post );
var query = "" +
"Update ProcessedPosts " +
"Set ActionID = (select ID from Actions where ActionName = @Action), AnalysisResults = @AnalysisResults, SeenByModules = @SeenByModules " +
//"FROM ProcessedPosts pp " +
//"inner join Actions act on act.ActionName = @Action " +
"where PostID like @PostID " +
";";
using ( var conn = DirtBagConnection.GetConn() ) {
conn.Execute( query, new { AnalysisResults = serialized, post.Action, post.PostID, post.SeenByModules } );
}
}
private static async void ProcessPosts( object s ) {
var sub = Client.GetSubreddit( Subreddit );
var newPosts = new List<Post>();
var hotPosts = new List<Post>();
var risingPosts = new List<Post>();
//avoid getting unnecessary posts to keep requests lower
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.New ) ) ) {
newPosts = sub.New.Take( 100 ).ToList();
}
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Hot ) ) ) {
hotPosts = sub.Hot.Take( 50 ).ToList();
}
if ( ActiveModules.Any( m => m.Settings.PostTypes.HasFlag( PostType.Rising ) ) ) {
risingPosts = sub.Rising.Take( 50 ).ToList();
}
var postComparer = new PostIdEqualityComparer();
var allPosts = new HashSet<Post>( postComparer );
allPosts.UnionWith( newPosts );
allPosts.UnionWith( hotPosts );
allPosts.UnionWith( risingPosts );
//Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful)
var alreadyProcessed = ProcessedPost.GetProcessed( allPosts.Select( p => p.Id ).ToList() );
var removedPreviously = new List<ProcessedPost>();
//select posts that have already been removed once and add them to list
removedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "remove" ) );
var reportedPreviously = new List<ProcessedPost>();
//select posts that have already been removed once and add them to list
reportedPreviously.AddRange( alreadyProcessed.Where( p => p.Action.ToLower() == "report" ) );
var postTasks = new List<Task<Dictionary<string, PostAnalysisResults>>>();
foreach ( var module in ActiveModules ) {
//hashset to prevent duplicates being passed.
var posts = new HashSet<Post>( postComparer );
if ( module.Settings.PostTypes.HasFlag( PostType.New ) ) {
posts.UnionWith( newPosts );
}
if ( module.Settings.PostTypes.HasFlag( PostType.Hot ) ) {
posts.UnionWith( hotPosts );
}
if ( module.Settings.PostTypes.HasFlag( PostType.Rising ) ) {
posts.UnionWith( risingPosts );
}
List<Post> postsList = new List<Post>();
if ( !module.MultiScan ) {
//only add unseen posts
postsList.AddRange( posts.Where( ph => alreadyProcessed.Count( ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag( module.ModuleEnum ) ) == 0 ) );
}
else {
postsList = posts.ToList();
}
if ( postsList.Count > 0 ) postTasks.Add( Task.Run( () => module.Analyze( postsList ) ) );
}
var results = new Dictionary<string, PostAnalysisResults>();
while ( postTasks.Count > 0 ) {
var finishedTask = await Task.WhenAny( postTasks );
postTasks.Remove( finishedTask );
var result = await finishedTask;
foreach ( var key in result.Keys ) {
if ( results.Keys.Contains( key ) ) {
results[key].Scores.AddRange( result[key].Scores );
results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule;
}
else {
results.Add( key, result[key] );
}
}
}
int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0;
foreach ( var result in results ) {
var combinedAnalysis = result.Value;
string action = "None"; //change to Enum at some point
bool unseen = false;
ProcessedPost original = alreadyProcessed.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id );
if ( original == null ) {
original = new ProcessedPost( Settings.Subreddit, combinedAnalysis.Post.Id, "invalid" );
unseen = true;
}
else {
var prevScores = original.AnalysisResults.Scores.Where( os => combinedAnalysis.Scores.Count( cs => cs.ModuleName == os.ModuleName ) == 0 ).ToList();
combinedAnalysis.Scores.AddRange( prevScores );
combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule;
}
if ( combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0 ) {
ProcessedPost removed = removedPreviously.SingleOrDefault( p => p.PostID == combinedAnalysis.Post.Id );
if ( removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore ) {
//only remove the post if it wasn't previously removed by the bot, OR if the score has increased
combinedAnalysis.Post.Remove();
if ( combinedAnalysis.HasFlair ) {
combinedAnalysis.Post.SetFlair( combinedAnalysis.FlairText, combinedAnalysis.FlairClass );
}
removedCounter++;
}
else {
ignoredCounter++;
}
action = "Remove";
}
else if ( combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0 ) {
if ( reportedPreviously.Count( p => p.PostID == combinedAnalysis.Post.Id ) == 0 ) {
//can't change report text or report an item again. Thanks Obama... err... Reddit...
combinedAnalysis.Post.Report( VotableThing.ReportType.Other, combinedAnalysis.ReportReason );
reportedCounter++;
}
action = "Report";
}
if ( combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule ) {
if ( combinedAnalysis.TotalScore > 0 ) {
original.AnalysisResults = combinedAnalysis;
}
else original.AnalysisResults = null;
original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule;
original.Action = action;
//processed post needs updated in
if ( unseen ) {
try {
ProcessedPost.AddProcessedPost( original );
}
catch ( Exception ex ) {
Console.WriteLine( "Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message );
}
}
else {
try {
ProcessedPost.UpdateProcessedPost( original );
}
catch ( Exception ex ) {
Console.WriteLine( "Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + ( ex.InnerException != null ? ex.InnerException.Message : "null" ) );
}
}
}
}
Console.WriteLine( $"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}" );
}