public void AddProcessedPostTest()
{
ProcessedPost p = new ProcessedPost("videos", "3Znqhs", "remove");
p.AnalysisResults = new Modules.PostAnalysisResults();
p.AnalysisResults.Scores.Add(new Modules.AnalysisScore(1, "thisasds totally notasdfadt. nope.. noasdft all", "testing1, or so I'm told", "TestModuleName", new Flair("flurrr", "red", 1)));
p.AnalysisResults.Scores.Add(new Modules.AnalysisScore(1, "this is totally not a test. nope.. not at all", "testing2, or so I'm told", "TestModuleName", new Flair("flurrr", "red", 1)));
p.AnalysisResults.Scores.Add(new Modules.AnalysisScore(1, "this asdfy not a test. nope.. not fsda", "testing3, or so I'm told", "TestModuleName", new Flair("flurrr", "red", 1)));
p.AnalysisResults.Scores.Add(new Modules.AnalysisScore(1, "this is totally not a test. nope.. not at all", "testing4, or so I'm told", "TestModuleName", new Flair("flurrr", "red", 1)));
p.AnalysisResults.Scores.Add(new Modules.AnalysisScore(1, "thasdfff is totallyvxzst. nope.. not fsfall", "testing5, or so I'm told", "TestModuleName", new Flair("flurrr", "red", 1)));
Logging.ProcessedPost.AddProcessedPost(p);
Assert.Fail();
}
private static async void ProcessMessages(object s)
{
var messages = Client.User.UnreadMessages;
var mods = new List <string>();
mods.AddRange(Client.GetSubreddit(Subreddit).Moderators.Select(m => m.Name.ToLower()).ToList()); //TODO when enabling multiple subs, fix this
foreach (var message in messages.Where(unread => unread.Kind == "t4").Cast <PrivateMessage>())
{
message.SetAsRead();
string subject = message.Subject.ToLower();
List <string> args = subject.Split('-').Select(p => p.Trim()).ToList();
bool force = args.Count > 1 && args.Contains("force");
if (!subject.Contains("validate") && !subject.Contains("check") &&
!subject.Contains("analyze") && !subject.Contains("test") && !subject.Contains("verify"))
{
message.Reply("Whatchu talkin bout Willis");
continue;
}
Post post;
try {
post = Client.GetPost(new Uri(message.Body));
}
catch {
message.Reply("That URL made me throw up in my mouth a little. Try again!");
continue;
}
if (post.SubredditName.ToLower() != Subreddit.ToLower()) //TODO when enabling multiple subreddits, this needs tweaked!
{
message.Reply($"I don't have any rules for {post.SubredditName}.");
}
else if (!mods.Contains(message.Author.ToLower()))
{
message.Reply($"You aren't a mod of {post.SubredditName}! What are you doing here? Go on! GIT!");
}
else
{
//omg finally analyze the damn thing
PostAnalysisResults result;
var original = ProcessedPost.GetProcessed(new List <string>()
{
post.Id
}).SingleOrDefault();
if ((int)original.SeenByModules == ActiveModules.Sum(a => (int)a.ModuleEnum) && !force && original.AnalysisResults != null)
{
result = original.AnalysisResults;
}
else if (post.AuthorName == "[deleted]")
{
message.Reply("The OP deleted the post, and I don't have it cached so I can't check it. Sorry (read in Canadian accent)!");
continue;
}
else
{
result = await AnalyzePost(post);
}
var reply = new StringBuilder();
reply.AppendLine(
$"Analysis results for \"[{post.Title}]({post.Permalink})\" submitted by /u/{post.AuthorName} to /r/{post.SubredditName}");
reply.AppendLine();
var action = "None";
if (Settings.RemoveScoreThreshold > 0 && result.TotalScore >= Settings.RemoveScoreThreshold)
{
action = "Remove";
}
else if (Settings.ReportScoreThreshold > 0 && result.TotalScore >= Settings.ReportScoreThreshold)
{
action = "Report";
}
reply.AppendLine($"##Action Taken: {action} with a score of {result.TotalScore}");
reply.AppendLine();
reply.AppendLine(
$"**/r/{post.SubredditName}'s thresholds** --- Remove : **{( Settings.RemoveScoreThreshold > 0 ? Settings.RemoveScoreThreshold.ToString() : "Disabled" )}** , Report : **{( Settings.ReportScoreThreshold > 0 ? Settings.ReportScoreThreshold.ToString() : "Disabled" )}**");
reply.AppendLine();
reply.AppendLine("Module| Score |Reason");
reply.AppendLine(":--|:--:|:--");
foreach (var score in result.Scores)
{
reply.AppendLine($"{score.ModuleName}|{score.Score}|{score.Reason}");
}
message.Reply(reply.ToString());
}
}
}
private static async void ProcessPosts(object s)
{
var sub = Client.GetSubreddit(Subreddit);
var newPosts = new List <Post>();
var hotPosts = new List <Post>();
var risingPosts = new List <Post>();
//avoid getting unnecessary posts to keep requests lower
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.New)))
{
newPosts = sub.New.Take(100).ToList();
}
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Hot)))
{
hotPosts = sub.Hot.Take(50).ToList();
}
if (ActiveModules.Any(m => m.Settings.PostTypes.HasFlag(PostType.Rising)))
{
risingPosts = sub.Rising.Take(50).ToList();
}
var postComparer = new PostIdEqualityComparer();
var allPosts = new HashSet <Post>(postComparer);
allPosts.UnionWith(newPosts);
allPosts.UnionWith(hotPosts);
allPosts.UnionWith(risingPosts);
//Get stats on already processed posts (This could be pulled from mod log at some point if ever desired / found to be more useful)
var alreadyProcessed = ProcessedPost.GetProcessed(allPosts.Select(p => p.Id).ToList());
var removedPreviously = new List <ProcessedPost>();
//select posts that have already been removed once and add them to list
removedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "remove"));
var reportedPreviously = new List <ProcessedPost>();
//select posts that have already been removed once and add them to list
reportedPreviously.AddRange(alreadyProcessed.Where(p => p.Action.ToLower() == "report"));
var postTasks = new List <Task <Dictionary <string, PostAnalysisResults> > >();
foreach (var module in ActiveModules)
{
//hashset to prevent duplicates being passed.
var posts = new HashSet <Post>(postComparer);
if (module.Settings.PostTypes.HasFlag(PostType.New))
{
posts.UnionWith(newPosts);
}
if (module.Settings.PostTypes.HasFlag(PostType.Hot))
{
posts.UnionWith(hotPosts);
}
if (module.Settings.PostTypes.HasFlag(PostType.Rising))
{
posts.UnionWith(risingPosts);
}
List <Post> postsList = new List <Post>();
if (!module.MultiScan)
{
//only add unseen posts
postsList.AddRange(posts.Where(ph => alreadyProcessed.Count(ap => ap.PostID == ph.Id && ap.SeenByModules.HasFlag(module.ModuleEnum)) == 0));
}
else
{
postsList = posts.ToList();
}
if (postsList.Count > 0)
{
postTasks.Add(Task.Run(() => module.Analyze(postsList)));
}
}
var results = new Dictionary <string, PostAnalysisResults>();
while (postTasks.Count > 0)
{
var finishedTask = await Task.WhenAny(postTasks);
postTasks.Remove(finishedTask);
var result = await finishedTask;
foreach (var key in result.Keys)
{
if (results.Keys.Contains(key))
{
results[key].Scores.AddRange(result[key].Scores);
results[key].AnalyzingModule = results[key].AnalyzingModule | result[key].AnalyzingModule;
}
else
{
results.Add(key, result[key]);
}
}
}
int ignoredCounter = 0, reportedCounter = 0, removedCounter = 0;
foreach (var result in results)
{
var combinedAnalysis = result.Value;
string action = "None"; //change to Enum at some point
bool unseen = false;
ProcessedPost original = alreadyProcessed.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id);
if (original == null)
{
original = new ProcessedPost(Settings.Subreddit, combinedAnalysis.Post.Id, "invalid");
unseen = true;
}
else
{
var prevScores = original.AnalysisResults.Scores.Where(os => combinedAnalysis.Scores.Count(cs => cs.ModuleName == os.ModuleName) == 0).ToList();
combinedAnalysis.Scores.AddRange(prevScores);
combinedAnalysis.AnalyzingModule = original.SeenByModules | combinedAnalysis.AnalyzingModule;
}
if (combinedAnalysis.TotalScore >= Settings.RemoveScoreThreshold && Settings.RemoveScoreThreshold > 0)
{
ProcessedPost removed = removedPreviously.SingleOrDefault(p => p.PostID == combinedAnalysis.Post.Id);
if (removed == null || removed.AnalysisResults.TotalScore < combinedAnalysis.TotalScore)
{
//only remove the post if it wasn't previously removed by the bot, OR if the score has increased
combinedAnalysis.Post.Remove();
if (combinedAnalysis.HasFlair)
{
combinedAnalysis.Post.SetFlair(combinedAnalysis.FlairText, combinedAnalysis.FlairClass);
}
removedCounter++;
}
else
{
ignoredCounter++;
}
action = "Remove";
}
else if (combinedAnalysis.TotalScore >= Settings.ReportScoreThreshold && Settings.ReportScoreThreshold > 0)
{
if (reportedPreviously.Count(p => p.PostID == combinedAnalysis.Post.Id) == 0)
{
//can't change report text or report an item again. Thanks Obama... err... Reddit...
combinedAnalysis.Post.Report(VotableThing.ReportType.Other, combinedAnalysis.ReportReason);
reportedCounter++;
}
action = "Report";
}
if (combinedAnalysis.TotalScore != original.AnalysisResults.TotalScore || action != original.Action || original.SeenByModules != combinedAnalysis.AnalyzingModule)
{
if (combinedAnalysis.TotalScore > 0)
{
original.AnalysisResults = combinedAnalysis;
}
else
{
original.AnalysisResults = null;
}
original.SeenByModules = original.SeenByModules | combinedAnalysis.AnalyzingModule;
original.Action = action;
//processed post needs updated in
if (unseen)
{
try {
ProcessedPost.AddProcessedPost(original);
}
catch (Exception ex) {
Console.WriteLine("Error adding new post as processed. Messaage : {0}", "\r\n Inner Exception : " + ex.InnerException.Message);
}
}
else
{
try {
ProcessedPost.UpdateProcessedPost(original);
}
catch (Exception ex) {
Console.WriteLine("Error updating processed post. Messaage : {0}", "\r\n Inner Exception : " + (ex.InnerException != null ? ex.InnerException.Message : "null"));
}
}
}
}
Console.WriteLine($"Successfully processed {results.Keys.Count} posts.\r\nIgnored posts: {ignoredCounter}\r\nReported Posts: {reportedCounter}\r\nRemoved Posts: {removedCounter}");
}
private void ProcessImagesInParallel(IEnumerable <MessageThing> unprocessedMessages)
{
_logger.LogInformation("Started processing unread messages.");
var tasks = new List <Task>();
var exceptions = new ConcurrentQueue <Exception>();
var semaphore = new SemaphoreSlim(0, _options.MaxThreadCount);
foreach (var message in unprocessedMessages)
{
tasks.Add(Task.Run(async() =>
{
await semaphore.WaitAsync();
var context = GetApplicationDbContext();
try
{
var link = "";
var post = await _redditService.GetPostAsync(message.ParentId);
var postIsRedditImage = !post.IsSelf && post.IsRedditMediaDomain && !post.IsVideo;
if (!postIsRedditImage)
{
throw new InvalidPostException("The requested post does not represent an image or its source is not a reddit media domain.");
}
var processedPost = await context.ProcessedPosts.FirstOrDefaultAsync(x => x.Fullname == message.ParentId);
if (processedPost != null)
{
link = processedPost.ImageUrl;
}
else
{
var image = await _imageService.GenerateImageAsync(post.Title, post.Url);
link = await _imgurService.UploadImageAsync(image);
processedPost = new ProcessedPost {
Fullname = message.ParentId, ImageUrl = link
};
await context.ProcessedPosts.AddAsync(processedPost);
await context.SaveChangesAsync();
}
var messageToUpdate = await context.Messages.FirstOrDefaultAsync(x => x.Fullname == message.Name);
messageToUpdate.IsProcessed = true;
messageToUpdate.PostId = processedPost.Id;
await context.SaveChangesAsync();
await _redditService.ReplyAsync(message.Name, link);
}
catch (InvalidPostException exception)
{
var messageToUpdate = await context.Messages.FirstOrDefaultAsync(x => x.Fullname == message.Name);
messageToUpdate.IsProcessed = true;
await context.SaveChangesAsync();
_logger.LogWarning(exception.ToString());
}
catch (Exception exception)
{
exceptions.Enqueue(exception);
}
finally
{
await _redditService.ReadMessageAsync(message.Name);
semaphore.Release();
context.Dispose();
}
}));
}
semaphore.Release(_options.MaxThreadCount);
Task.WaitAll(tasks.ToArray());
_logger.LogInformation("Ended processing unread messages.");
if (exceptions.Count > 0)
{
throw new AggregateException(exceptions);
}
}
