private async Task ProcessModLog()
{
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
var req = yt.Videos.List("snippet");
var lastRemoval = PostRemoval.GetLastProcessedRemovalDate(Subreddit);
var processedCount = 0;
var modActions = RedditClient.GetSubreddit(Subreddit).GetModerationLog(ModActionType.RemoveLink).GetListing(300, 100);
var newPosts = new Dictionary <string, List <UserPost> >();
foreach (var modAct in modActions)
{
if (modAct.TimeStamp <= lastRemoval || processedCount > 100)
{
break; //probably dumb and unnecessary
}
processedCount++;
var post = RedditClient.GetThingByFullname(modAct.TargetThingFullname) as Post;
var userPost = new UserPost();
userPost.ThingID = post.Id;
userPost.Link = post.Url.ToString();
userPost.PostTime = post.CreatedUTC;
userPost.UserName = post.AuthorName;
userPost.Subreddit = Subreddit;
var removal = new PostRemoval(modAct);
removal.Post = userPost;
var newPost = PostRemoval.AddRemoval(removal);
if (newPost != null)
{
var ytID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
if (!string.IsNullOrEmpty(ytID))
{
if (!newPosts.ContainsKey(ytID))
{
newPosts.Add(ytID, new List <UserPost>());
}
newPosts[ytID].Add(newPost);
}
}
if (processedCount % 75 == 0)
{
await UpdateChannels(newPosts, req);
newPosts.Clear();
}
}
await UpdateChannels(newPosts, req);
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <Post> posts)
{
var toReturn = new Dictionary <string, PostAnalysisResults>();
var youTubePosts = new Dictionary <string, List <Post> >();
foreach (var post in posts)
{
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
var ytID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
if (!string.IsNullOrEmpty(ytID))
{
if (!youTubePosts.ContainsKey(ytID))
{
youTubePosts.Add(ytID, new List <Post>());
}
youTubePosts[ytID].Add(post);
}
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
var req = yt.Videos.List("snippet");
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
var ids = youTubePosts.Keys.Skip(i).Take(50);
req.Id = string.Join(",", ids);
var ytScrape = ScrapeYouTube(youTubePosts.Skip(i).Take(50).ToDictionary(p => p.Key, p => p.Value), toReturn);
var response = await req.ExecuteAsync();
foreach (var vid in response.Items)
{
var redditPosts = youTubePosts[vid.Id];
//var scores = toReturn[post.Id].Scores;
var termMatches = TermMatching.Matches(vid.Snippet.Description).Cast <Match>().Select(m => m.Value).ToList();
termMatches.AddRange(TermMatching.Matches(vid.Snippet.Title).Cast <Match>().Select(m => m.Value).ToList().Distinct());
if (termMatches.Count > 0)
{
foreach (var post in redditPosts)
{
toReturn[post.Id].Scores.Add(new AnalysisScore(STRINGMATCH_SCORE * Settings.ScoreMultiplier, "YouTube video title or description has the following term(s): " + string.Join(", ", termMatches), "Match: " + string.Join(", ", termMatches), ModuleName, RemovalFlair));
}
}
}
await ytScrape;
}
return(toReturn);
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <Post> posts)
{
//return await Task.Run( () => {
var modLog = ProcessModLog();
var toReturn = new Dictionary <string, PostAnalysisResults>();
var youTubePosts = new Dictionary <string, List <Post> >();
foreach (var post in posts)
{
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
var ytID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
if (!string.IsNullOrEmpty(ytID))
{
if (!youTubePosts.ContainsKey(ytID))
{
youTubePosts.Add(ytID, new List <Post>());
}
youTubePosts[ytID].Add(post);
}
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
var req = yt.Videos.List("snippet");
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
req.Id = string.Join(",", youTubePosts.Keys.Skip(i).Take(50));
var response = await req.ExecuteAsync();
foreach (var vid in response.Items)
{
foreach (var post in youTubePosts[vid.Id])
{
UserPost.InsertPost(new UserPost {
ChannelID = vid.Snippet.ChannelId, ChannelName = vid.Snippet.ChannelTitle, ThingID = post.Id, Link = post.Url.ToString(), PostTime = post.CreatedUTC, UserName = post.AuthorName, Subreddit = post.SubredditName
});
}
}
}
//Task.Run( () => {
//} );
await modLog;
return(toReturn);
//} );
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <rs.Post> posts)
{
var toReturn = new Dictionary <string, PostAnalysisResults>();
var youTubePosts = new Dictionary <string, List <rs.Post> >();
var amWrangler = new BotFunctions.AutoModWrangler(Program.Client.GetSubreddit(Program.Subreddit));
var bannedChannels = amWrangler.GetBannedList(Models.BannedEntity.EntityType.Channel);
foreach (var post in posts) //TODO error handling
{
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
string postYTID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
if (!string.IsNullOrWhiteSpace(postYTID))
{
if (!youTubePosts.ContainsKey(postYTID))
{
youTubePosts.Add(postYTID, new List <rs.Post>());
}
youTubePosts[postYTID].Add(post);
}
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
var req = yt.Videos.List("snippet");
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
req.Id = string.Join(",", youTubePosts.Keys.Skip(i).Take(50));
var response = req.ExecuteAsync();
await Task.WhenAll(response, bannedChannels);
foreach (var vid in response.Result.Items)
{
//if the channel is banned
var chan = bannedChannels.Result.Where(c => c.EntityString == vid.Snippet.ChannelId).FirstOrDefault();
if (chan != null)
{
foreach (var ytPost in youTubePosts[vid.Id])
{
//ring 'er up
toReturn[ytPost.Id].Scores.Add(new AnalysisScore(9999, $"Channel ID: {chan.EntityString} was banned by {chan.BannedBy} on {chan.BanDate} for reason: {chan.BanReason}", "Banned Channel", ModuleName));
}
}
}
}
return(toReturn);
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <Post> posts)
{
var toReturn = new Dictionary <string, PostAnalysisResults>();
foreach (var post in posts) //TODO error handling
{
var youTubePosts = new Dictionary <string, List <Post> >();
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
string postYTID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
Task <Logging.UserPostingHistory> hist;
if (!string.IsNullOrEmpty(postYTID))
{
//It's a YouTube vid so we can kick off the analysis and get cookin
hist = Logging.UserPostingHistory.GetUserPostingHistory(post.AuthorName);
if (!youTubePosts.ContainsKey(postYTID))
{
youTubePosts.Add(postYTID, new List <Post>());
}
youTubePosts[postYTID].Add(post);
}
else
{
//not a YouTube post, so bail out
continue;
}
bool success = false;
int nonYTPosts = 0;
int tries = 0;
while (!success && tries < 3)
{
success = true;
try {
var recentPosts = RedditClient.Search <RedditSharp.Things.Post>($"author:{post.AuthorName} self:no", RedditSharp.Sorting.New).GetListing(100, 100);
foreach (var recentPost in recentPosts)
{
string ytID = YouTubeHelpers.ExtractVideoId(recentPost.Url.ToString());
if (!string.IsNullOrEmpty(ytID))
{
if (!youTubePosts.ContainsKey(ytID))
{
youTubePosts.Add(ytID, new List <Post>());
}
youTubePosts[ytID].Add(post);
}
else
{
nonYTPosts++;
}
}
}
catch (Exception ex) {
success = false;
tries++;
if (tries > 3)
{
Console.WriteLine($"Failed to get search results: {ex.Message}");
processedCache.Remove(post.Id);
break;
}
await Task.Delay(100);
}
}
if (tries > 3)
{
continue;
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
Dictionary <string, List <string> > postHistory = ( await hist ).PostingHistory;
string postChannelID = "";
string postChannelName = "";
var req = yt.Videos.List("snippet");
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
req.Id = string.Join(",", youTubePosts.Keys.Skip(i).Take(50));
var response = await req.ExecuteAsync();
foreach (var vid in response.Items)
{
foreach (var ytPost in youTubePosts[vid.Id])
{
if (!postHistory.ContainsKey(vid.Snippet.ChannelId))
{
postHistory.Add(vid.Snippet.ChannelId, new List <string>());
}
//check to see if it already exists (aka wasnt deleted and showed up in search results)
if (!postHistory[vid.Snippet.ChannelId].Contains(ytPost.Id))
{
postHistory[vid.Snippet.ChannelId].Add(ytPost.Id);
}
if (vid.Id == postYTID)
{
postChannelID = vid.Snippet.ChannelId;
postChannelName = vid.Snippet.ChannelTitle;
}
}
}
}
if (string.IsNullOrEmpty(postChannelID))
{
//shouldn't ever happen, but might if the video is deleted or the channel deleted or something
Console.WriteLine($"Channel for post {post.Id} by {post.AuthorName} couldn't be found");
continue;
}
int totalPosts = postHistory.Sum(ph => ph.Value.Count) + nonYTPosts;
int channelPosts = postHistory[postChannelID].Count;
if (!IncludePostInPercentage)
{
totalPosts--;
channelPosts--;
postHistory[postChannelID].Remove(post.Id);
}
double percent = ((double)channelPosts / totalPosts) * 100;
if (percent > PercentageThreshold && channelPosts > GracePeriod)
{
var score = new AnalysisScore();
score.ModuleName = "SelfPromotionCombustor";
score.ReportReason = $"SelfPromo: {Math.Round( percent, 2 )}%";
score.Reason = $"Self Promotion for channel '{postChannelName}' with a posting percentage of {Math.Round( percent, 2 )}. Found PostIDs: {string.Join( ", ", postHistory[postChannelID] )}";
score.Score = OVER_PERCENT_SCORE * Settings.ScoreMultiplier;
score.RemovalFlair = RemovalFlair;
toReturn[post.Id].Scores.Add(score);
}
}
return(toReturn);
}
/// <summary>
/// Builds the HTML embed iframe for the specified video.
/// </summary>
/// <param name="videoId">The YouTube ID of the video.</param>
/// <param name="width">The desired width of the iframe.</param>
/// <param name="height">The desired height of the iframe.</param>
/// <param name="showRelations">By the default, YouTube will show
/// related videos at the end of videos. Setting this to
/// <var>FALSE</var> will disable the feature.</param>
/// <param name="wmode">The flash video player doesn't really
/// play well with layers (mostly in IE). Setting this
/// parameter to <var>transarent</var> will solve most
/// of these issues.</param>
public static string YouTubeEmbed(this string videoId, int width, int height, bool showRelations, string wmode)
{
return(YouTubeHelpers.GetEmbedHtml(videoId, width, height, showRelations, wmode));
}
/// <summary>
/// Builds the HTML embed iframe for the specified video.
/// </summary>
/// <param name="videoId">The YouTube ID of the video.</param>
/// <param name="width">The desired width of the iframe.</param>
/// <param name="height">The desired height of the iframe.</param>
public static string YouTubeEmbed(this string videoId, int width, int height)
{
return(YouTubeHelpers.GetEmbedHtml(videoId, width, height));
}
/// <summary>
/// Uses regular expressions for finding a YouTube video ID in the string.
/// </summary>
/// <param name="subject">The string to search.</param>
/// <param name="videoId">The YouTube video ID if found, otherwise <var>NULL</var>.</param>
/// <returns>Returns <var>TRUE</var> if a video ID is found, otherwise <var>FALSE</var>.</returns>
public static bool GetYouTubeId(this string subject, out string videoId)
{
return(YouTubeHelpers.GetIdFromString(subject, out videoId));
}
/// <summary>
/// Uses regular expressions for finding a YouTube video ID in the string.
/// </summary>
/// <param name="subject">The string to search.</param>
/// <returns>The YouTube video ID if found, otherwise <var>NULL</var>.</returns>
public static string GetYouTubeId(this string subject)
{
return(YouTubeHelpers.GetIdFromString(subject));
}
/// <summary>
/// Attempts to find a YouTube video ID the specified string and get
/// information about that video.
/// </summary>
/// <param name="subject">The string to search.</param>
public static YouTubeVideo GetYouTubeVideo(this string subject)
{
return(YouTubeHelpers.GetVideoFromString(subject));
}
/// <summary>
/// Gets the thumbnail URL for a video with the specified ID. The default thumbnail (index = 0)
/// measures 480x360 pixels, while the others measures 120x90 pixels.
/// </summary>
/// <param name="videoId">The ID of the video.</param>
/// <param name="index">The index of the thumbnail URL to return
/// (valid range is from 0 to 3 - both inclusive).</param>
/// <returns>The thumbnail URL if the video ID is valid, otherwise <var>NULL</var>.</returns>
public static string GetYouTubeThumbnail(this string videoId, int index)
{
return(YouTubeHelpers.GetYouTubeThumbnail(videoId, index));
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <Post> posts)
{
var toReturn = new Dictionary <string, PostAnalysisResults>();
var youTubePosts = new Dictionary <string, List <Post> >();
foreach (var post in posts)
{
var ytID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
if (!string.IsNullOrEmpty(ytID))
{
if (!youTubePosts.ContainsKey(ytID))
{
youTubePosts.Add(ytID, new List <Post>());
}
youTubePosts[ytID].Add(post);
}
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
var req = yt.Videos.List("snippet,contentDetails,statistics");
var settings = (YouTubeSpamDetectorSettings)Settings;
double availWeight = 0;
availWeight += settings.ChannelAgeThreshold.Enabled ? settings.ChannelAgeThreshold.Weight : 0;
availWeight += settings.ViewCountThreshold.Enabled ? settings.ViewCountThreshold.Weight : 0;
availWeight += settings.NegativeVoteRatio.Enabled ? settings.NegativeVoteRatio.Weight : 0;
availWeight += settings.RedditAccountAgeThreshold.Enabled ? settings.RedditAccountAgeThreshold.Weight : 0;
availWeight += settings.LicensedChannel.Enabled ? settings.LicensedChannel.Weight : 0;
availWeight += settings.ImgurSubmissionRatio.Enabled ? settings.ImgurSubmissionRatio.Weight : 0;
availWeight += settings.CommentCountThreshold.Enabled ? settings.CommentCountThreshold.Weight : 0;
availWeight += settings.VoteCountThreshold.Enabled ? settings.VoteCountThreshold.Weight : 0;
var chanAgeScore = (settings.ChannelAgeThreshold.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var viewCountScore = (settings.ViewCountThreshold.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var negativeVoteScore = (settings.NegativeVoteRatio.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var redditAccountAgeScore = (settings.RedditAccountAgeThreshold.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var licensedScore = (settings.LicensedChannel.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var imgurSubmissionRatioScore = (settings.ImgurSubmissionRatio.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var commentCountScore = (settings.CommentCountThreshold.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
var totalVotesScore = (settings.VoteCountThreshold.Weight / availWeight) * MAX_MODULE_SCORE * Settings.ScoreMultiplier;
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
var channels = new Dictionary <string, List <Post> >();
req.Id = string.Join(",", youTubePosts.Keys.Skip(i).Take(50));
var response = await req.ExecuteAsync();
foreach (var vid in response.Items)
{
foreach (var post in youTubePosts[vid.Id])
{
var scores = toReturn[post.Id].Scores;
if (!channels.ContainsKey(vid.Snippet.ChannelId))
{
channels[vid.Snippet.ChannelId] = new List <Post>();
}
channels[vid.Snippet.ChannelId].Add(post);
if (settings.ViewCountThreshold.Enabled && vid.Statistics?.ViewCount.Value <= (ulong)Math.Abs(settings.ViewCountThreshold.Value)) //TODO Fix this math.abs nonsense with some validation
{
scores.Add(new AnalysisScore(viewCountScore, "View Count is below threshold", "Low Views", ModuleName));
}
if (settings.LicensedChannel.Enabled && vid.ContentDetails.LicensedContent.Value)
{
scores.Add(new AnalysisScore(licensedScore, "Channel is likely monetized", "Possibly Monetized", ModuleName));
}
if (settings.CommentCountThreshold.Enabled && vid.Statistics?.CommentCount <= (ulong)Math.Abs(settings.CommentCountThreshold.Value)) //TODO Fix this math.abs nonsense with some validation
{
scores.Add(new AnalysisScore(commentCountScore, "Number of comments is below threshold", "Low comments", ModuleName));
}
if (settings.NegativeVoteRatio.Enabled && vid.Statistics?.DislikeCount > vid.Statistics?.LikeCount)
{
scores.Add(new AnalysisScore(negativeVoteScore, "More dislikes than likes on video", ">50% dislikes", ModuleName));
}
if (settings.VoteCountThreshold.Enabled && vid.Statistics?.DislikeCount + vid.Statistics?.LikeCount <= (ulong)Math.Abs(settings.VoteCountThreshold.Value)) //TODO Fix this math.abs nonsense with some validation
{
scores.Add(new AnalysisScore(totalVotesScore, "Total vote count is below threshold", "Low Total Votes", ModuleName));
}
DateTime authorCreated = DateTime.UtcNow;
bool shadowbanned = false;
try {
authorCreated = post.Author.Created;
}
catch (WebException ex) {
if ((ex.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound)
{
authorCreated = DateTime.UtcNow;
shadowbanned = true;
}
else
{
throw;
}
}
if (settings.RedditAccountAgeThreshold.Enabled && authorCreated.AddDays(settings.RedditAccountAgeThreshold.Value) >= DateTime.UtcNow)
{
scores.Add(new AnalysisScore(redditAccountAgeScore, "Reddit Account age is below threshold", "New Reddit Acct", ModuleName));
}
if (settings.ImgurSubmissionRatio.Enabled && !shadowbanned && ((double)100 / post.Author.Posts.Take(100).Count(p => p.Domain.ToLower().Contains("imgur"))) * 100 >= settings.ImgurSubmissionRatio.Value)
{
scores.Add(new AnalysisScore(imgurSubmissionRatioScore, "User has Imgur submissions above threshold for last 100 posts", "Lots of Imgur", ModuleName));
}
}
}
if (settings.ChannelAgeThreshold.Enabled)
{
var chanReq = yt.Channels.List("snippet");
chanReq.Id = string.Join(",", channels.Keys);
var chanResponse = chanReq.Execute();
//get the channel info
foreach (var channel in chanResponse.Items)
{
//if the channel was created less than the settings.ChannelAgeThreshold days ago
DateTime channelCreationDate = channel.Snippet.PublishedAt.HasValue ? channel.Snippet.PublishedAt.Value : DateTime.UtcNow;
foreach (var post in channels[channel.Id])
{
if (channelCreationDate.AddDays(settings.ChannelAgeThreshold.Value) >= post.CreatedUTC)
{
//Add the score to the posts
toReturn[post.Id].Scores.Add(new AnalysisScore(chanAgeScore, "Channel Age Below Threshold", "Channel Age", ModuleName));
}
}
}
}
}
return(toReturn);
}
