private AnalysisScore ScoreYouTubeMetaData(string pageHtml)
{
var doc = new HtmlDocument();
doc.LoadHtml(pageHtml);
var nodes = doc.DocumentNode.SelectNodes("/html/head/meta[@name=\"attribution\"]");
AnalysisScore score = null;
if (nodes != null && nodes.Count > 0)
{
var node = nodes.First();
var owner = node.GetAttributeValue("content", "");
if (owner.Substring(owner.Length - 1) == "/")
{
owner = owner.Substring(0, owner.Length - 1);
}
var match = LicenserMatching.Match(owner).Value;
score = new AnalysisScore(ATTRIBUTION_SCORE * Settings.ScoreMultiplier, string.Format("Video is monetized by '{0}'", owner), "Monetized", ModuleName);
if (!string.IsNullOrEmpty(match))
{
score = new AnalysisScore(ATTRIBUTION_MATCH_SCORE * Settings.ScoreMultiplier, string.Format("Video is licensed through a network : '{0}'", KnownLicensers[match]), string.Format("Video licensed by '{0}'", KnownLicensers[match]), ModuleName, RemovalFlair);
return(score);
}
}
return(score);
}
public async Task <Dictionary <string, PostAnalysisResults> > Analyze(List <Post> posts)
{
var toReturn = new Dictionary <string, PostAnalysisResults>();
foreach (var post in posts) //TODO error handling
{
var youTubePosts = new Dictionary <string, List <Post> >();
toReturn.Add(post.Id, new PostAnalysisResults(post, ModuleEnum));
string postYTID = YouTubeHelpers.ExtractVideoId(post.Url.ToString());
Task <Logging.UserPostingHistory> hist;
if (!string.IsNullOrEmpty(postYTID))
{
//It's a YouTube vid so we can kick off the analysis and get cookin
hist = Logging.UserPostingHistory.GetUserPostingHistory(post.AuthorName);
if (!youTubePosts.ContainsKey(postYTID))
{
youTubePosts.Add(postYTID, new List <Post>());
}
youTubePosts[postYTID].Add(post);
}
else
{
//not a YouTube post, so bail out
continue;
}
bool success = false;
int nonYTPosts = 0;
int tries = 0;
while (!success && tries < 3)
{
success = true;
try {
var recentPosts = RedditClient.Search <RedditSharp.Things.Post>($"author:{post.AuthorName} self:no", RedditSharp.Sorting.New).GetListing(100, 100);
foreach (var recentPost in recentPosts)
{
string ytID = YouTubeHelpers.ExtractVideoId(recentPost.Url.ToString());
if (!string.IsNullOrEmpty(ytID))
{
if (!youTubePosts.ContainsKey(ytID))
{
youTubePosts.Add(ytID, new List <Post>());
}
youTubePosts[ytID].Add(post);
}
else
{
nonYTPosts++;
}
}
}
catch (Exception ex) {
success = false;
tries++;
if (tries > 3)
{
Console.WriteLine($"Failed to get search results: {ex.Message}");
processedCache.Remove(post.Id);
break;
}
await Task.Delay(100);
}
}
if (tries > 3)
{
continue;
}
var yt = new YouTubeService(new BaseClientService.Initializer {
ApiKey = YouTubeAPIKey
});
Dictionary <string, List <string> > postHistory = ( await hist ).PostingHistory;
string postChannelID = "";
string postChannelName = "";
var req = yt.Videos.List("snippet");
for (var i = 0; i < youTubePosts.Keys.Count; i += 50)
{
req.Id = string.Join(",", youTubePosts.Keys.Skip(i).Take(50));
var response = await req.ExecuteAsync();
foreach (var vid in response.Items)
{
foreach (var ytPost in youTubePosts[vid.Id])
{
if (!postHistory.ContainsKey(vid.Snippet.ChannelId))
{
postHistory.Add(vid.Snippet.ChannelId, new List <string>());
}
//check to see if it already exists (aka wasnt deleted and showed up in search results)
if (!postHistory[vid.Snippet.ChannelId].Contains(ytPost.Id))
{
postHistory[vid.Snippet.ChannelId].Add(ytPost.Id);
}
if (vid.Id == postYTID)
{
postChannelID = vid.Snippet.ChannelId;
postChannelName = vid.Snippet.ChannelTitle;
}
}
}
}
if (string.IsNullOrEmpty(postChannelID))
{
//shouldn't ever happen, but might if the video is deleted or the channel deleted or something
Console.WriteLine($"Channel for post {post.Id} by {post.AuthorName} couldn't be found");
continue;
}
int totalPosts = postHistory.Sum(ph => ph.Value.Count) + nonYTPosts;
int channelPosts = postHistory[postChannelID].Count;
if (!IncludePostInPercentage)
{
totalPosts--;
channelPosts--;
postHistory[postChannelID].Remove(post.Id);
}
double percent = ((double)channelPosts / totalPosts) * 100;
if (percent > PercentageThreshold && channelPosts > GracePeriod)
{
var score = new AnalysisScore();
score.ModuleName = "SelfPromotionCombustor";
score.ReportReason = $"SelfPromo: {Math.Round( percent, 2 )}%";
score.Reason = $"Self Promotion for channel '{postChannelName}' with a posting percentage of {Math.Round( percent, 2 )}. Found PostIDs: {string.Join( ", ", postHistory[postChannelID] )}";
score.Score = OVER_PERCENT_SCORE * Settings.ScoreMultiplier;
score.RemovalFlair = RemovalFlair;
toReturn[post.Id].Scores.Add(score);
}
}
return(toReturn);
}
