public void MatchRoute(INode n, long Key) { if (n is null) { throw new ArgumentNullException(nameof(n)); } if (Key == 0) { throw new ArgumentException("Can not match node with key = 0", nameof(n)); } if (!this.MatchedRoutes.ContainsKey(Key)) { this.MatchedRoutes.TryAdd(Key, n); if (this.AnalysisResults.ColumnInstances.ContainsKey(Key)) { if (!this.Scores.TryGetValue(Key, out AnalysisScore score)) { score = new AnalysisScore(); this.Scores.TryAdd(Key, score); } score.Value = n.GetScore(this.Result.BaseRate); score.OldCount = (double)this.AnalysisResults.GraphInstances / this.AnalysisResults.ColumnInstances[Key]; score.OldValue = n.GetScore(this.Result.BaseRate); score.ColumnInstances++; LongByte lb = new LongByte(Key); while (lb > 0) { if (!this.Scores.TryGetValue(lb, out AnalysisScore s)) { s = new AnalysisScore(); this.Scores.TryAdd(lb, s); } s.ColumnInstances++; lb.TrimLeft(); } } } }
public async Task<Dictionary<string, PostAnalysisResults>> Analyze( List<Post> posts ) { var toReturn = new Dictionary<string, PostAnalysisResults>(); foreach ( var post in posts ) { //TODO error handling var youTubePosts = new Dictionary<string, List<Post>>(); toReturn.Add( post.Id, new PostAnalysisResults( post, ModuleEnum ) ); string postYTID = YouTubeHelpers.ExtractVideoId( post.Url.ToString() ); Task<Logging.UserPostingHistory> hist; if ( !string.IsNullOrEmpty( postYTID ) ) { //It's a YouTube vid so we can kick off the analysis and get cookin hist = Logging.UserPostingHistory.GetUserPostingHistory( post.AuthorName ); if ( !youTubePosts.ContainsKey( postYTID ) ) youTubePosts.Add( postYTID, new List<Post>() ); youTubePosts[postYTID].Add( post ); } else { //not a YouTube post, so bail out continue; } bool success = false; int nonYTPosts = 0; int tries = 0; while ( !success && tries < 3 ) { success = true; try { var recentPosts = RedditClient.Search<RedditSharp.Things.Post>( $"author:{post.AuthorName} self:no", RedditSharp.Sorting.New ).GetListing( 100, 100 ); foreach ( var recentPost in recentPosts ) { string ytID = YouTubeHelpers.ExtractVideoId( recentPost.Url.ToString() ); if ( !string.IsNullOrEmpty( ytID ) ) { if ( !youTubePosts.ContainsKey( ytID ) ) youTubePosts.Add( ytID, new List<Post>() ); youTubePosts[ytID].Add( post ); } else { nonYTPosts++; } } } catch(Exception ex) { success = false; tries++; if(tries > 3 ) { Console.WriteLine( $"Failed to get search results: {ex.Message}" ); processedCache.Remove( post.Id ); break; } await Task.Delay( 100 ); } } if ( tries > 3 ) { continue; } var yt = new YouTubeService( new BaseClientService.Initializer { ApiKey = YouTubeAPIKey } ); Dictionary<string, List<string>> postHistory = ( await hist ).PostingHistory; string postChannelID = ""; string postChannelName = ""; var req = yt.Videos.List( "snippet" ); for ( var i = 0; i < youTubePosts.Keys.Count; i += 50 ) { req.Id = string.Join( ",", youTubePosts.Keys.Skip( i ).Take( 50 ) ); var response = await req.ExecuteAsync(); foreach ( var vid in response.Items ) { foreach ( var ytPost in youTubePosts[vid.Id] ) { if ( !postHistory.ContainsKey( vid.Snippet.ChannelId ) ) postHistory.Add( vid.Snippet.ChannelId, new List<string>() ); //check to see if it already exists (aka wasnt deleted and showed up in search results) if ( !postHistory[vid.Snippet.ChannelId].Contains( ytPost.Id ) ) postHistory[vid.Snippet.ChannelId].Add( ytPost.Id ); if ( vid.Id == postYTID ) { postChannelID = vid.Snippet.ChannelId; postChannelName = vid.Snippet.ChannelTitle; } } } } if ( string.IsNullOrEmpty( postChannelID ) ) { //shouldn't ever happen, but might if the video is deleted or the channel deleted or something Console.WriteLine( $"Channel for post {post.Id} by {post.AuthorName} couldn't be found" ); continue; } int totalPosts = postHistory.Sum( ph => ph.Value.Count ) + nonYTPosts; int channelPosts = postHistory[postChannelID].Count; if ( !IncludePostInPercentage ) { totalPosts--; channelPosts--; postHistory[postChannelID].Remove( post.Id ); } double percent = ( (double) channelPosts / totalPosts ) * 100; if ( percent > PercentageThreshold && channelPosts > GracePeriod ) { var score = new AnalysisScore(); score.ModuleName = "SelfPromotionCombustor"; score.ReportReason = $"SelfPromo: {Math.Round( percent, 2 )}%"; score.Reason = $"Self Promotion for channel '{postChannelName}' with a posting percentage of {Math.Round( percent, 2 )}. Found PostIDs: {string.Join( ", ", postHistory[postChannelID] )}"; score.Score = OVER_PERCENT_SCORE * Settings.ScoreMultiplier; score.RemovalFlair = RemovalFlair; toReturn[post.Id].Scores.Add( score ); } } return toReturn; }
private AnalysisScore ScoreYouTubeMetaData( string pageHtml ) { var doc = new HtmlDocument(); doc.LoadHtml( pageHtml ); var nodes = doc.DocumentNode.SelectNodes( "/html/head/meta[@name=\"attribution\"]" ); AnalysisScore score = null; if ( nodes != null && nodes.Count > 0 ) { var node = nodes.First(); var owner = node.GetAttributeValue( "content", "" ); if ( owner.Substring( owner.Length - 1 ) == "/" ) owner = owner.Substring( 0, owner.Length - 1 ); var match = LicenserMatching.Match( owner ).Value; score = new AnalysisScore( ATTRIBUTION_SCORE * Settings.ScoreMultiplier, string.Format( "Video is monetized by '{0}'", owner ), "Monetized", ModuleName ); if ( !string.IsNullOrEmpty( match ) ) { score = new AnalysisScore( ATTRIBUTION_MATCH_SCORE * Settings.ScoreMultiplier, string.Format( "Video is licensed through a network : '{0}'", KnownLicensers[match] ), string.Format( "Video licensed by '{0}'", KnownLicensers[match] ), ModuleName, RemovalFlair ); return score; } } return score; }