/// <summary> /// Identify if new scan result is a better match than previous i.e. has a higher confidence. Assumes unique list of MatchRecords. /// </summary> /// <param name="MatchRecords"></param> /// <param name="compareResult"></param> /// <param name="removeOld"></param> /// <returns></returns> private bool BetterMatch(List <MatchRecord> MatchRecords, MatchRecord newMatchRecord, bool removeOld = true) { bool betterMatch = false; bool noMatch = true; //if list is empty the new match is the best match if (!MatchRecords.Any()) { return(true); } MatchRecord?matchRecordToRemove = null; foreach (MatchRecord MatchRecord in MatchRecords) { foreach (string matchRecordTag in MatchRecord.Rule.Tags ?? new string[] { "" }) { foreach (string newMatchRecordTag in newMatchRecord.Rule.Tags ?? new string[] { "" }) { if (matchRecordTag == newMatchRecordTag) { if (newMatchRecord.Tags.Any(x => x.Contains("AzureKeyVault"))) { } noMatch = false; if (newMatchRecord.Confidence > MatchRecord.Confidence) { if (removeOld) { matchRecordToRemove = MatchRecord; } betterMatch = true; break;//as this method is used with uniquematche=true only one to worry about } } } if (betterMatch) { break; } } } if (removeOld && matchRecordToRemove != null) { MatchRecords.Remove(matchRecordToRemove);//safer to remove outside for enumeration } return(betterMatch || noMatch); }
/// <summary> /// Analyzes given line of code returning matching scan results for the /// file passed in only; Use AllResults to get results across the entire set /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo) { // Get rules for the given content type IEnumerable <ConvertedOatRule> rules = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)); List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only TextContainer textContainer = new TextContainer(text, languageInfo.Name); foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer)) { // If we have within captures it means we had conditions, and we only want the conditioned captures var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause); if (withinCaptures.Any()) { foreach (var cap in withinCaptures) { ProcessBoundary(cap); } } // Otherwise we can use all the captures else { foreach (var cap in ruleCapture.Captures) { ProcessBoundary(cap); } } void ProcessBoundary(ClauseCapture cap) { List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc) { if (ruleCapture.Rule is ConvertedOatRule oatRule) { if (tcc?.Result is List <(int, Boundary)> captureResults) { foreach (var match in captureResults) { var patternIndex = match.Item1; var boundary = match.Item2; //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata"))) { continue; } if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length) { _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name); continue; } if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence)) { continue; } Location StartLocation = textContainer.GetLocation(boundary.Index); Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length); MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule) { FileName = filePath, FullText = textContainer.FullContent, LanguageInfo = languageInfo, Boundary = boundary, StartLocationLine = StartLocation.Line, EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex], Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line), Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length) }; bool addNewRecord = true; if (_uniqueTagMatchesOnly) { if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags)) { if (_stopAfterFirstMatch) { addNewRecord = false; //we've seen already; don't improve the match } else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match { addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch); if (addNewRecord) { lock (_controllRunningListAdd) { addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files } } } } } if (addNewRecord) { newMatches.Add(newMatch); lock (_controllRunningListAdd) { _runningResultsList.Add(newMatch); } } AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" }); } } } } resultsList.AddRange(newMatches); } } if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0)) { // Deal with overrides List <MatchRecord> removes = new List <MatchRecord>(); foreach (MatchRecord m in resultsList) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary?.Index >= m.Boundary?.Index && om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length) { removes.Add(om); } } } } } // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); } return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code returning matching scan results for the /// file passed in only; Use AllResults to get results across the entire set /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo) { // Get rules for the given content type var rulesByLanguage = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)); var rules = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity))); List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only TextContainer textContainer = new TextContainer(text, languageInfo.Name); foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer)) { foreach (var cap in ruleCapture.Captures) { ProcessBoundary(cap); } void ProcessBoundary(ClauseCapture cap) { List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc) { if (ruleCapture.Rule is ConvertedOatRule oatRule) { if (tcc?.Result is List <(int, Boundary)> captureResults) { foreach (var match in captureResults) { var patternIndex = match.Item1; var boundary = match.Item2; //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata"))) { continue; } if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length) { _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name); continue; } if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence)) { continue; } Location StartLocation = textContainer.GetLocation(boundary.Index); Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length); MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule) { FileName = filePath, FullTextContainer = textContainer, LanguageInfo = languageInfo, Boundary = boundary, StartLocationLine = StartLocation.Line, EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex], Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line), Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length) }; newMatches.Add(newMatch); if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any()) { AddRuleTagHashes(oatRule.AppInspectorRule.Tags); } } } } } resultsList.AddRange(newMatches); } } if (_uniqueTagMatchesOnly) { var replacementList = new List <MatchRecord>(); foreach (var entry in resultsList) { if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags)) { if (!_stopAfterFirstMatch) { var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y))); if (replaceable.Any()) { replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList(); if (replaceable.Any()) { replacementList.RemoveAll(x => replaceable.Contains(x)); replacementList.Add(entry); } } else { replacementList.Add(entry); } } } else { replacementList.Add(entry); } } resultsList = replacementList; } List <MatchRecord> removes = new List <MatchRecord>(); foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0)) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary?.Index >= m.Boundary?.Index && om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length) { removes.Add(om); } } } } // Remove overriden rules } resultsList.RemoveAll(x => removes.Contains(x)); foreach (var entry in resultsList) { _runningResultsList.Enqueue(entry); } return(resultsList.ToArray()); }