/// <summary> /// Analyzes given line of code /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public Issue[] Analyze(string text, string[] languages) { // Get rules for the given content type IEnumerable <Rule> rules = GetRulesForLanguages(languages); List <Issue> resultsList = new List <Issue>(); // Go through each rule foreach (Rule r in rules) { List <Issue> matchList = new List <Issue>(); // Skip rules that don't apply based on settings if (r.Disabled || !SeverityLevel.HasFlag(r.Severity)) { continue; } // Go through each matching pattern of the rule foreach (SearchPattern p in r.Patterns) { RegexOptions reopt = RegexOptions.None; if (p.Modifiers != null && p.Modifiers.Length > 0) { reopt |= (p.Modifiers.Contains("IGNORECASE")) ? RegexOptions.IgnoreCase : RegexOptions.None; reopt |= (p.Modifiers.Contains("MULTILINE")) ? RegexOptions.Multiline : RegexOptions.None; } Regex patRegx = new Regex(p.Pattern, reopt); MatchCollection matches = patRegx.Matches(text); if (matches.Count > 0) { foreach (System.Text.RegularExpressions.Match m in matches) { matchList.Add(new Issue() { Index = m.Index, Length = m.Length, Rule = r }); } break; // from pattern loop } } // We got matching rule and suppression are enabled, // let's see if we have a supression on the line if (EnableSuppressions && matchList.Count > 0) { Suppression supp = new Suppression(text); foreach (Issue result in matchList) { // If rule is NOT being suppressed then useit if (!supp.IsIssueSuppressed(result.Rule.Id)) { resultsList.Add(result); } } } // Otherwise put matchlist to resultlist else { resultsList.AddRange(matchList); } } // Deal with overrides List <Issue> removes = new List <Issue>(); foreach (Issue m in resultsList) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (Issue om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (m.Index == om.Index) { removes.Add(om); } } } } } // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public ScanResult[] Analyze(string text, LanguageInfo languageInfo) { string[] languages = new string[] { languageInfo.Name }; // Get rules for the given content type IEnumerable <Rule> rules = GetRulesForLanguages(languages); List <ScanResult> resultsList = new List <ScanResult>(); TextContainer textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch); // Go through each rule foreach (Rule rule in rules) { if (_logger != null) { _logger.Debug("Processing for rule: " + rule.Id); } // Skip rules that don't apply based on settings if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity)) { continue; } // Skip further processing of rule for efficiency if user requested first match only (speed/quality) if (_stopAfterFirstPatternMatch && _uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags)) { continue; } List <ScanResult> matchList = new List <ScanResult>(); // Go through each matching pattern of the rule foreach (SearchPattern pattern in rule.Patterns) { //Skill patterns that don't apply based on settings if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence)) { continue; } // Process all matches for the patttern (this may be only 1 is _stopAfterFirstMatch is set foreach (Boundary match in textContainer.EnumerateMatchingBoundaries(pattern)) { bool passedConditions = true; foreach (SearchCondition condition in rule.Conditions) { bool res = textContainer.IsPatternMatch(condition.Pattern, match, condition); if (res && condition.NegateFinding) { passedConditions = false; break; } if (!res && condition.NegateFinding) { passedConditions = true; break; } if (!res) { passedConditions = false; break; } } //restrict tags from build files to tags with "metadata" to avoid false feature positives that are not part of executable code if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata"))) { passedConditions = false; } if (passedConditions) { ScanResult newMatch = new ScanResult() { Boundary = match, StartLocation = textContainer.GetLocation(match.Index), EndLocation = textContainer.GetLocation(match.Index + match.Length), PatternMatch = pattern, Confidence = pattern.Confidence, Rule = rule }; if (_uniqueTagMatchesOnly) { if (!UniqueTagsCheck(newMatch.Rule.Tags)) //tag(s) previously seen { if (_stopAfterFirstPatternMatch) //recheck stop at pattern level also within same rule { passedConditions = false; //user performance option i.e. only wants to identify if tag is detected nothing more } else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match { passedConditions = BestMatch(matchList, newMatch); //; check all patterns in current rule if (passedConditions) { passedConditions = BestMatch(resultsList, newMatch);//check all rules in permanent list } } } } if (passedConditions) { matchList.Add(newMatch); } AddRuleTagHashes(rule.Tags); } } } resultsList.AddRange(matchList); } // Deal with overrides List <ScanResult> removes = new List <ScanResult>(); foreach (ScanResult scanResult in resultsList) { if (scanResult.Rule.Overrides is string[] overrides) { foreach (string @override in overrides) { // Find all overriden rules and mark them for removal from issues list foreach (ScanResult overRideMatch in resultsList.FindAll(x => x.Rule.Id == @override)) { if (overRideMatch.Boundary.Index >= scanResult.Boundary.Index && overRideMatch.Boundary.Index <= scanResult.Boundary.Index + scanResult.Boundary.Length) { removes.Add(overRideMatch); } } } } } if (removes.Count > 0) { // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); } return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code returning matching scan results for the /// file passed in only; Use AllResults to get results across the entire set /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo) { // Get rules for the given content type IEnumerable <ConvertedOatRule> rules = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)); List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only TextContainer textContainer = new TextContainer(text, languageInfo.Name); foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer)) { // If we have within captures it means we had conditions, and we only want the conditioned captures var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause); if (withinCaptures.Any()) { foreach (var cap in withinCaptures) { ProcessBoundary(cap); } } // Otherwise we can use all the captures else { foreach (var cap in ruleCapture.Captures) { ProcessBoundary(cap); } } void ProcessBoundary(ClauseCapture cap) { List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc) { if (ruleCapture.Rule is ConvertedOatRule oatRule) { if (tcc?.Result is List <(int, Boundary)> captureResults) { foreach (var match in captureResults) { var patternIndex = match.Item1; var boundary = match.Item2; //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata"))) { continue; } if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length) { _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name); continue; } if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence)) { continue; } Location StartLocation = textContainer.GetLocation(boundary.Index); Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length); MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule) { FileName = filePath, FullText = textContainer.FullContent, LanguageInfo = languageInfo, Boundary = boundary, StartLocationLine = StartLocation.Line, EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex], Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line), Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length) }; bool addNewRecord = true; if (_uniqueTagMatchesOnly) { if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags)) { if (_stopAfterFirstMatch) { addNewRecord = false; //we've seen already; don't improve the match } else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match { addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch); if (addNewRecord) { lock (_controllRunningListAdd) { addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files } } } } } if (addNewRecord) { newMatches.Add(newMatch); lock (_controllRunningListAdd) { _runningResultsList.Add(newMatch); } } AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" }); } } } } resultsList.AddRange(newMatches); } } if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0)) { // Deal with overrides List <MatchRecord> removes = new List <MatchRecord>(); foreach (MatchRecord m in resultsList) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary?.Index >= m.Boundary?.Index && om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length) { removes.Add(om); } } } } } // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); } return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public Issue[] Analyze(string text, string[] languages) { // Get rules for the given content type IEnumerable <Rule> rules = GetRulesForLanguages(languages); List <Issue> resultsList = new List <Issue>(); TextContainer textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty); // Go through each rule foreach (Rule rule in rules) { List <Issue> matchList = new List <Issue>(); // Skip rules that don't apply based on settings if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity)) { continue; } // Go through each matching pattern of the rule foreach (SearchPattern pattern in rule.Patterns) { // Get all matches for the patttern List <Boundary> matches = textContainer.MatchPattern(pattern); if (matches.Count > 0) { foreach (Boundary match in matches) { bool passedConditions = true; foreach (SearchCondition condition in rule.Conditions) { bool res = textContainer.MatchPattern(condition.Pattern, match, condition); if (res && condition.NegateFinding) { passedConditions = false; break; } if (!res && condition.NegateFinding) { passedConditions = true; break; } if (!res) { passedConditions = false; break; } } if (passedConditions) { Issue issue = new Issue() { Boundary = match, StartLocation = textContainer.GetLocation(match.Index), EndLocation = textContainer.GetLocation(match.Index + match.Length), Rule = rule }; matchList.Add(issue); } } } } // We got matching rule and suppression are enabled, // let's see if we have a supression on the line if (EnableSuppressions && matchList.Count > 0) { Suppression supp; foreach (Issue result in matchList) { supp = new Suppression(textContainer.GetLineContent(result.StartLocation.Line)); // If rule is NOT being suppressed then report it SuppressedIssue supissue = supp.GetSuppressedIssue(result.Rule.Id); if (supissue == null) { resultsList.Add(result); } // Otherwise add the suppression info instead else { Boundary bound = textContainer.GetLineBoundary(result.Boundary.Index); bound.Index += supissue.Boundary.Index; bound.Length = supissue.Boundary.Length; //resultsList.Add(); Issue info = new Issue() { IsSuppressionInfo = true, Boundary = bound, StartLocation = textContainer.GetLocation(bound.Index), EndLocation = textContainer.GetLocation(bound.Index + bound.Length), Rule = result.Rule }; // Add info only if it's not exists on the same location if (resultsList.FirstOrDefault(x => x.Rule.Id == info.Rule.Id && x.Boundary.Index == info.Boundary.Index) == null) { resultsList.Add(info); } } } } // Otherwise put matchlist to resultlist else { resultsList.AddRange(matchList); } } // Deal with overrides List <Issue> removes = new List <Issue>(); foreach (Issue m in resultsList) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (Issue om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary.Index >= m.Boundary.Index && om.Boundary.Index <= m.Boundary.Index + m.Boundary.Length) { removes.Add(om); } } } } } // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code returning matching scan results for the /// file passed in only; Use AllResults to get results across the entire set /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo) { // Get rules for the given content type var rulesByLanguage = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)); var rules = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity))); List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only TextContainer textContainer = new TextContainer(text, languageInfo.Name); foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer)) { foreach (var cap in ruleCapture.Captures) { ProcessBoundary(cap); } void ProcessBoundary(ClauseCapture cap) { List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc) { if (ruleCapture.Rule is ConvertedOatRule oatRule) { if (tcc?.Result is List <(int, Boundary)> captureResults) { foreach (var match in captureResults) { var patternIndex = match.Item1; var boundary = match.Item2; //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata"))) { continue; } if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length) { _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name); continue; } if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence)) { continue; } Location StartLocation = textContainer.GetLocation(boundary.Index); Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length); MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule) { FileName = filePath, FullTextContainer = textContainer, LanguageInfo = languageInfo, Boundary = boundary, StartLocationLine = StartLocation.Line, EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex], Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line), Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length) }; newMatches.Add(newMatch); if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any()) { AddRuleTagHashes(oatRule.AppInspectorRule.Tags); } } } } } resultsList.AddRange(newMatches); } } if (_uniqueTagMatchesOnly) { var replacementList = new List <MatchRecord>(); foreach (var entry in resultsList) { if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags)) { if (!_stopAfterFirstMatch) { var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y))); if (replaceable.Any()) { replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList(); if (replaceable.Any()) { replacementList.RemoveAll(x => replaceable.Contains(x)); replacementList.Add(entry); } } else { replacementList.Add(entry); } } } else { replacementList.Add(entry); } } resultsList = replacementList; } List <MatchRecord> removes = new List <MatchRecord>(); foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0)) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary?.Index >= m.Boundary?.Index && om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length) { removes.Add(om); } } } } // Remove overriden rules } resultsList.RemoveAll(x => removes.Contains(x)); foreach (var entry in resultsList) { _runningResultsList.Enqueue(entry); } return(resultsList.ToArray()); }
/// <summary> /// Analyzes given line of code /// </summary> /// <param name="text">Source code</param> /// <param name="languages">List of languages</param> /// <returns>Array of matches</returns> public Issue[] Analyze(string text, LanguageInfo languageInfo) { string[] languages = new string[] { languageInfo.Name }; // Get rules for the given content type IEnumerable <Rule> rules = GetRulesForLanguages(languages); List <Issue> resultsList = new List <Issue>(); TextContainer textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch); // Go through each rule foreach (Rule rule in rules) { if (_logger != null) { _logger.Debug("Processing for rule: " + rule.Id); } // Skip pattern matching this rule if uniquetag option and not in exceptions list bool multipleMatchesOk = !_uniqueTagMatchesOnly || UniqueTagsCheck(rule.Tags); if (!multipleMatchesOk) { continue; } List <Issue> matchList = new List <Issue>(); // Skip rules that don't apply based on settings if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity)) { continue; } // Go through each matching pattern of the rule foreach (SearchPattern pattern in rule.Patterns) { //Skill patterns that don't apply based on settings if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence)) { continue; } // Get all matches for the patttern List <Boundary> matches = textContainer.MatchPattern(pattern); if (matches.Count > 0) { foreach (Boundary match in matches) { bool passedConditions = true; foreach (SearchCondition condition in rule.Conditions) { bool res = textContainer.MatchPattern(condition.Pattern, match, condition); if (res && condition.NegateFinding) { passedConditions = false; break; } if (!res && condition.NegateFinding) { passedConditions = true; break; } if (!res) { passedConditions = false; break; } } //do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata"))) { passedConditions = false; } if (passedConditions) { Issue issue = new Issue() { Boundary = match, StartLocation = textContainer.GetLocation(match.Index), EndLocation = textContainer.GetLocation(match.Index + match.Length), PatternMatch = pattern, Confidence = pattern.Confidence, Rule = rule }; //check at pattern level to avoid adding duplicates if (_uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags)) { break; } AddRuleTagHashes(rule.Tags); matchList.Add(issue); } } } } // We got matching rule and suppression are enabled, // let's see if we have a supression on the line if (EnableSuppressions && matchList.Count > 0) { Suppression supp; foreach (Issue result in matchList) { supp = new Suppression(textContainer.GetLineContent(result.StartLocation.Line)); // If rule is NOT being suppressed then report it SuppressedIssue supissue = supp.GetSuppressedIssue(result.Rule.Id); if (supissue == null) { resultsList.Add(result); } // Otherwise add the suppression info instead else { Boundary bound = textContainer.GetLineBoundary(result.Boundary.Index); bound.Index += supissue.Boundary.Index; bound.Length = supissue.Boundary.Length; //resultsList.Add(); Issue info = new Issue() { IsSuppressionInfo = true, Boundary = bound, StartLocation = textContainer.GetLocation(bound.Index), EndLocation = textContainer.GetLocation(bound.Index + bound.Length), Rule = result.Rule }; // Add info only if it's not on the same location if (resultsList.FirstOrDefault(x => x.Rule.Id == info.Rule.Id && x.Boundary.Index == info.Boundary.Index) == null) { resultsList.Add(info); } else if (_logger != null) { _logger.Debug("Not added due to proximity to another rule"); } } } } // Otherwise put matchlist to resultlist else { resultsList.AddRange(matchList); } } // Deal with overrides List <Issue> removes = new List <Issue>(); foreach (Issue m in resultsList) { if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0) { foreach (string ovrd in m.Rule.Overrides) { // Find all overriden rules and mark them for removal from issues list foreach (Issue om in resultsList.FindAll(x => x.Rule.Id == ovrd)) { if (om.Boundary.Index >= m.Boundary.Index && om.Boundary.Index <= m.Boundary.Index + m.Boundary.Length) { removes.Add(om); } } } } } // Remove overriden rules resultsList.RemoveAll(x => removes.Contains(x)); return(resultsList.ToArray()); }