/// <summary>
/// Identify if new scan result is a better match than previous i.e. has a higher confidence. Assumes unique list of MatchRecords.
/// </summary>
/// <param name="MatchRecords"></param>
/// <param name="compareResult"></param>
/// <param name="removeOld"></param>
/// <returns></returns>
private bool BetterMatch(List <MatchRecord> MatchRecords, MatchRecord newMatchRecord, bool removeOld = true)
{
bool betterMatch = false;
bool noMatch = true;
//if list is empty the new match is the best match
if (!MatchRecords.Any())
{
return(true);
}
MatchRecord?matchRecordToRemove = null;
foreach (MatchRecord MatchRecord in MatchRecords)
{
foreach (string matchRecordTag in MatchRecord.Rule.Tags ?? new string[] { "" })
{
foreach (string newMatchRecordTag in newMatchRecord.Rule.Tags ?? new string[] { "" })
{
if (matchRecordTag == newMatchRecordTag)
{
if (newMatchRecord.Tags.Any(x => x.Contains("AzureKeyVault")))
{
}
noMatch = false;
if (newMatchRecord.Confidence > MatchRecord.Confidence)
{
if (removeOld)
{
matchRecordToRemove = MatchRecord;
}
betterMatch = true;
break;//as this method is used with uniquematche=true only one to worry about
}
}
}
if (betterMatch)
{
break;
}
}
}
if (removeOld && matchRecordToRemove != null)
{
MatchRecords.Remove(matchRecordToRemove);//safer to remove outside for enumeration
}
return(betterMatch || noMatch);
}
/// <summary>
/// Analyzes given line of code returning matching scan results for the
/// file passed in only; Use AllResults to get results across the entire set
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
{
// Get rules for the given content type
IEnumerable <ConvertedOatRule> rules = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only
TextContainer textContainer = new TextContainer(text, languageInfo.Name);
foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
{
// If we have within captures it means we had conditions, and we only want the conditioned captures
var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause);
if (withinCaptures.Any())
{
foreach (var cap in withinCaptures)
{
ProcessBoundary(cap);
}
}
// Otherwise we can use all the captures
else
{
foreach (var cap in ruleCapture.Captures)
{
ProcessBoundary(cap);
}
}
void ProcessBoundary(ClauseCapture cap)
{
List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only
if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
{
if (ruleCapture.Rule is ConvertedOatRule oatRule)
{
if (tcc?.Result is List <(int, Boundary)> captureResults)
{
foreach (var match in captureResults)
{
var patternIndex = match.Item1;
var boundary = match.Item2;
//restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
{
_logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
continue;
}
if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
{
continue;
}
Location StartLocation = textContainer.GetLocation(boundary.Index);
Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length);
MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule)
{
FileName = filePath,
FullText = textContainer.FullContent,
LanguageInfo = languageInfo,
Boundary = boundary,
StartLocationLine = StartLocation.Line,
EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex],
Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
};
bool addNewRecord = true;
if (_uniqueTagMatchesOnly)
{
if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags))
{
if (_stopAfterFirstMatch)
{
addNewRecord = false; //we've seen already; don't improve the match
}
else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match
{
addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch);
if (addNewRecord)
{
lock (_controllRunningListAdd)
{
addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files
}
}
}
}
}
if (addNewRecord)
{
newMatches.Add(newMatch);
lock (_controllRunningListAdd)
{
_runningResultsList.Add(newMatch);
}
}
AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" });
}
}
}
}
resultsList.AddRange(newMatches);
}
}
if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
{
// Deal with overrides
List <MatchRecord> removes = new List <MatchRecord>();
foreach (MatchRecord m in resultsList)
{
if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
{
foreach (string ovrd in m.Rule.Overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
{
if (om.Boundary?.Index >= m.Boundary?.Index &&
om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
{
removes.Add(om);
}
}
}
}
}
// Remove overriden rules
resultsList.RemoveAll(x => removes.Contains(x));
}
return(resultsList.ToArray());
}
/// <summary>
/// Analyzes given line of code returning matching scan results for the
/// file passed in only; Use AllResults to get results across the entire set
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
{
// Get rules for the given content type
var rulesByLanguage = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
var rules = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)));
List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only
TextContainer textContainer = new TextContainer(text, languageInfo.Name);
foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
{
foreach (var cap in ruleCapture.Captures)
{
ProcessBoundary(cap);
}
void ProcessBoundary(ClauseCapture cap)
{
List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only
if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
{
if (ruleCapture.Rule is ConvertedOatRule oatRule)
{
if (tcc?.Result is List <(int, Boundary)> captureResults)
{
foreach (var match in captureResults)
{
var patternIndex = match.Item1;
var boundary = match.Item2;
//restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
{
_logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
continue;
}
if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
{
continue;
}
Location StartLocation = textContainer.GetLocation(boundary.Index);
Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length);
MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule)
{
FileName = filePath,
FullTextContainer = textContainer,
LanguageInfo = languageInfo,
Boundary = boundary,
StartLocationLine = StartLocation.Line,
EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex],
Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
};
newMatches.Add(newMatch);
if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any())
{
AddRuleTagHashes(oatRule.AppInspectorRule.Tags);
}
}
}
}
}
resultsList.AddRange(newMatches);
}
}
if (_uniqueTagMatchesOnly)
{
var replacementList = new List <MatchRecord>();
foreach (var entry in resultsList)
{
if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags))
{
if (!_stopAfterFirstMatch)
{
var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y)));
if (replaceable.Any())
{
replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList();
if (replaceable.Any())
{
replacementList.RemoveAll(x => replaceable.Contains(x));
replacementList.Add(entry);
}
}
else
{
replacementList.Add(entry);
}
}
}
else
{
replacementList.Add(entry);
}
}
resultsList = replacementList;
}
List <MatchRecord> removes = new List <MatchRecord>();
foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
{
if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
{
foreach (string ovrd in m.Rule.Overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
{
if (om.Boundary?.Index >= m.Boundary?.Index &&
om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
{
removes.Add(om);
}
}
}
}
// Remove overriden rules
}
resultsList.RemoveAll(x => removes.Contains(x));
foreach (var entry in resultsList)
{
_runningResultsList.Enqueue(entry);
}
return(resultsList.ToArray());
}