/// <summary>
/// Returns language for given file name
/// </summary>
/// <param name="fileName">File name</param>
/// <returns>Language</returns>
public static bool FromFileName(string fileName, ref LanguageInfo info)
{
if (fileName == null)
{
return(false);
}
string file = Path.GetFileName(fileName).ToLower(CultureInfo.CurrentCulture);
string ext = Path.GetExtension(file);
// Look for whole filename first
foreach (LanguageInfo item in Instance.Languages)
{
if (item.Name == file)
{
info = item;
return(true);
}
}
// Look for extension only ext is defined
if (!string.IsNullOrEmpty(ext))
{
foreach (LanguageInfo item in Instance.Languages)
{
if (item.Name == "typescript-config")//special case where extension used for exact match to a single type
{
if (item.Extensions?.Any(x => x.ToLower().Equals(file)) ?? false)
{
info = item;
return(true);
}
}
else if (Array.Exists(item.Extensions ?? Array.Empty <string>(), x => x.EndsWith(ext, StringComparison.InvariantCultureIgnoreCase)))
{
info = item;
return(true);
}
}
}
return(false);
}
/// <summary>
/// Returns language for given file name
/// </summary>
/// <param name="fileName">File name</param>
/// <returns>Language</returns>
public static bool FromFileName(string fileName, ref LanguageInfo info)
{
if (fileName == null)
{
return(false);
}
bool result = false;
string file = Path.GetFileName(fileName).ToLower(CultureInfo.CurrentCulture);
string ext = Path.GetExtension(file);
// Look for whole filename first
foreach (LanguageInfo item in Instance.Languages)
{
if (item.Name == file)
{
info = item;
return(true);
}
}
// Look for extension only ext is defined
if (!string.IsNullOrEmpty(ext))
{
foreach (LanguageInfo item in Instance.Languages)
{
if (Array.Exists(item.Extensions, x => x.EndsWith(ext)))
{
info = item;
return(true);
}
}
}
return(result);
}
/// <summary>
/// Analyzes given line of code
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public ScanResult[] Analyze(string text, LanguageInfo languageInfo)
{
string[] languages = new string[] { languageInfo.Name };
// Get rules for the given content type
IEnumerable <Rule> rules = GetRulesForLanguages(languages);
List <ScanResult> resultsList = new List <ScanResult>();
TextContainer textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch);
// Go through each rule
foreach (Rule rule in rules)
{
if (_logger != null)
{
_logger.Debug("Processing for rule: " + rule.Id);
}
// Skip rules that don't apply based on settings
if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity))
{
continue;
}
// Skip further processing of rule for efficiency if user requested first match only (speed/quality)
if (_stopAfterFirstPatternMatch && _uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags))
{
continue;
}
List <ScanResult> matchList = new List <ScanResult>();
// Go through each matching pattern of the rule
foreach (SearchPattern pattern in rule.Patterns)
{
//Skill patterns that don't apply based on settings
if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence))
{
continue;
}
// Process all matches for the patttern (this may be only 1 is _stopAfterFirstMatch is set
foreach (Boundary match in textContainer.EnumerateMatchingBoundaries(pattern))
{
bool passedConditions = true;
foreach (SearchCondition condition in rule.Conditions)
{
bool res = textContainer.IsPatternMatch(condition.Pattern, match, condition);
if (res && condition.NegateFinding)
{
passedConditions = false;
break;
}
if (!res && condition.NegateFinding)
{
passedConditions = true;
break;
}
if (!res)
{
passedConditions = false;
break;
}
}
//restrict tags from build files to tags with "metadata" to avoid false feature positives that are not part of executable code
if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata")))
{
passedConditions = false;
}
if (passedConditions)
{
ScanResult newMatch = new ScanResult()
{
Boundary = match,
StartLocation = textContainer.GetLocation(match.Index),
EndLocation = textContainer.GetLocation(match.Index + match.Length),
PatternMatch = pattern,
Confidence = pattern.Confidence,
Rule = rule
};
if (_uniqueTagMatchesOnly)
{
if (!UniqueTagsCheck(newMatch.Rule.Tags)) //tag(s) previously seen
{
if (_stopAfterFirstPatternMatch) //recheck stop at pattern level also within same rule
{
passedConditions = false; //user performance option i.e. only wants to identify if tag is detected nothing more
}
else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match
{
passedConditions = BestMatch(matchList, newMatch); //; check all patterns in current rule
if (passedConditions)
{
passedConditions = BestMatch(resultsList, newMatch);//check all rules in permanent list
}
}
}
}
if (passedConditions)
{
matchList.Add(newMatch);
}
AddRuleTagHashes(rule.Tags);
}
}
}
resultsList.AddRange(matchList);
}
// Deal with overrides
List <ScanResult> removes = new List <ScanResult>();
foreach (ScanResult scanResult in resultsList)
{
if (scanResult.Rule.Overrides is string[] overrides)
{
foreach (string @override in overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (ScanResult overRideMatch in resultsList.FindAll(x => x.Rule.Id == @override))
{
if (overRideMatch.Boundary.Index >= scanResult.Boundary.Index &&
overRideMatch.Boundary.Index <= scanResult.Boundary.Index + scanResult.Boundary.Length)
{
removes.Add(overRideMatch);
}
}
}
}
}
if (removes.Count > 0)
{
// Remove overriden rules
resultsList.RemoveAll(x => removes.Contains(x));
}
return(resultsList.ToArray());
}
/// <summary>
/// Analyzes given line of code returning matching scan results for the
/// file passed in only; Use AllResults to get results across the entire set
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
{
// Get rules for the given content type
IEnumerable <ConvertedOatRule> rules = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only
TextContainer textContainer = new TextContainer(text, languageInfo.Name);
foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
{
// If we have within captures it means we had conditions, and we only want the conditioned captures
var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause);
if (withinCaptures.Any())
{
foreach (var cap in withinCaptures)
{
ProcessBoundary(cap);
}
}
// Otherwise we can use all the captures
else
{
foreach (var cap in ruleCapture.Captures)
{
ProcessBoundary(cap);
}
}
void ProcessBoundary(ClauseCapture cap)
{
List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only
if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
{
if (ruleCapture.Rule is ConvertedOatRule oatRule)
{
if (tcc?.Result is List <(int, Boundary)> captureResults)
{
foreach (var match in captureResults)
{
var patternIndex = match.Item1;
var boundary = match.Item2;
//restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
{
_logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
continue;
}
if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
{
continue;
}
Location StartLocation = textContainer.GetLocation(boundary.Index);
Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length);
MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule)
{
FileName = filePath,
FullText = textContainer.FullContent,
LanguageInfo = languageInfo,
Boundary = boundary,
StartLocationLine = StartLocation.Line,
EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex],
Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
};
bool addNewRecord = true;
if (_uniqueTagMatchesOnly)
{
if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags))
{
if (_stopAfterFirstMatch)
{
addNewRecord = false; //we've seen already; don't improve the match
}
else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match
{
addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch);
if (addNewRecord)
{
lock (_controllRunningListAdd)
{
addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files
}
}
}
}
}
if (addNewRecord)
{
newMatches.Add(newMatch);
lock (_controllRunningListAdd)
{
_runningResultsList.Add(newMatch);
}
}
AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" });
}
}
}
}
resultsList.AddRange(newMatches);
}
}
if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
{
// Deal with overrides
List <MatchRecord> removes = new List <MatchRecord>();
foreach (MatchRecord m in resultsList)
{
if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
{
foreach (string ovrd in m.Rule.Overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
{
if (om.Boundary?.Index >= m.Boundary?.Index &&
om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
{
removes.Add(om);
}
}
}
}
}
// Remove overriden rules
resultsList.RemoveAll(x => removes.Contains(x));
}
return(resultsList.ToArray());
}
/// <summary>
/// Returns the language for a given file name if detected.
/// </summary>
/// <param name="fileName">The FileName to check.</param>
/// <param name="info">If this returns true, a valid LanguageInfo object. If false, undefined.</param>
/// <returns>True if the language could be detected based on the filename.</returns>
public static bool FromFileNameOut(string fileName, out LanguageInfo info)
{
info = new LanguageInfo();
return(FromFileName(fileName, ref info));
}
/// <summary>
/// Analyzes given line of code returning matching scan results for the
/// file passed in only; Use AllResults to get results across the entire set
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
{
// Get rules for the given content type
var rulesByLanguage = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
var rules = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)));
List <MatchRecord> resultsList = new List <MatchRecord>();//matches for this file only
TextContainer textContainer = new TextContainer(text, languageInfo.Name);
foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
{
foreach (var cap in ruleCapture.Captures)
{
ProcessBoundary(cap);
}
void ProcessBoundary(ClauseCapture cap)
{
List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only
if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
{
if (ruleCapture.Rule is ConvertedOatRule oatRule)
{
if (tcc?.Result is List <(int, Boundary)> captureResults)
{
foreach (var match in captureResults)
{
var patternIndex = match.Item1;
var boundary = match.Item2;
//restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
{
_logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
continue;
}
if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
{
continue;
}
Location StartLocation = textContainer.GetLocation(boundary.Index);
Location EndLocation = textContainer.GetLocation(boundary.Index + boundary.Length);
MatchRecord newMatch = new MatchRecord(oatRule.AppInspectorRule)
{
FileName = filePath,
FullTextContainer = textContainer,
LanguageInfo = languageInfo,
Boundary = boundary,
StartLocationLine = StartLocation.Line,
EndLocationLine = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
MatchingPattern = oatRule.AppInspectorRule.Patterns[patternIndex],
Excerpt = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
Sample = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
};
newMatches.Add(newMatch);
if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any())
{
AddRuleTagHashes(oatRule.AppInspectorRule.Tags);
}
}
}
}
}
resultsList.AddRange(newMatches);
}
}
if (_uniqueTagMatchesOnly)
{
var replacementList = new List <MatchRecord>();
foreach (var entry in resultsList)
{
if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags))
{
if (!_stopAfterFirstMatch)
{
var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y)));
if (replaceable.Any())
{
replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList();
if (replaceable.Any())
{
replacementList.RemoveAll(x => replaceable.Contains(x));
replacementList.Add(entry);
}
}
else
{
replacementList.Add(entry);
}
}
}
else
{
replacementList.Add(entry);
}
}
resultsList = replacementList;
}
List <MatchRecord> removes = new List <MatchRecord>();
foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
{
if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
{
foreach (string ovrd in m.Rule.Overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
{
if (om.Boundary?.Index >= m.Boundary?.Index &&
om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
{
removes.Add(om);
}
}
}
}
// Remove overriden rules
}
resultsList.RemoveAll(x => removes.Contains(x));
foreach (var entry in resultsList)
{
_runningResultsList.Enqueue(entry);
}
return(resultsList.ToArray());
}
/// <summary>
/// Analyzes given line of code
/// </summary>
/// <param name="text">Source code</param>
/// <param name="languages">List of languages</param>
/// <returns>Array of matches</returns>
public Issue[] Analyze(string text, LanguageInfo languageInfo)
{
string[] languages = new string[] { languageInfo.Name };
// Get rules for the given content type
IEnumerable <Rule> rules = GetRulesForLanguages(languages);
List <Issue> resultsList = new List <Issue>();
TextContainer textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch);
// Go through each rule
foreach (Rule rule in rules)
{
if (_logger != null)
{
_logger.Debug("Processing for rule: " + rule.Id);
}
// Skip pattern matching this rule if uniquetag option and not in exceptions list
bool multipleMatchesOk = !_uniqueTagMatchesOnly || UniqueTagsCheck(rule.Tags);
if (!multipleMatchesOk)
{
continue;
}
List <Issue> matchList = new List <Issue>();
// Skip rules that don't apply based on settings
if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity))
{
continue;
}
// Go through each matching pattern of the rule
foreach (SearchPattern pattern in rule.Patterns)
{
//Skill patterns that don't apply based on settings
if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence))
{
continue;
}
// Get all matches for the patttern
List <Boundary> matches = textContainer.MatchPattern(pattern);
if (matches.Count > 0)
{
foreach (Boundary match in matches)
{
bool passedConditions = true;
foreach (SearchCondition condition in rule.Conditions)
{
bool res = textContainer.MatchPattern(condition.Pattern, match, condition);
if (res && condition.NegateFinding)
{
passedConditions = false;
break;
}
if (!res && condition.NegateFinding)
{
passedConditions = true;
break;
}
if (!res)
{
passedConditions = false;
break;
}
}
//do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program
if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata")))
{
passedConditions = false;
}
if (passedConditions)
{
Issue issue = new Issue()
{
Boundary = match,
StartLocation = textContainer.GetLocation(match.Index),
EndLocation = textContainer.GetLocation(match.Index + match.Length),
PatternMatch = pattern,
Confidence = pattern.Confidence,
Rule = rule
};
//check at pattern level to avoid adding duplicates
if (_uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags))
{
break;
}
AddRuleTagHashes(rule.Tags);
matchList.Add(issue);
}
}
}
}
// We got matching rule and suppression are enabled,
// let's see if we have a supression on the line
if (EnableSuppressions && matchList.Count > 0)
{
Suppression supp;
foreach (Issue result in matchList)
{
supp = new Suppression(textContainer.GetLineContent(result.StartLocation.Line));
// If rule is NOT being suppressed then report it
SuppressedIssue supissue = supp.GetSuppressedIssue(result.Rule.Id);
if (supissue == null)
{
resultsList.Add(result);
}
// Otherwise add the suppression info instead
else
{
Boundary bound = textContainer.GetLineBoundary(result.Boundary.Index);
bound.Index += supissue.Boundary.Index;
bound.Length = supissue.Boundary.Length;
//resultsList.Add();
Issue info = new Issue()
{
IsSuppressionInfo = true,
Boundary = bound,
StartLocation = textContainer.GetLocation(bound.Index),
EndLocation = textContainer.GetLocation(bound.Index + bound.Length),
Rule = result.Rule
};
// Add info only if it's not on the same location
if (resultsList.FirstOrDefault(x => x.Rule.Id == info.Rule.Id && x.Boundary.Index == info.Boundary.Index) == null)
{
resultsList.Add(info);
}
else if (_logger != null)
{
_logger.Debug("Not added due to proximity to another rule");
}
}
}
}
// Otherwise put matchlist to resultlist
else
{
resultsList.AddRange(matchList);
}
}
// Deal with overrides
List <Issue> removes = new List <Issue>();
foreach (Issue m in resultsList)
{
if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
{
foreach (string ovrd in m.Rule.Overrides)
{
// Find all overriden rules and mark them for removal from issues list
foreach (Issue om in resultsList.FindAll(x => x.Rule.Id == ovrd))
{
if (om.Boundary.Index >= m.Boundary.Index &&
om.Boundary.Index <= m.Boundary.Index + m.Boundary.Length)
{
removes.Add(om);
}
}
}
}
}
// Remove overriden rules
resultsList.RemoveAll(x => removes.Contains(x));
return(resultsList.ToArray());
}