public void WriteMatch(MatchRecord match)
{
string output = _formatString.Replace("%F", match.Filename);
output = output.Replace("%t", match.Language);
output = output.Replace("%L", match.Issue.StartLocation.Line.ToString());
output = output.Replace("%C", match.Issue.StartLocation.Column.ToString());
output = output.Replace("%l", match.Issue.EndLocation.Line.ToString());
output = output.Replace("%c", match.Issue.EndLocation.Column.ToString());
output = output.Replace("%I", match.Issue.Boundary.Index.ToString());
output = output.Replace("%i", match.Issue.Boundary.Length.ToString());
output = output.Replace("%R", match.Issue.Rule.Id);
output = output.Replace("%N", match.Issue.Rule.Name);
output = output.Replace("%S", match.Issue.Rule.Severity.ToString());
output = output.Replace("%X", match.Issue.Confidence.ToString());//override rule confidence because of unstructured text vs source
output = output.Replace("%D", match.Issue.Rule.Description);
output = output.Replace("%m", match.TextSample);
output = output.Replace("%T", string.Join(',', match.Issue.Rule.Tags));
TextWriter.WriteLine(output);
}
public MatchItems(MatchRecord matchRecord)
{
FileName = matchRecord.Filename;
SourceType = matchRecord.Language;
StartLocationLine = matchRecord.Issue.StartLocation.Line;
StartLocationColumn = matchRecord.Issue.StartLocation.Column;
EndLocationLine = matchRecord.Issue.EndLocation.Line;
EndLocationColumn = matchRecord.Issue.EndLocation.Column;
BoundaryIndex = matchRecord.Issue.Boundary.Index;
BoundaryLength = matchRecord.Issue.Boundary.Length;
RuleId = matchRecord.Issue.Rule.Id;
Severity = matchRecord.Issue.Rule.Severity.ToString();
RuleName = matchRecord.Issue.Rule.Name;
RuleDescription = matchRecord.Issue.Rule.Description;
PatternConfidence = matchRecord.Issue.Confidence.ToString();
PatternType = matchRecord.Issue.PatternMatch.PatternType.ToString();
MatchingPattern = matchRecord.Issue.PatternMatch.Pattern;
Sample = matchRecord.TextSample;
Excerpt = matchRecord.Excerpt;
Tags = matchRecord.Issue.Rule.Tags;
}
/// <summary>
/// Attempt to map application type tags or file type or language to identify
/// WebApplications, Windows Services, Client Apps, WebServices, Azure Functions etc.
/// </summary>
/// <param name="match"></param>
static public String DetectSolutionType(MatchRecord match)
{
string result = "";
if (match.Issue.Rule.Tags.Any(s => s.Contains("Application.Type")))
{
foreach (string tag in match.Issue.Rule.Tags)
{
int index = tag.IndexOf("Application.Type");
if (-1 != index)
{
result = tag.Substring(index + 17);
break;
}
}
}
else
{
switch (match.Filename)
{
case "web.config":
result = "Web.Application";
break;
case "app.config":
result = ".NETclient";
break;
default:
switch (Path.GetExtension(match.Filename))
{
case ".cshtml":
result = "Web.Application";
break;
case ".htm":
case ".html":
case ".js":
result = "Web.Application";
break;
case "powershell":
case "shellscript":
case "wincmdscript":
result = "script";
break;
default:
switch (match.Language.Name)
{
case "ruby":
case "perl":
case "php":
result = "Web.Application";
break;
}
break;
}
break;
}
}
return(result.ToLower());
}
/// <summary>
/// Part of post processing to test for matches against app defined properties
/// defined in MetaData class
/// Exludes a match if specified in preferences as a counted tag with exclude true
/// </summary>
/// <param name="matchRecord"></param>
public bool AddStandardProperties(ref MatchRecord matchRecord)
{
bool includeAsMatch = true;
//testing for presence of a tag against the specified set in preferences for report org
foreach (string key in _propertyTagSearchPatterns.Keys)
{
var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
if (matchRecord.Issue.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
KeyedPropertyLists[key].Add(matchRecord.TextSample);
}
}
// Author etc. or STANDARD METADATA properties we capture from any supported file type; others just captured as general tag matches...
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Author")))
{
this.Authors = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
{
this.Authors = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Description")))
{
this.Description = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Name")))
{
this.ApplicationName = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Version")))
{
this.SourceVersion = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
{
this.CPUTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
{
this.Outputs.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Platform.OS")))
{
this.OSTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
//Special handling; attempt to detect app types...review for multiple pattern rule limitation
String solutionType = Utils.DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
AppTypes.Add(solutionType);
}
//Update metric counters for default or user specified tags
foreach (TagCounter counter in TagCounters)
{
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains(counter.Tag)))
{
counter.Count++;
includeAsMatch = counter.IncludeAsMatch;//Exclude as feature matches per preferences from reporting full match details
}
}
//once patterns checked; prepare text for output blocking browser xss
matchRecord.TextSample = System.Net.WebUtility.HtmlEncode(matchRecord.TextSample);
return(includeAsMatch);
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
//determine if file is a compressed item to unpackage for processing
string language = Language.FromFileName(filePath);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
language = Path.GetFileName(filePath);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(language);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, language);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
HashSet <string> uniqueTagsControl = new HashSet <string>();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -u option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = language,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool valid = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (valid)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
{
#region minorRollupTrackingAndProgress
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesAnalyzed++;
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
//exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
return;
}
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
// Skip files written in unknown language
if (!Language.FromFileName(filePath, ref languageInfo))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(languageInfo.Name);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//reported: if a zip contains more zip files in it the total count may be off -complex. ~workaround: freeze UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo.Name);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program
if (languageInfo.Type == LanguageInfo.LangFileType.Build && match.Rule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Part of post processing to test for matches against app defined properties
/// defined in MetaData class
/// Exludes a match if specified in preferences as a counted tag with exclude true
/// </summary>
/// <param name="matchRecord"></param>
public bool AddStandardProperties(MatchRecord matchRecord)
{
bool includeAsMatch = true;
//testing for presence of a tag against the specified set in preferences for report org
foreach (string key in _propertyTagSearchPatterns.Keys)
{
var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
if (matchRecord.Issue.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
KeyedPropertyLists[key].Add(matchRecord.TextSample);
}
}
//update counts for default or user specified tags
foreach (TagCounter counter in TagCounters)
{
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains(counter.Tag)))
{
counter.Count++;
includeAsMatch = counter.IncludeAsMatch;
}
}
// Author etc. or standard properties we capture
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Author")))
{
this.Authors = ExtractJSONValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
{
this.Authors = ExtractXMLValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Description")))
{
this.Description = ExtractJSONValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Name")))
{
this.ApplicationName = ExtractJSONValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Version")))
{
this.SourceVersion = ExtractJSONValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Hardware.Processor")))
{
this.CPUTargets.Add(ExtractJSONValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.BuildOutput.Category")))
{
this.Outputs.Add(ExtractXMLValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Platform.OS")))
{
this.OSTargets.Add(ExtractJSONValue(matchRecord.TextSample).ToLower());
}
//special handling; attempt to detect app types...review for multiple pattern rule limitation
String solutionType = Utils.DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
AppTypes.Add(solutionType);
}
return(includeAsMatch);
}