/// <summary>
/// Attempt to map application type tags or file type or language to identify
/// WebApplications, Windows Services, Client Apps, WebServices, Azure Functions etc.
/// </summary>
/// <param name="match"></param>
public string DetectSolutionType(MatchRecord match)
{
string result = "";
if (match.Tags is not null && match.Tags.Any(s => s.Contains("Application.Type")))
{
foreach (string tag in match.Tags ?? new string[] { })
{
int index = tag.IndexOf("Application.Type");
if (-1 != index)
{
result = tag.Substring(index + 17);
break;
}
}
}
/// <summary>
/// Assist in aggregating reporting properties of matches as they are added
/// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
/// </summary>
/// <param name="matchRecord"></param>
public void AddTagsFromMatchRecord(MatchRecord matchRecord)
{
//special handling for standard characteristics in report
foreach (var tag in matchRecord.Tags ?? Array.Empty <string>())
{
switch (tag)
{
case "Metadata.Application.Author":
case "Metadata.Application.Publisher":
Metadata.Authors = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Description":
Metadata.Description = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Name":
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Version":
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Target.Processor":
CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Metadata.Application.Output.Type":
Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Dependency.SourceInclude":
return; //design to keep noise out of detailed match list
default:
if (tag.Split('.').Contains("Metric"))
{
_ = TagCounters.TryAdd(tag, new MetricTagCounter()
{
Tag = tag
});
}
else if (tag.Contains(".Platform.OS"))
{
OSTargets.TryAdd(tag[(tag.LastIndexOf('.', tag.Length - 1) + 1)..], 0);
public void WriteMatch(MatchRecord match)
{
string output = _formatString.Replace("%F", match.Filename);
output = output.Replace("%l", match.Language.Name);
output = output.Replace("%t", match.Language.Type.ToString());
output = output.Replace("%L", match.Issue.StartLocation.Line.ToString());
output = output.Replace("%C", match.Issue.StartLocation.Column.ToString());
output = output.Replace("%l", match.Issue.EndLocation.Line.ToString());
output = output.Replace("%c", match.Issue.EndLocation.Column.ToString());
output = output.Replace("%I", match.Issue.Boundary.Index.ToString());
output = output.Replace("%i", match.Issue.Boundary.Length.ToString());
output = output.Replace("%R", match.Issue.Rule.Id);
output = output.Replace("%N", match.Issue.Rule.Name);
output = output.Replace("%S", match.Issue.Rule.Severity.ToString());
output = output.Replace("%X", match.Issue.Confidence.ToString()); //override rule confidence because of unstructured text vs source
output = output.Replace("%D", match.Issue.Rule.Description);
output = output.Replace("%m", System.Net.WebUtility.HtmlEncode(match.TextSample)); //readability for non-browser format type
output = output.Replace("%T", string.Join(',', match.Issue.Rule.Tags));
TextWriter.WriteLine(output);
}
public LimitedMatchRecord(MatchRecord matchRecord)
{
FileName = matchRecord.Filename;
SourceLabel = matchRecord.Language.Name;
SourceType = matchRecord.Language.Type.ToString();
StartLocationLine = matchRecord.Issue.StartLocation.Line;
StartLocationColumn = matchRecord.Issue.StartLocation.Column;
EndLocationLine = matchRecord.Issue.EndLocation.Line;
EndLocationColumn = matchRecord.Issue.EndLocation.Column;
BoundaryIndex = matchRecord.Issue.Boundary.Index;
BoundaryLength = matchRecord.Issue.Boundary.Length;
RuleId = matchRecord.Issue.Rule.Id;
Severity = matchRecord.Issue.Rule.Severity.ToString();
RuleName = matchRecord.Issue.Rule.Name;
RuleDescription = matchRecord.Issue.Rule.Description;
PatternConfidence = matchRecord.Issue.Confidence.ToString();
PatternType = matchRecord.Issue.PatternMatch.PatternType.ToString();
MatchingPattern = matchRecord.Issue.PatternMatch.Pattern;
Sample = matchRecord.TextSample;
Excerpt = matchRecord.Excerpt;
Tags = matchRecord.Issue.Rule.Tags;
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
{
#region minorRollupTrackingAndProgress
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesAnalyzed++;
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output
{
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
}
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
else
{
_appProfile.MetaData.TotalMatchesCount -= 1;//reduce e.g. tag counters only as per preferences file
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Attempt to map application type tags or file type or language to identify
/// WebApplications, Windows Services, Client Apps, WebServices, Azure Functions etc.
/// </summary>
/// <param name="match"></param>
static public String DetectSolutionType(MatchRecord match)
{
string result = "";
if (match.Issue.Rule.Tags.Any(s => s.Contains("Application.Type")))
{
foreach (string tag in match.Issue.Rule.Tags)
{
int index = tag.IndexOf("Application.Type");
if (-1 != index)
{
result = tag.Substring(index + 17);
break;
}
}
}
else
{
switch (match.Filename)
{
case "web.config":
result = "Web.Application";
break;
case "app.config":
result = ".NETclient";
break;
default:
switch (Path.GetExtension(match.Filename))
{
case ".cshtml":
result = "Web.Application";
break;
case ".htm":
case ".html":
case ".js":
case ".ts":
result = "Web.Application";
break;
case "powershell":
case "shellscript":
case "wincmdscript":
result = "script";
break;
default:
switch (match.Language.Name)
{
case "ruby":
case "perl":
case "php":
result = "Web.Application";
break;
}
break;
}
break;
}
}
return(result.ToLower());
}
/// <summary>
/// Part of post processing to test for matches against app defined properties
/// defined in MetaData class
/// Exludes a match if specified in preferences as a counted tag with exclude true
/// </summary>
/// <param name="matchRecord"></param>
public bool AddStandardProperties(ref MatchRecord matchRecord)
{
bool includeAsMatch = true;
//testing for presence of a tag against the specified set in preferences for report org
foreach (string key in _propertyTagSearchPatterns.Keys)
{
var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
if (matchRecord.Issue.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
KeyedPropertyLists[key].Add(matchRecord.TextSample);
}
}
// Author etc. or STANDARD METADATA properties we capture from any supported file type; others just captured as general tag matches...
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Author")))
{
this.Authors = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
{
this.Authors = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Description")))
{
this.Description = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Name")))
{
this.ApplicationName = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Version")))
{
this.SourceVersion = ExtractValue(matchRecord.TextSample);
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
{
this.CPUTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
{
this.Outputs.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Platform.OS")))
{
this.OSTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
}
//Special handling; attempt to detect app types...review for multiple pattern rule limitation
String solutionType = Utils.DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
AppTypes.Add(solutionType);
}
//Update metric counters for default or user specified tags
foreach (TagCounter counter in TagCounters)
{
if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains(counter.Tag)))
{
counter.Count++;
includeAsMatch = counter.IncludeAsMatch;//Exclude as feature matches per preferences from reporting full match details
}
}
//once patterns checked; prepare text for output blocking browser xss
matchRecord.TextSample = System.Net.WebUtility.HtmlEncode(matchRecord.TextSample);
return(includeAsMatch);
}
/// <summary>
/// Assist in aggregating reporting properties of matches as they are added
/// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
/// </summary>
/// <param name="matchRecord"></param>
public void AddMatchRecord(MatchRecord matchRecord)
{
//aggregate lists of matches against standard set of properties to report on
foreach (string key in _propertyTagSearchPatterns.Keys)
{
var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
if (matchRecord.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
Metadata.KeyedPropertyLists[key].Add(matchRecord.Sample);
}
}
// single standard properties we capture from any supported file type; others just captured as general tag matches...
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Author")))
{
Metadata.Authors = ExtractValue(matchRecord.Sample);
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
{
Metadata.Authors = ExtractValue(matchRecord.Sample);
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Description")))
{
Metadata.Description = ExtractValue(matchRecord.Sample);
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Name")))
{
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Version")))
{
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
{
Metadata.CPUTargets.Add(ExtractValue(matchRecord.Sample).ToLower());
}
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
{
Metadata.Outputs.Add(ExtractValue(matchRecord.Sample).ToLower());
}
if (matchRecord.Tags.Any(v => v.Contains("Platform.OS")))
{
Metadata.OSTargets.Add(ExtractValue(matchRecord.Sample).ToLower());
}
if (matchRecord.Tags.Any(v => v.Contains("Metric.")))
{
Metadata.TagCounters.Add(new MetricTagCounter()
{
Tag = matchRecord.Tags[0],
Count = 0
});
}
//safeguard sample output now that we've matched properties for blocking browser xss
matchRecord.Sample = System.Net.WebUtility.HtmlEncode(matchRecord.Sample);
//Special handling; attempt to detect app types...review for multiple pattern rule limitation
String solutionType = DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
Metadata.AppTypes.Add(solutionType);
}
//Update metric counters for default or user specified tags; don't add as match detail
bool counterOnlyTag = false;
foreach (MetricTagCounter counter in Metadata.TagCounters)
{
if (matchRecord.Tags.Any(v => v.Contains(counter.Tag)))
{
counterOnlyTag = true;
counter.Count++;
break;
}
}
//omit adding if only a counter metric tag
if (!counterOnlyTag)
{
//update list of unique tags as we go
foreach (string tag in matchRecord.Tags)
{
Metadata.UniqueTags.Add(tag);
}
Metadata.Matches.Add(matchRecord);
}
else
{
Metadata.TotalMatchesCount -= 1;//reduce e.g. tag counters not included as detailed match
}
}
/// <summary>
/// Assist in aggregating reporting properties of matches as they are added
/// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
/// </summary>
/// <param name="matchRecord"></param>
public void AddMatchRecord(MatchRecord matchRecord)
{
//aggregate lists of matches against standard set of properties to report on
foreach (string key in _propertyTagSearchPatterns.Keys)
{
if (matchRecord.Tags.Any(v => _propertyTagSearchPatterns[key].IsMatch(v)))
{
_ = Metadata.KeyedPropertyLists[key].TryAdd(matchRecord.Sample, 0);
}
}
//Update metric counters for default or user specified tags; don't add as match detail
foreach (var tag in matchRecord.Tags)
{
switch (tag)
{
case "Metadata.Application.Author":
case "Metadata.Application.Publisher":
Metadata.Authors = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Description":
Metadata.Description = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Name":
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Version":
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Target.Processor":
_ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Metadata.Application.Output.Type":
_ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Platform.OS":
_ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
default:
if (tag.Contains("Metric."))
{
Metadata.TagCounters.Push(new MetricTagCounter()
{
Tag = tag
});
}
break;
}
}
//safeguard sample output now that we've matched properties for blocking browser xss
matchRecord.Sample = System.Net.WebUtility.HtmlEncode(matchRecord.Sample);
//Special handling; attempt to detect app types...review for multiple pattern rule limitation
string solutionType = DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
_ = Metadata.AppTypes.TryAdd(solutionType, 0);
}
bool CounterOnlyTagSet = false;
var selected = Metadata.TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Tag)));
foreach (var select in selected)
{
CounterOnlyTagSet = true;
select.IncrementCount();
}
//omit adding if ther a counter metric tag
if (!CounterOnlyTagSet)
{
//update list of unique tags as we go
foreach (string tag in matchRecord.Tags)
{
_ = Metadata.UniqueTags.TryAdd(tag, 0);
}
Metadata.Matches.Add(matchRecord);
}
else
{
Metadata.IncrementTotalMatchesCount(-1);//reduce e.g. tag counters not included as detailed match
}
}
/// <summary>
/// Assist in aggregating reporting properties of matches as they are added
/// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
/// </summary>
/// <param name="matchRecord"></param>
public void AddMatchRecord(MatchRecord matchRecord)
{
bool allowAdd = true;
//special handling for standard characteristics in report
foreach (var tag in matchRecord.Tags ?? new string[] { })
{
switch (tag)
{
case "Metadata.Application.Author":
case "Metadata.Application.Publisher":
Metadata.Authors = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Description":
Metadata.Description = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Name":
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Version":
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Target.Processor":
_ = CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Metadata.Application.Output.Type":
_ = Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Dependency.SourceInclude":
allowAdd = false; //design to keep noise out of detailed match list
break;
default:
if (tag.Contains("Metric."))
{
_ = TagCounters.TryAdd(tag, new MetricTagCounter()
{
Tag = tag
});
}
else if (tag.Contains(".Platform.OS"))
{
_ = OSTargets.TryAdd(tag.Substring(tag.LastIndexOf('.', tag.Length - 1) + 1), 0);
}
else if (tag.Contains("CloudServices.Hosting"))
{
_ = CloudTargets.TryAdd(tag.Substring(tag.LastIndexOf('.', tag.Length - 1) + 1), 0);
}
break;
}
}
//Special handling; attempt to detect app types...review for multiple pattern rule limitation
string solutionType = DetectSolutionType(matchRecord);
if (!string.IsNullOrEmpty(solutionType))
{
_ = AppTypes.TryAdd(solutionType, 0);
}
bool CounterOnlyTagSet = false;
var selected = TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Value.Tag ?? "")));
foreach (var select in selected)
{
CounterOnlyTagSet = true;
select.Value.IncrementCount();
}
//omit adding if ther a counter metric tag
if (!CounterOnlyTagSet)
{
//update list of unique tags as we go
foreach (string tag in matchRecord.Tags ?? new string[] { })
{
_ = UniqueTags.TryAdd(tag, 0);
}
if (allowAdd)
{
Metadata?.Matches?.Add(matchRecord);
}
}
else
{
Metadata.IncrementTotalMatchesCount(-1);//reduce e.g. tag counters not included as detailed match
}
}