public AppMetaData(string sourcePath, List <string> rulePaths)
{
//Initial value for ApplicationName may be replaced if rule pattern match found later
if (Directory.Exists(sourcePath))
{
try
{
ApplicationName = sourcePath.Substring(sourcePath.LastIndexOf(Path.DirectorySeparatorChar)).Replace(Path.DirectorySeparatorChar, ' ').Trim();
}
catch (Exception)
{
ApplicationName = Path.GetFileNameWithoutExtension(sourcePath);
}
}
else
{
ApplicationName = Path.GetFileNameWithoutExtension(sourcePath);
}
//initialize standard set groups using dynamic lists variables that may have more than one value; some are filled
//using tag tests and others by different means like file type examination
KeyedPropertyLists = new Dictionary <string, HashSet <string> >
{
["strGrpRulePaths"] = rulePaths.ToHashSet(),
["strGrpPackageTypes"] = new HashSet <string>(),
["strGrpAppTypes"] = new HashSet <string>(),
["strGrpFileTypes"] = new HashSet <string>(),
["strGrpUniqueTags"] = new HashSet <string>(),
["strGrpOutputs"] = new HashSet <string>(),
["strGrpTargets"] = new HashSet <string>(),
["strGrpOSTargets"] = new HashSet <string>(),
["strGrpFileExtensions"] = new HashSet <string>(),
["strGrpFileNames"] = new HashSet <string>(),
["strGrpCPUTargets"] = new HashSet <string>(),
["strGrpCloudTargets"] = new HashSet <string>(),
["strGrpUniqueDependencies"] = new HashSet <string>()
};
//predefined standard tags to track; only some are propertygrouplist are tag based
_propertyTagSearchPatterns = new Dictionary <string, string>();
_propertyTagSearchPatterns.Add("strGrpOSTargets", ".OS.Targets");
_propertyTagSearchPatterns.Add("strGrpCloudTargets", ".Cloud");
_propertyTagSearchPatterns.Add("strGrpOutputs", ".Outputs");
_propertyTagSearchPatterns.Add("strGrpCPUTargets", ".CPU");
//read default/user preferences on what tags to count
if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref)))
{
TagCounters = JsonConvert.DeserializeObject <List <TagCounter> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref)));
}
else
{
TagCounters = new List <TagCounter>();
}
HashSet <string> dupCountersCheck = new HashSet <string>();
foreach (TagCounter counter in TagCounters)
{
if (!dupCountersCheck.Add(counter.Tag))
{
WriteOnce.SafeLog("Duplidate counter specified in preferences", NLog.LogLevel.Error);
}
}
Languages = new Dictionary <string, int>();
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
//determine if file is a compressed item to unpackage for processing
string language = Language.FromFileName(filePath);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
language = Path.GetFileName(filePath);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(language);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, language);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
HashSet <string> uniqueTagsControl = new HashSet <string>();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -u option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = language,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool valid = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (valid)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
void UnZipAndProcess(string filename)
{
if (!File.Exists(filename))
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
}
// Ignore images and other junk like that
var fileExtension = new FileInfo(filename).Extension;
var mimeType = MimeTypeMap.GetMimeType(fileExtension);
bool mimeMatch = false;
if (!IgnoreMimeRegex.IsMatch(mimeType))
{
var isValidExtension = COMPRESSED_EXTENSIONS.Any(fileExtension.Contains);
if (isValidExtension || fileExtension == "ts")
{
mimeMatch = true;
}
else if (mimeType.Contains("zip", StringComparison.CurrentCultureIgnoreCase) || // Should have been caught in file extensions above, but still OK.
mimeType.Contains("tar", StringComparison.CurrentCultureIgnoreCase) ||
mimeType.Contains("compressed", StringComparison.CurrentCultureIgnoreCase))
{
mimeMatch = true;
}
if (mimeMatch)
{
// Now process the file
switch (fileExtension)
{
case ".tgz":
ProcessTarGzFile(filename);
break;
case ".gz":
if (filename.Contains(".tar.gz"))
{
ProcessTarGzFile(filename);
}
else
{
WriteOnce.SafeLog("no support for .gz unless .tar.gz: " + fileExtension, LogLevel.Warn);
_appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
}
break;
case ".jar":
case ".zip":
ProcessZipFile(filename);
break;
case ".gem":
case ".tar":
case ".nupkg":
WriteOnce.SafeLog($"Processing of {fileExtension} not implemented yet.", LogLevel.Warn);
break;
default:
WriteOnce.SafeLog("no support for compressed type: " + fileExtension, LogLevel.Warn);
break;
}
_appProfile.MetaData.PackageTypes.Add("compressed");
}
else
{
_appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
}
}
}
/// <summary>
/// Add default and/or custom rules paths
/// Iterate paths and add to ruleset
/// </summary>
void ConfigRules()
{
WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace);
RuleSet rulesSet = new RuleSet(Program.Logger);
List <string> rulePaths = new List <string>();
if (!_arg_ignoreDefaultRules)
{
rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
}
if (!string.IsNullOrEmpty(_arg_customRulesPath))
{
rulePaths.Add(_arg_customRulesPath);
}
foreach (string rulePath in rulePaths)
{
if (Directory.Exists(rulePath))
{
rulesSet.AddDirectory(rulePath);
}
else if (File.Exists(rulePath))
{
rulesSet.AddFile(rulePath);
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
}
}
//error check based on ruleset not path enumeration
if (rulesSet.Count() == 0)
{
throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
}
//instantiate a RuleProcessor with the added rules and exception for dependency
_rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, Program.Logger);
if (_arg_outputUniqueTagsOnly)
{
List <TagException> tagExceptions;
if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref)))
{
tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref)));
string[] exceptions = new string[tagExceptions.Count];
for (int i = 0; i < tagExceptions.Count; i++)
{
exceptions[i] = tagExceptions[i].Tag;
}
_rulesProcessor.UniqueTagExceptions = exceptions;
}
}
_appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly);
_appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " +
WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower();
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
{
#region minorRollupTrackingAndProgress
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesAnalyzed++;
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
//exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
return;
}
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
// Skip files written in unknown language
if (!Language.FromFileName(filePath, ref languageInfo))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(languageInfo.Name);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//reported: if a zip contains more zip files in it the total count may be off -complex. ~workaround: freeze UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo.Name);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program
if (languageInfo.Type == LanguageInfo.LangFileType.Build && match.Rule.Tags.Any(v => !v.Contains("Metadata")))
{
continue;
}
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
public int Run()
{
bool issues = false;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Verify Rules"));
//load [each] rules file separately to report out where a failure is happening
RuleSet rules = new RuleSet(WriteOnce.Log);
IEnumerable <string> fileListing = new List <string>();
foreach (string rulePath in _rulePaths)
{
if (Directory.Exists(rulePath))
{
fileListing = Directory.EnumerateFiles(rulePath, "*.json", SearchOption.AllDirectories);
}
else if (File.Exists(rulePath) && Path.GetExtension(rulePath) == ".json")
{
fileListing = new List <string>()
{
new string(rulePath)
}
}
;
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
}
//test loading each file
foreach (string filename in fileListing)
{
try
{
rules.AddFile(filename);
WriteOnce.Info(string.Format("Rule file added {0}", filename), true, WriteOnce.ConsoleVerbosity.High);
}
catch (Exception e)
{
WriteOnce.Error(string.Format("Rule file add failed {0}", filename));
WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error);
issues = true;
}
}
}
//option to write validating data
if (_arg_consoleVerbosityLevel == WriteOnce.ConsoleVerbosity.High)
{
WritePartialRuleDetails(rules);
}
//final status report
if (issues)
{
WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low);
}
else
{
WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low);
}
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Verify Rules"));
WriteOnce.FlushAll();
if (!String.IsNullOrEmpty(_arg_outputFile))
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low);
}
return(issues ? (int)ExitCode.NotVerified : (int)ExitCode.Verified);
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
//exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}