/// <summary> /// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "analyze")); try { _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType = ArchiveFileType.UNKNOWN; try //fix for #146 { archiveFileType = MiniMagic.DetectFileType(filename); } catch (Exception e) { WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, LogLevel.Error); //log details Exception f = new Exception(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILE_TYPE_OPEN, filename)); //report friendly version throw f; } if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "analyze")); } } catch (Exception e) { WriteOnce.Error(e.Message); //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)Utils.ExitCode.Success); }
/// <summary> /// Main WORKHORSE for analyzing file; called from file based or decompression functions /// </summary> /// <param name="filename"></param> /// <param name="fileText"></param> void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo) { #region minorRollupTrackingAndProgress WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace); _appProfile.MetaData.FilesAnalyzed++; int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped; int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100); //earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI if (percentCompleted > 100) { percentCompleted = 100; } else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output { WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false); } #endregion //process file against rules Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo); //if any matches found for this file... if (matches.Count() > 0) { _appProfile.MetaData.FilesAffected++; _appProfile.MetaData.TotalMatchesCount += matches.Count(); // Iterate through each match issue foreach (Issue match in matches) { WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace); //maintain a list of unique tags; multi-purpose but primarily for filtering -d option bool dupTagFound = false; foreach (string t in match.Rule.Tags) { dupTagFound = !_uniqueTagsControl.Add(t); } //save all unique dependencies even if Dependency tag pattern is not-unique var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase); String textMatch; if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v))) { textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name); } else { textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length); } //wrap rule issue result to add metadata MatchRecord record = new MatchRecord() { Filename = filePath, Language = languageInfo, Filesize = fileText.Length, TextSample = textMatch, Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line), Issue = match }; //preserve issue level characteristics as rolled up meta data of interest bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record); //bail after extracting any dependency unique items IF user requested if (_arg_outputUniqueTagsOnly && dupTagFound) { continue; } else if (addAsFeatureMatch) { _appProfile.MatchList.Add(record); } else { _appProfile.MetaData.TotalMatchesCount -= 1;//reduce e.g. tag counters only as per preferences file } } } else { WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace); } }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("TagDiffCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "tagdiff")); ExitCode exitCode = ExitCode.CriticalError; //save to quiet analyze cmd and restore WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; AnalyzeCommand.ExitCode analyzeCmdResult1 = AnalyzeCommand.ExitCode.CriticalError; AnalyzeCommand.ExitCode analyzeCmdResult2 = AnalyzeCommand.ExitCode.CriticalError; try { #region setup analyze calls string tmp1 = Path.GetTempFileName(); string tmp2 = Path.GetTempFileName(); AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_src1, OutputFilePath = tmp1, OutputFileFormat = "json", CustomRulesPath = _arg_customRulesPath, IgnoreDefaultRules = _arg_ignoreDefault, FilePathExclusions = _arg_fileExclusionList, SimpleTagsOnly = true, ConsoleVerbosityLevel = "none", Log = _arg_logger }); AnalyzeCommand cmd2 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_src2, OutputFilePath = tmp2, OutputFileFormat = "json", CustomRulesPath = _arg_customRulesPath, IgnoreDefaultRules = _arg_ignoreDefault, FilePathExclusions = _arg_fileExclusionList, SimpleTagsOnly = true, ConsoleVerbosityLevel = "none", Log = _arg_logger }); analyzeCmdResult1 = (AnalyzeCommand.ExitCode)cmd1.Run(); analyzeCmdResult2 = (AnalyzeCommand.ExitCode)cmd2.Run(); ConfigureFileOutput(); //restore WriteOnce.Verbosity = saveVerbosity; #endregion bool equalTagsCompare1 = true; bool equalTagsCompare2 = true; //process results for each analyze call before comparing results if (analyzeCmdResult1 == AnalyzeCommand.ExitCode.CriticalError) { throw new Exception(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src1)); } else if (analyzeCmdResult2 == AnalyzeCommand.ExitCode.CriticalError) { throw new Exception(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src2)); } else if (analyzeCmdResult1 == AnalyzeCommand.ExitCode.NoMatches || analyzeCmdResult2 == AnalyzeCommand.ExitCode.NoMatches) { throw new Exception(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_NO_TAGS_FOUND)); } else //compare tag results; assumed (result1&2 == AnalyzeCommand.ExitCode.Success) { //setup output here rather than top to avoid analyze command output in this command output ConfigureFileOutput(); ConfigureConsoleOutput();//recheck string file1TagsJson = File.ReadAllText(tmp1); string file2TagsJson = File.ReadAllText(tmp2); var file1Tags = JsonConvert.DeserializeObject <TagsFile>(file1TagsJson); var file2Tags = JsonConvert.DeserializeObject <TagsFile>(file2TagsJson); //can't simply compare counts as content may differ; must compare both in directions in two passes a->b; b->a WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src1), Path.GetFileName(_arg_src2)), true, WriteOnce.ConsoleVerbosity.High); equalTagsCompare1 = CompareTags(file1Tags.Tags, file2Tags.Tags); //reverse order for second pass WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src2), Path.GetFileName(_arg_src1)), true, WriteOnce.ConsoleVerbosity.High); equalTagsCompare2 = CompareTags(file2Tags.Tags, file1Tags.Tags); //final results bool resultsDiffer = !(equalTagsCompare1 && equalTagsCompare2); if (_arg_tagTestType == TagTestType.Inequality && !resultsDiffer) { exitCode = ExitCode.TestFailed; } else if (_arg_tagTestType == TagTestType.Equality && resultsDiffer) { exitCode = ExitCode.TestFailed; } else { exitCode = ExitCode.TestPassed; } WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_DIFFER), false); WriteOnce.Result(resultsDiffer.ToString()); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "tagdiff")); if (!String.IsNullOrEmpty(_arg_outputFile) && Utils.CLIExecutionContext) { WriteOnce.Info(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, WriteOnce.ConsoleVerbosity.Low, false); } WriteOnce.FlushAll(); } //cleanup try { File.Delete(tmp1); File.Delete(tmp2); } catch { //no action needed; } } catch (Exception e) { WriteOnce.Verbosity = saveVerbosity; if (analyzeCmdResult1 == AnalyzeCommand.ExitCode.Success && analyzeCmdResult2 == AnalyzeCommand.ExitCode.Success) //error not previously logged { WriteOnce.Error(e.Message); } else { WriteOnce.Error(e.Message, true, WriteOnce.ConsoleVerbosity.Low, false);//console but don't log again } //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return((int)exitCode); }
/// <summary> /// Add default and/or custom rules paths /// Iterate paths and add to ruleset /// </summary> void ConfigRules() { WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace); RuleSet rulesSet = null; List <string> rulePaths = new List <string>(); if (!_arg_ignoreDefaultRules) { rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRulesPackedFile)); rulesSet = Utils.GetDefaultRuleSet(_arg_logger); } if (!string.IsNullOrEmpty(_arg_customRulesPath)) { if (rulesSet == null) { rulesSet = new RuleSet(_arg_logger); } rulePaths.Add(_arg_customRulesPath); if (Directory.Exists(_arg_customRulesPath)) { rulesSet.AddDirectory(_arg_customRulesPath); } else if (File.Exists(_arg_customRulesPath)) { rulesSet.AddFile(_arg_customRulesPath); } else { throw new Exception(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, _arg_customRulesPath)); } } //error check based on ruleset not path enumeration if (rulesSet == null || rulesSet.Count() == 0) { throw new Exception(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED)); } //instantiate a RuleProcessor with the added rules and exception for dependency _rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, _arg_logger); if (_arg_outputUniqueTagsOnly) { List <TagException> tagExceptions; if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref))) { tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref))); string[] exceptions = new string[tagExceptions.Count]; for (int i = 0; i < tagExceptions.Count; i++) { exceptions[i] = tagExceptions[i].Tag; } _rulesProcessor.UniqueTagExceptions = exceptions; } } _appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly, _arg_suppressBrowserOpen); _appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " + WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower(); }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public TagDiffResult GetResult() { WriteOnce.SafeLog("TagDiffCommand::Run", LogLevel.Trace); WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Tag Diff")); TagDiffResult tagDiffResult = new TagDiffResult() { AppVersion = Utils.GetVersionString() }; //save to quiet analyze cmd and restore WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; try { #region setup analyze calls AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeOptions { SourcePath = _options.SourcePath1, CustomRulesPath = _options.CustomRulesPath, IgnoreDefaultRules = _options.IgnoreDefaultRules, FilePathExclusions = _options.FilePathExclusions, ConsoleVerbosityLevel = "none", Log = _options.Log }); AnalyzeCommand cmd2 = new AnalyzeCommand(new AnalyzeOptions { SourcePath = _options.SourcePath2, CustomRulesPath = _options.CustomRulesPath, IgnoreDefaultRules = _options.IgnoreDefaultRules, FilePathExclusions = _options.FilePathExclusions, ConsoleVerbosityLevel = "none", Log = _options.Log }); AnalyzeResult analyze1 = cmd1.GetResult(); AnalyzeResult analyze2 = cmd2.GetResult(); //restore WriteOnce.Verbosity = saveVerbosity; #endregion setup analyze calls bool equalTagsCompare1 = true; bool equalTagsCompare2 = true; //process results for each analyze call before comparing results if (analyze1.ResultCode == AnalyzeResult.ExitCode.CriticalError) { throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_CRITICAL_FILE_ERR, _options.SourcePath1)); } else if (analyze2.ResultCode == AnalyzeResult.ExitCode.CriticalError) { throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_CRITICAL_FILE_ERR, _options.SourcePath2)); } else if (analyze1.ResultCode == AnalyzeResult.ExitCode.NoMatches || analyze2.ResultCode == AnalyzeResult.ExitCode.NoMatches) { throw new OpException(MsgHelp.GetString(MsgHelp.ID.TAGDIFF_NO_TAGS_FOUND)); } else //compare tag results; assumed (result1&2 == AnalyzeCommand.ExitCode.Success) { int count1 = 0; int sizeTags1 = analyze1.Metadata.UniqueTags.Count; string[] file1Tags = new string[sizeTags1]; foreach (string tag in analyze1.Metadata.UniqueTags.Keys.ToList <string>()) { file1Tags[count1++] = tag; } int count2 = 0; int sizeTags2 = analyze2.Metadata.UniqueTags.Count; string[] file2Tags = new string[sizeTags2]; foreach (string tag in analyze2.Metadata.UniqueTags.Keys.ToList <string>()) { file2Tags[count2++] = tag; } //can't simply compare counts as content may differ; must compare both in directions in two passes a->b; b->a equalTagsCompare1 = CompareTags(file1Tags, file2Tags, ref tagDiffResult, TagDiff.DiffSource.Source1); //reverse order for second pass equalTagsCompare2 = CompareTags(file2Tags, file1Tags, ref tagDiffResult, TagDiff.DiffSource.Source2); //final results bool resultsDiffer = !(equalTagsCompare1 && equalTagsCompare2); if (_arg_tagTestType == TagTestType.Inequality && !resultsDiffer) { tagDiffResult.ResultCode = TagDiffResult.ExitCode.TestFailed; } else if (_arg_tagTestType == TagTestType.Equality && resultsDiffer) { tagDiffResult.ResultCode = TagDiffResult.ExitCode.TestFailed; } else { tagDiffResult.ResultCode = TagDiffResult.ExitCode.TestPassed; } } } catch (OpException e) { WriteOnce.Verbosity = saveVerbosity; WriteOnce.Error(e.Message); //caught for CLI callers with final exit msg about checking log or throws for DLL callers throw; } return(tagDiffResult); }
/// <summary> /// Setup application inspector logging; 1 file per process /// </summary> /// <param name="opts"></param> /// <returns></returns> public static Logger SetupLogging(CommandOptions opts, bool onErrorConsole = false) { //prevent being called again if already set unless closed first if (Logger != null) { return(Logger); } var config = new NLog.Config.LoggingConfiguration(); if (String.IsNullOrEmpty(opts.LogFilePath)) { opts.LogFilePath = Utils.GetPath(Utils.AppPath.defaultLog); } //clean up previous for convenience in reading if (File.Exists(opts.LogFilePath)) { // Read the file and display it line by line. System.IO.StreamReader file = new System.IO.StreamReader(opts.LogFilePath); String line = file.ReadLine(); file.Close(); if (!String.IsNullOrEmpty(line)) { if (line.Contains("AppInsLog"))//prevent file other than our logs from deletion { File.Delete(opts.LogFilePath); } else { if (Utils.CLIExecutionContext && onErrorConsole) { WriteOnce.Error(MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_LOG_PATH, opts.LogFilePath), true, WriteOnce.ConsoleVerbosity.Low, false); } throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_LOG_PATH, opts.LogFilePath)); } } } else { try { File.WriteAllText(opts.LogFilePath, "");//verify log file path is writable } catch (Exception e) { WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error); if (Utils.CLIExecutionContext && onErrorConsole) { WriteOnce.Error(MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_FILE_OR_DIR, opts.LogFilePath), true, WriteOnce.ConsoleVerbosity.Low, false); } throw new OpException((MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_FILE_OR_DIR, opts.LogFilePath))); } } LogLevel log_level = LogLevel.Error;//default if (String.IsNullOrEmpty(opts.LogFileLevel)) { opts.LogFileLevel = "Error"; } try { log_level = LogLevel.FromString(opts.LogFileLevel); } catch (Exception) { if (Utils.CLIExecutionContext && onErrorConsole) { WriteOnce.Error(MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_ARG_VALUE, "-v"), true, WriteOnce.ConsoleVerbosity.Low, false); } throw new OpException((MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_ARG_VALUE, "-v"))); } using (var fileTarget = new FileTarget() { Name = "LogFile", FileName = opts.LogFilePath, Layout = @"${date:format=yyyy-MM-dd HH\:mm\:ss} ${threadid} ${level:uppercase=true} - AppInsLog - ${message}", ForceMutexConcurrentWrites = true }) { config.AddTarget(fileTarget); config.LoggingRules.Add(new LoggingRule("CST.ApplicationInspector", log_level, fileTarget)); } LogFilePath = opts.LogFilePath;//preserve for console path msg LogManager.Configuration = config; Logger = LogManager.GetLogger("CST.ApplicationInspector"); return(Logger); }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("TagTestCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "tagtest")); //init based on true or false present argument value ExitCode exitCode = ExitCode.CriticalError; WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; AnalyzeCommand.ExitCode analyzeCmdResult = AnalyzeCommand.ExitCode.CriticalError; try { //one file vs ruleset string tmp1 = Path.GetTempFileName(); //setup analyze call with silent option AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_srcPath, OutputFilePath = tmp1, OutputFileFormat = "json", IgnoreDefaultRules = true, CustomRulesPath = _arg_customRulesPath, FilePathExclusions = _arg_fileExclusionList, SimpleTagsOnly = true, ConsoleVerbosityLevel = "None", Log = _arg_logger }); //get and perform initial analyze on results analyzeCmdResult = (AnalyzeCommand.ExitCode)cmd1.Run(); //must be done here to avoid losing our handle from analyze command overwriting WriteOnce.Writer ConfigureFileOutput(); //restore WriteOnce.Verbosity = saveVerbosity; if (analyzeCmdResult == AnalyzeCommand.ExitCode.CriticalError) { throw new Exception(ErrMsg.GetString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR)); } else if (analyzeCmdResult == AnalyzeCommand.ExitCode.NoMatches) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low); if (_arg_tagTestType == TagTestType.RulesPresent) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "tagtest")); exitCode = _arg_tagTestType == TagTestType.RulesPresent ? ExitCode.TestFailed : ExitCode.TestPassed; } else //assumed (result == AnalyzeCommand.ExitCode.MatchesFound) { string file1TagsJson = File.ReadAllText(tmp1); var file1Tags = JsonConvert.DeserializeObject <TagsFile>(file1TagsJson); File.Delete(tmp1); exitCode = ExitCode.TestPassed; foreach (Rule r in _rulesSet) { //supports both directions by generalizing string[] testList1 = _arg_tagTestType == TagTestType.RulesNotPresent ? r.Tags : file1Tags.Tags; string[] testList2 = _arg_tagTestType == TagTestType.RulesNotPresent ? file1Tags.Tags : r.Tags; foreach (string t in testList2) { if (TagTest(testList1, t)) { WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_FOUND, t), true, WriteOnce.ConsoleVerbosity.High); } else { exitCode = ExitCode.TestFailed; WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_MISSING, t), true, WriteOnce.ConsoleVerbosity.High); } if (exitCode != ExitCode.TestPassed) { break; } } if (exitCode != ExitCode.TestPassed) { break; } } //results WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low); if (exitCode == ExitCode.TestFailed) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagtest")); if (!String.IsNullOrEmpty(_arg_outputFile) && Utils.CLIExecutionContext) { WriteOnce.Info(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, WriteOnce.ConsoleVerbosity.Low, false); } WriteOnce.FlushAll(); } } catch (Exception e) { WriteOnce.Verbosity = saveVerbosity; if (analyzeCmdResult == AnalyzeCommand.ExitCode.Success) //then error was not previously logged { WriteOnce.Error(e.Message); } else { WriteOnce.Error(e.Message, true, WriteOnce.ConsoleVerbosity.Low, false); } //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return((int)exitCode); }
/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType; try //fix for #146 { archiveFileType = MiniMagic.DetectFileType(filename); } catch (Exception e) { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILE_TYPE_OPEN, filename)); } if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); if (!_arg_autoBrowserOpen) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html")); } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.Success); }
public override int Run() { bool issues = false; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Verify Rules")); //load [each] rules file separately to report out where a failure is happening RuleSet rules = new RuleSet(WriteOnce.Log); IEnumerable <string> fileListing = new List <string>(); foreach (string rulePath in _rulePaths) { if (Directory.Exists(rulePath)) { fileListing = Directory.EnumerateFiles(rulePath, "*.json", SearchOption.AllDirectories); } else if (File.Exists(rulePath) && Path.GetExtension(rulePath) == ".json") { fileListing = new List <string>() { new string(rulePath) } } ; else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } //test loading each file foreach (string filename in fileListing) { try { rules.AddFile(filename); WriteOnce.Info(string.Format("Rule file added {0}", filename), true, WriteOnce.ConsoleVerbosity.High); } catch (Exception e) { WriteOnce.Error(string.Format("Rule file add failed {0}", filename)); WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error); issues = true; } } } //option to write validating data if (_arg_consoleVerbosityLevel == WriteOnce.ConsoleVerbosity.High) { WritePartialRuleDetails(rules); } //final status report if (issues) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Verify Rules")); WriteOnce.FlushAll(); if (!String.IsNullOrEmpty(_arg_outputFile)) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low); } return(issues ? (int)ExitCode.NotVerified : (int)ExitCode.Verified); }
public AppMetaData(string sourcePath, List <string> rulePaths) { //Initial value for ApplicationName may be replaced if rule pattern match found later if (Directory.Exists(sourcePath)) { try { ApplicationName = sourcePath.Substring(sourcePath.LastIndexOf(Path.DirectorySeparatorChar)).Replace(Path.DirectorySeparatorChar, ' ').Trim(); } catch (Exception) { ApplicationName = Path.GetFileNameWithoutExtension(sourcePath); } } else { ApplicationName = Path.GetFileNameWithoutExtension(sourcePath); } //initialize standard set groups using dynamic lists variables that may have more than one value; some are filled //using tag tests and others by different means like file type examination KeyedPropertyLists = new Dictionary <string, HashSet <string> > { ["strGrpRulePaths"] = rulePaths.ToHashSet(), ["strGrpPackageTypes"] = new HashSet <string>(), ["strGrpAppTypes"] = new HashSet <string>(), ["strGrpFileTypes"] = new HashSet <string>(), ["strGrpUniqueTags"] = new HashSet <string>(), ["strGrpOutputs"] = new HashSet <string>(), ["strGrpTargets"] = new HashSet <string>(), ["strGrpOSTargets"] = new HashSet <string>(), ["strGrpFileExtensions"] = new HashSet <string>(), ["strGrpFileNames"] = new HashSet <string>(), ["strGrpCPUTargets"] = new HashSet <string>(), ["strGrpCloudTargets"] = new HashSet <string>(), ["strGrpUniqueDependencies"] = new HashSet <string>() }; //predefined standard tags to track; only some are propertygrouplist are tag based _propertyTagSearchPatterns = new Dictionary <string, string>(); _propertyTagSearchPatterns.Add("strGrpOSTargets", ".OS.Targets"); _propertyTagSearchPatterns.Add("strGrpCloudTargets", ".Cloud"); _propertyTagSearchPatterns.Add("strGrpOutputs", ".Outputs"); _propertyTagSearchPatterns.Add("strGrpCPUTargets", ".CPU"); //read default/user preferences on what tags to count if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref))) { TagCounters = JsonConvert.DeserializeObject <List <TagCounter> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref))); } else { TagCounters = new List <TagCounter>(); } HashSet <string> dupCountersCheck = new HashSet <string>(); foreach (TagCounter counter in TagCounters) { if (!dupCountersCheck.Add(counter.Tag)) { WriteOnce.SafeLog("Duplidate counter specified in preferences", NLog.LogLevel.Error); } } Languages = new Dictionary <string, int>(); }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public TagDiffResult GetResult() { WriteOnce.SafeLog("TagDiffCommand::Run", LogLevel.Trace); WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Tag Diff")); TagDiffResult tagDiffResult = new() { AppVersion = Common.Utils.GetVersionString() }; //save to quiet analyze cmd and restore WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; try { if (_options is null) { throw new ArgumentNullException("_options"); } AnalyzeCommand cmd1 = new(new AnalyzeOptions() { SourcePath = _options.SourcePath1, CustomRulesPath = _options.CustomRulesPath, IgnoreDefaultRules = _options.IgnoreDefaultRules, FilePathExclusions = _options.FilePathExclusions, ConsoleVerbosityLevel = "none", Log = _options.Log, TagsOnly = true, ConfidenceFilters = _options.ConfidenceFilters, FileTimeOut = _options.FileTimeOut, ProcessingTimeOut = _options.ProcessingTimeOut, NoFileMetadata = true, NoShowProgress = true, ScanUnknownTypes = _options.ScanUnknownTypes, SingleThread = _options.SingleThread, }); AnalyzeCommand cmd2 = new(new AnalyzeOptions() { SourcePath = _options.SourcePath2, CustomRulesPath = _options.CustomRulesPath, IgnoreDefaultRules = _options.IgnoreDefaultRules, FilePathExclusions = _options.FilePathExclusions, ConsoleVerbosityLevel = "none", Log = _options.Log, TagsOnly = true, ConfidenceFilters = _options.ConfidenceFilters, FileTimeOut = _options.FileTimeOut, ProcessingTimeOut = _options.ProcessingTimeOut, NoFileMetadata = true, NoShowProgress = true, ScanUnknownTypes = _options.ScanUnknownTypes, SingleThread = _options.SingleThread, }); AnalyzeResult analyze1 = cmd1.GetResult(); AnalyzeResult analyze2 = cmd2.GetResult(); //restore WriteOnce.Verbosity = saveVerbosity; //process results for each analyze call before comparing results if (analyze1.ResultCode == AnalyzeResult.ExitCode.CriticalError) { throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_CRITICAL_FILE_ERR, string.Join(',', _options.SourcePath1))); } else if (analyze2.ResultCode == AnalyzeResult.ExitCode.CriticalError) { throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_CRITICAL_FILE_ERR, string.Join(',', _options.SourcePath2))); } else if (analyze1.ResultCode == AnalyzeResult.ExitCode.NoMatches || analyze2.ResultCode == AnalyzeResult.ExitCode.NoMatches) { throw new OpException(MsgHelp.GetString(MsgHelp.ID.TAGDIFF_NO_TAGS_FOUND)); } else //compare tag results; assumed (result1&2 == AnalyzeCommand.ExitCode.Success) { var list1 = analyze1.Metadata.UniqueTags ?? new List <string>(); var list2 = analyze2.Metadata.UniqueTags ?? new List <string>(); var removed = list1.Except(list2); var added = list2.Except(list1); foreach (var add in added) { tagDiffResult.TagDiffList.Add(new TagDiff() { Source = TagDiff.DiffSource.Source2, Tag = add }); } foreach (var remove in removed) { tagDiffResult.TagDiffList.Add(new TagDiff() { Source = TagDiff.DiffSource.Source1, Tag = remove }); } if (tagDiffResult.TagDiffList.Count > 0) { tagDiffResult.ResultCode = _arg_tagTestType == TagTestType.Inequality ? TagDiffResult.ExitCode.TestPassed : TagDiffResult.ExitCode.TestFailed; } else { tagDiffResult.ResultCode = _arg_tagTestType == TagTestType.Inequality ? TagDiffResult.ExitCode.TestFailed : TagDiffResult.ExitCode.TestPassed; } return(tagDiffResult); } } catch (OpException e) { WriteOnce.Verbosity = saveVerbosity; WriteOnce.Error(e.Message); //caught for CLI callers with final exit msg about checking log or throws for DLL callers throw; } } }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public TagTestResult GetResult() { WriteOnce.SafeLog("TagTestCommand::Run", LogLevel.Trace); WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Tag Test")); TagTestResult tagTestResult = new TagTestResult() { AppVersion = Utils.GetVersionString() }; //init based on true or false present argument value WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; AnalyzeResult.ExitCode analyzeCmdResult = AnalyzeResult.ExitCode.CriticalError; try { //setup analyze call with silent option AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeOptions { SourcePath = _options.SourcePath, IgnoreDefaultRules = true, CustomRulesPath = _options.CustomRulesPath, FilePathExclusions = _options.FilePathExclusions, ConsoleVerbosityLevel = "none", Log = _options.Log }); //get and perform initial analyze on results AnalyzeResult analyze1 = cmd1.GetResult(); //restore WriteOnce.Verbosity = saveVerbosity; if (analyze1.ResultCode == AnalyzeResult.ExitCode.CriticalError) { throw new OpException(MsgHelp.GetString(MsgHelp.ID.CMD_CRITICAL_FILE_ERR)); } else if (analyzeCmdResult == AnalyzeResult.ExitCode.NoMatches) { WriteOnce.General(MsgHelp.FormatString(MsgHelp.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low); tagTestResult.ResultCode = _arg_tagTestType == TagTestType.RulesPresent ? TagTestResult.ExitCode.TestFailed : TagTestResult.ExitCode.TestPassed; } else //assumed (result == AnalyzeCommand.ExitCode.MatchesFound) { tagTestResult.ResultCode = TagTestResult.ExitCode.TestPassed; int count = 0; int sizeTags = analyze1.Metadata.UniqueTags.Count; string[] tagsFound = new string[sizeTags]; foreach (string tag in analyze1.Metadata.UniqueTags.Keys.ToList <string>()) { tagsFound[count++] = tag; } foreach (Rule rule in _rulesSet) { //supports both directions by generalizing string[] testList1 = _arg_tagTestType == TagTestType.RulesNotPresent ? rule.Tags : tagsFound; string[] testList2 = _arg_tagTestType == TagTestType.RulesNotPresent ? tagsFound : rule.Tags; foreach (string t in testList2) { if (TagTest(testList1, t)) { tagTestResult.TagsStatusList.Add(new TagStatus() { Tag = t, Detected = true }); } else { tagTestResult.ResultCode = TagTestResult.ExitCode.TestFailed; tagTestResult.TagsStatusList.Add(new TagStatus() { Tag = t, Detected = false }); } } } } } catch (OpException e) { WriteOnce.Verbosity = saveVerbosity; WriteOnce.Error(e.Message); //caught for CLI callers with final exit msg about checking log or throws for DLL callers throw; } return(tagTestResult); }
/// <summary> /// Option for DLL use as alternate to Run which only outputs a file to return results as string /// CommandOption defaults will not have been set when used as DLL via CLI processing so some checks added /// </summary> /// <returns>output results</returns> public VerifyRulesResult GetResult() { WriteOnce.SafeLog("VerifyRulesCommand::Run", LogLevel.Trace); WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Verify Rules")); VerifyRulesResult verifyRulesResult = new() { AppVersion = Utils.GetVersionString() }; try { RulesVerifier verifier = new(null, _options.Log); verifyRulesResult.ResultCode = VerifyRulesResult.ExitCode.Verified; var stati = new List <RuleStatus>(); var analyzer = new Analyzer(); analyzer.SetOperation(new WithinOperation(analyzer)); analyzer.SetOperation(new OATRegexWithIndexOperation(analyzer)); analyzer.SetOperation(new OATSubstringIndexOperation(analyzer)); RuleSet?ruleSet = new(_options.Log); if (_options.VerifyDefaultRules) { ruleSet = RuleSetUtils.GetDefaultRuleSet(_options.Log); } try { if (_options.CustomRulesPath != null) { if (Directory.Exists(_options.CustomRulesPath)) { ruleSet.AddDirectory(_options.CustomRulesPath); } else if (File.Exists(_options.CustomRulesPath)) { ruleSet.AddFile(_options.CustomRulesPath); } } } catch (JsonSerializationException e) { WriteOnce.Error(e.Message); verifyRulesResult.ResultCode = VerifyRulesResult.ExitCode.CriticalError; return(verifyRulesResult); } foreach (var rule in ruleSet.GetOatRules()) { stati.Add(new RuleStatus() { RulesId = rule.AppInspectorRule.Id, RulesName = rule.Name, Verified = verifier.Verify(rule.AppInspectorRule), OatIssues = analyzer.EnumerateRuleIssues(rule) }); } verifyRulesResult.RuleStatusList = stati; verifyRulesResult.ResultCode = stati.All(x => x.Verified && !x.OatIssues.Any()) ? VerifyRulesResult.ExitCode.Verified : VerifyRulesResult.ExitCode.NotVerified; } catch (OpException e) { WriteOnce.Error(e.Message); //caught for CLI callers with final exit msg about checking log or throws for DLL callers throw; } return(verifyRulesResult); } }
/// <summary> /// Add default and/or custom rules paths /// Iterate paths and add to ruleset /// </summary> void ConfigRules() { WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace); RuleSet rulesSet = new RuleSet(_arg_logger); List <string> rulePaths = new List <string>(); if (!_arg_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string[] resourceName = assembly.GetManifestResourceNames(); string filePath = "Microsoft.ApplicationInspector.Commands.defaultRules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { rulesSet.AddString(file.ReadToEnd(), filePath, null); } } if (!string.IsNullOrEmpty(_arg_customRulesPath)) { rulePaths.Add(_arg_customRulesPath); } foreach (string rulePath in rulePaths) { if (Directory.Exists(rulePath)) { rulesSet.AddDirectory(rulePath); } else if (File.Exists(rulePath)) { rulesSet.AddFile(rulePath); } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } } //error check based on ruleset not path enumeration if (rulesSet.Count() == 0) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED)); } //instantiate a RuleProcessor with the added rules and exception for dependency _rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, _arg_logger); if (_arg_outputUniqueTagsOnly) { List <TagException> tagExceptions; if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref))) { tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref))); string[] exceptions = new string[tagExceptions.Count]; for (int i = 0; i < tagExceptions.Count; i++) { exceptions[i] = tagExceptions[i].Tag; } _rulesProcessor.UniqueTagExceptions = exceptions; } } _appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly, _arg_autoBrowserOpen); _appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " + WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower(); }
//Main entry from CLI public override int Run() { WriteOnce.SafeLog("ExportTagsCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Exporttags")); SortedDictionary <string, string> uniqueTags = new SortedDictionary <string, string>(); try { foreach (Rule r in _rules) { //builds a list of unique tags foreach (string t in r.Tags) { if (uniqueTags.ContainsKey(t)) { continue; } else { uniqueTags.Add(t, t); } } } //separate loop so results are sorted (Sorted type) foreach (string s in uniqueTags.Values) { WriteOnce.Result(s, true); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Exporttags")); if (!String.IsNullOrEmpty(_arg_outputFile) && Utils.CLIExecutionContext) { WriteOnce.Info(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, WriteOnce.ConsoleVerbosity.Low, false); } WriteOnce.FlushAll(); } catch (Exception e) { WriteOnce.Error(e.Message); //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return((int)ExitCode.Success); }