/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public override int Run() { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "tagtest")); //init based on true or false present argument value bool testSuccess = true; //one file vs ruleset string tmp1 = Path.GetTempFileName(); WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; AnalyzeCommand.ExitCode result = AnalyzeCommand.ExitCode.CriticalError; //setup analyze call with silent option #region analyzesetup try { AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_srcPath, OutputFilePath = tmp1, OutputFileFormat = "json", CustomRulesPath = _arg_customRulesPath, SimpleTagsOnly = true, ConsoleVerbosityLevel = "None", Log = _arg_logger }); //quiet analysis commands result = (AnalyzeCommand.ExitCode)cmd1.Run(); } catch (Exception e) { WriteOnce.Verbosity = saveVerbosity; throw e; } //restore WriteOnce.Verbosity = saveVerbosity; if (result == AnalyzeCommand.ExitCode.CriticalError) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR)); } else if (result == AnalyzeCommand.ExitCode.NoMatches) { //results WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low); if (_arg_tagTestType == TagTestType.RulesPresent) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.FlushAll(); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagtest")); return((int)ExitCode.CriticalError); } #endregion //assumed (result == AnalyzeCommand.ExitCode.MatchesFound) string file1TagsJson = File.ReadAllText(tmp1); var file1TagsObj = JsonConvert.DeserializeObject <TagsFile[]>(file1TagsJson); var file1Tags = file1TagsObj.First(); // here we have a single FileList object File.Delete(tmp1); foreach (Rule r in _rulesSet) { //supports both directions by generalizing string[] testList1 = _arg_tagTestType == TagTestType.RulesNotPresent ? r.Tags : file1Tags.Tags; string[] testList2 = _arg_tagTestType == TagTestType.RulesNotPresent ? file1Tags.Tags : r.Tags; foreach (string t in testList2) { if (TagTest(testList1, t)) { WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_FOUND, t), true, WriteOnce.ConsoleVerbosity.High); } else { testSuccess = false; WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_MISSING, t), true, WriteOnce.ConsoleVerbosity.High); } } } //results WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low); if (testSuccess) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.FlushAll(); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagtest")); return(testSuccess ? (int)ExitCode.NoDiff : (int)ExitCode.DiffFound); }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public override int Run() { bool issues = false; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Verify Rules")); //load [each] rules file separately to report out where a failure is happening RuleSet rules = new RuleSet(WriteOnce.Log); IEnumerable <string> fileListing = new List <string>(); foreach (string rulePath in _rulePaths) { if (Directory.Exists(rulePath)) { fileListing = Directory.EnumerateFiles(rulePath, "*.json", SearchOption.AllDirectories); } else if (File.Exists(rulePath) && Path.GetExtension(rulePath) == ".json") { fileListing = new List <string>() { new string(rulePath) } } ; else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } //test loading each file foreach (string filename in fileListing) { try { rules.AddFile(filename); WriteOnce.Info(string.Format("Rule file added {0}", filename), true, WriteOnce.ConsoleVerbosity.High); } catch (Exception e) { WriteOnce.Error(string.Format("Rule file add failed {0}", filename)); WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error); issues = true; } } } //option to write validating data if (_arg_consoleVerbosityLevel.ToLower() == "high") { WritePartialRuleDetails(rules); } //final status report if (issues) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Verify Rules")); WriteOnce.FlushAll(); if (!String.IsNullOrEmpty(_arg_outputFile)) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low); } return(issues ? (int)ExitCode.NotVerified : (int)ExitCode.Verified); }
/// <summary> /// Main WORKHORSE for analyzing file; called from file based or decompression functions /// </summary> /// <param name="filename"></param> /// <param name="fileText"></param> void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo) { #region minorRollupTrackingAndProgress WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace); _appProfile.MetaData.FilesAnalyzed++; int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped; int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100); //earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI if (percentCompleted > 100) { percentCompleted = 100; } else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output { WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false); } #endregion //process file against rules Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo); //if any matches found for this file... if (matches.Count() > 0) { _appProfile.MetaData.FilesAffected++; _appProfile.MetaData.TotalMatchesCount += matches.Count(); // Iterate through each match issue foreach (Issue match in matches) { WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace); //maintain a list of unique tags; multi-purpose but primarily for filtering -d option bool dupTagFound = false; foreach (string t in match.Rule.Tags) { dupTagFound = !_uniqueTagsControl.Add(t); } //save all unique dependencies even if Dependency tag pattern is not-unique var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase); String textMatch; if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v))) { textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name); } else { textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length); } //wrap rule issue result to add metadata MatchRecord record = new MatchRecord() { Filename = filePath, Language = languageInfo, Filesize = fileText.Length, TextSample = textMatch, Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line), Issue = match }; //preserve issue level characteristics as rolled up meta data of interest bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record); //bail after extracting any dependency unique items IF user requested if (_arg_outputUniqueTagsOnly && dupTagFound) { continue; } else if (addAsFeatureMatch) { _appProfile.MatchList.Add(record); } } } else { WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace); } }
/// <summary> /// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType; try //fix for #146 { archiveFileType = MiniMagic.DetectFileType(filename); } catch (Exception e) { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILE_TYPE_OPEN, filename)); } if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); if (!_arg_autoBrowserOpen) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low); } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.Success); }
/// <summary> /// Add default and/or custom rules paths /// Iterate paths and add to ruleset /// </summary> void ConfigRules() { WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace); RuleSet rulesSet = new RuleSet(_arg_logger); List <string> rulePaths = new List <string>(); if (!_arg_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string[] resourceName = assembly.GetManifestResourceNames(); string filePath = "Microsoft.ApplicationInspector.Commands.defaultRules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { rulesSet.AddString(file.ReadToEnd(), filePath, null); } } if (!string.IsNullOrEmpty(_arg_customRulesPath)) { rulePaths.Add(_arg_customRulesPath); } foreach (string rulePath in rulePaths) { if (Directory.Exists(rulePath)) { rulesSet.AddDirectory(rulePath); } else if (File.Exists(rulePath)) { rulesSet.AddFile(rulePath); } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } } //error check based on ruleset not path enumeration if (rulesSet.Count() == 0) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED)); } //instantiate a RuleProcessor with the added rules and exception for dependency _rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, _arg_logger); if (_arg_outputUniqueTagsOnly) { List <TagException> tagExceptions; if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref))) { tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref))); string[] exceptions = new string[tagExceptions.Count]; for (int i = 0; i < tagExceptions.Count; i++) { exceptions[i] = tagExceptions[i].Tag; } _rulesProcessor.UniqueTagExceptions = exceptions; } } _appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly, _arg_autoBrowserOpen); _appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " + WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower(); }
//Main entry from CLI public override int Run() { WriteOnce.SafeLog("ExportTagsCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Exporttags")); SortedDictionary <string, string> uniqueTags = new SortedDictionary <string, string>(); try { foreach (Rule r in _rules) { //builds a list of unique tags foreach (string t in r.Tags) { if (uniqueTags.ContainsKey(t)) { continue; } else { uniqueTags.Add(t, t); } } } //separate loop so results are sorted (Sorted type) foreach (string s in uniqueTags.Values) { WriteOnce.Result(s, true); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Exporttags")); if (!String.IsNullOrEmpty(_arg_outputFile) && Utils.CLIExecutionContext) { WriteOnce.Info(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, WriteOnce.ConsoleVerbosity.Low, false); } WriteOnce.FlushAll(); } catch (Exception e) { WriteOnce.Error(e.Message); //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return((int)ExitCode.Success); }
/// <summary> /// Main entry from CLI /// </summary> /// <returns></returns> public override int Run() { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Tagdiff")); if (_arg_src1 == _arg_src2) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_SAME_FILE_ARG)); } else if (string.IsNullOrEmpty(_arg_src1) || string.IsNullOrEmpty(_arg_src2)) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_INVALID_ARG_VALUE)); } #region setup analyze calls //save to quiet analyze cmd WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity; string tmp1 = Path.GetTempFileName(); string tmp2 = Path.GetTempFileName(); AnalyzeCommand.ExitCode result1 = AnalyzeCommand.ExitCode.CriticalError; AnalyzeCommand.ExitCode result2 = AnalyzeCommand.ExitCode.CriticalError; try { AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_src1, OutputFilePath = tmp1, OutputFileFormat = "json", CustomRulesPath = _arg_customRulesPath, IgnoreDefaultRules = _arg_ignoreDefault, SimpleTagsOnly = true, ConsoleVerbosityLevel = "none", Log = _arg_logger }); AnalyzeCommand cmd2 = new AnalyzeCommand(new AnalyzeCommandOptions { SourcePath = _arg_src2, OutputFilePath = tmp2, OutputFileFormat = "json", CustomRulesPath = _arg_customRulesPath, IgnoreDefaultRules = _arg_ignoreDefault, SimpleTagsOnly = true, ConsoleVerbosityLevel = "none", Log = _arg_logger }); result1 = (AnalyzeCommand.ExitCode)cmd1.Run(); result2 = (AnalyzeCommand.ExitCode)cmd2.Run(); } catch (Exception e) { //restore WriteOnce.Verbosity = saveVerbosity; throw e; } //restore WriteOnce.Verbosity = saveVerbosity; #endregion bool successResult; bool equal1 = true; bool equal2 = true; //process results for each analyze call before comparing results if (result1 == AnalyzeCommand.ExitCode.CriticalError) { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src1)); } else if (result2 == AnalyzeCommand.ExitCode.CriticalError) { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src2)); } else if (result1 == AnalyzeCommand.ExitCode.NoMatches || result2 == AnalyzeCommand.ExitCode.NoMatches) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_NO_TAGS_FOUND)); } else //compare tag results; assumed (result1&2 == AnalyzeCommand.ExitCode.MatchesFound) { //setup output here rather than top to avoid analyze command output in this command output TextWriter outputWriter; if (!string.IsNullOrEmpty(_arg_outputFile)) { outputWriter = File.CreateText(_arg_outputFile); outputWriter.WriteLine(Utils.GetVersionString()); WriteOnce.Writer = outputWriter; } string file1TagsJson = File.ReadAllText(tmp1); string file2TagsJson = File.ReadAllText(tmp2); var file1Tags = JsonConvert.DeserializeObject <TagsFile>(file1TagsJson); var file2Tags = JsonConvert.DeserializeObject <TagsFile>(file2TagsJson); //can't simply compare counts as content may differ; must compare both in directions in two passes a->b; b->a //first pass WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src1), Path.GetFileName(_arg_src2)), true, WriteOnce.ConsoleVerbosity.High); equal1 = CompareTags(file1Tags.Tags, file2Tags.Tags); //reverse order for second pass WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src2), Path.GetFileName(_arg_src1)), true, WriteOnce.ConsoleVerbosity.High); equal2 = CompareTags(file2Tags.Tags, file1Tags.Tags); //final results bool resultsDiffer = !(equal1 && equal2); if (_arg_tagTestType == TagTestType.Inequality && resultsDiffer) { successResult = true; } else if (_arg_tagTestType == TagTestType.Equality && !resultsDiffer) { successResult = true; } else { successResult = false; } WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_RESULTS_DIFFER), false); WriteOnce.Result(resultsDiffer.ToString()); } //cleanup try { File.Delete(tmp1); File.Delete(tmp2); } catch { //no action needed; } WriteOnce.FlushAll(); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagdiff")); return(successResult ? (int)ExitCode.TestPassed : (int)ExitCode.TestFailed); }