public void ScopeTest() { RuleProcessor processor = new RuleProcessor(LoadRules(false)) { EnableSuppressions = true }; // Ignore inline comment string testString = "var hash = MD5.Create(); // MD5 is not allowed"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "MD5 in inline comment should be ignored"); Assert.AreEqual(11, issues[0].Boundary.Index, "MD5 inline index is wrong"); // ignore multinline comment testString = " /*\r\nMD5 is not allowed\r\n */ \r\nvar hash = MD5.Create();"; issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "MD5 in multi line comment should be ignored"); Assert.AreEqual(42, issues[0].Boundary.Index, "MD5 multi line index is wrong"); // TODO test testString = "//TODO: fix it later"; processor.SeverityLevel |= Severity.ManualReview; issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "TODO should be flagged"); }
public void UseCase_SeverityFilter_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); string testString = "eval(something)"; Issue[] issues = processor.Analyze(testString, "javascript"); Assert.AreEqual(0, issues.Length, "Manual Review should not be flagged"); processor.SeverityLevel |= Severity.ManualReview; issues = processor.Analyze(testString, "javascript"); Assert.AreEqual(1, issues.Length, "Manual Review should be flagged"); }
public void UseCase_Normal_Test() { Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null); rules.AddDirectory(@"rules\custom", "my rules"); RuleProcessor processor = new RuleProcessor(rules); string lang = Language.FromFileName("testfilename.cpp"); string testString = "strcpy(dest,src);"; // strcpy test Issue[] issues = processor.Analyze(testString, lang); Assert.AreEqual(1, issues.Length, "strcpy should be flagged"); Assert.AreEqual(0, issues[0].Index, "strcpy invalid index"); Assert.AreEqual(16, issues[0].Length, "strcpy invalid length "); Assert.AreEqual("DS185832", issues[0].Rule.Id, "strcpy invalid rule"); // Fix it test Assert.AreNotEqual(issues[0].Rule.Fixes.Length, 0, "strcpy invalid Fixes"); CodeFix fix = issues[0].Rule.Fixes[0]; string fixedCode = RuleProcessor.Fix(testString, fix); Assert.AreEqual("strcpy_s(dest, <size of dest>, src);", fixedCode, "strcpy invalid code fix"); Assert.IsTrue(fix.Name.Contains("Change to strcpy_s"), "strcpy wrong fix name"); // QUICKFIX test processor.SeverityLevel |= Severity.ManualReview; testString = "//QUICKFIX: fix this later"; issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "QUICKFIX should be flagged"); Assert.AreEqual(2, issues[0].Index, "QUICKFIX invalid index"); Assert.AreEqual(8, issues[0].Length, "QUICKFIX invalid length "); Assert.AreEqual("DS276209", issues[0].Rule.Id, "QUICKFIX invalid rule"); Assert.AreEqual(0, issues[0].Rule.Fixes.Length, "QUICKFIX invalid Fixes"); Assert.AreEqual("my rules", issues[0].Rule.RuntimeTag, "QUICKFIX invalid tag"); // Same issue twice test testString = "MD5 hash = MD5.Create();"; issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "Same issue should be twice on line"); Assert.AreEqual(issues[0].Rule, issues[1].Rule, "Same issues should have sames rule IDs"); // Overlaping issues testString = " MD5 hash = new MD5CryptoServiceProvider();"; issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "Overlaping issue count doesn't add up"); // Override test testString = "strncat(dest, \"this is also bad\", strlen(dest))"; issues = processor.Analyze(testString, new string[] { "c", "cpp" }); Assert.AreEqual(2, issues.Length, "Override test failed"); }
public void IsMatch_FalseTest() { RuleSet ruleset = RuleSet.FromDirectory(Path.Combine("rules", "valid"), null); RuleProcessor processor = new RuleProcessor(ruleset); string testString = "this is a test string"; // Normal functionality test Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(0, issues.Length, "Match.Success should be false"); // Non existent langugage issues = processor.Analyze(testString, ""); Assert.AreEqual(0, issues.Length, "Match.Success should be false, when no language is passed"); }
public void UseCase_Normal_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); string lang = Language.FromFileName("testfilename.cpp"); string testString = "strcpy(dest,src);"; // strcpy test Issue[] issues = processor.Analyze(testString, lang); Assert.AreEqual(1, issues.Length, "strcpy should be flagged"); Assert.AreEqual(0, issues[0].Boundary.Index, "strcpy invalid index"); Assert.AreEqual(16, issues[0].Boundary.Length, "strcpy invalid length "); Assert.AreEqual(1, issues[0].StartLocation.Line, "strcpy invalid start location line "); Assert.AreEqual(1, issues[0].StartLocation.Column, "strcpy invalid start location column "); Assert.AreEqual(1, issues[0].EndLocation.Line, "strcpy invalid end location line "); Assert.AreEqual(16, issues[0].EndLocation.Column, "strcpy invalid end location column "); Assert.AreEqual("DS185832", issues[0].Rule.Id, "strcpy invalid rule"); // Fix it test Assert.AreNotEqual(issues[0].Rule.Fixes.Length, 0, "strcpy invalid Fixes"); CodeFix fix = issues[0].Rule.Fixes[0]; string fixedCode = RuleProcessor.Fix(testString, fix); Assert.AreEqual("strcpy_s(dest, <size of dest>, src);", fixedCode, "strcpy invalid code fix"); Assert.IsTrue(fix.Name.Contains("Change to strcpy_s"), "strcpy wrong fix name"); }
public void UseCase_SeverityFilter_Test() { Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null); rules.AddDirectory(@"rules\custom", null); RuleProcessor processor = new RuleProcessor(rules); string testString = "eval(something)"; Issue[] issues = processor.Analyze(testString, "javascript"); Assert.AreEqual(0, issues.Length, "Manual Review should not be flagged"); processor.SeverityLevel |= Severity.ManualReview; issues = processor.Analyze(testString, "javascript"); Assert.AreEqual(1, issues.Length, "Manual Review should be flagged"); }
public void IsMatch_InvalidLanguageTest() { RuleProcessor processor = new RuleProcessor(); string testString = "this is a test string"; // Langugage is empty Issue[] issues = processor.Analyze(testString, string.Empty); }
public void IsMatch_InvalidInputTest() { RuleProcessor processor = new RuleProcessor(); // Langugage is null Issue[] issues = processor.Analyze(null, ""); Assert.AreEqual(0, issues.Length, "Match.Success should be false"); }
public void UseCase_OverlapingIssues_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); // Overlaping issues string testString = " MD5 hash = new MD5CryptoServiceProvider();"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "Overlaping issue count doesn't add up"); }
public void UseCase_OverrideTest_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); // Override test string testString = "strncat(dest, \"this is also bad\", strlen(dest))"; Issue[] issues = processor.Analyze(testString, new string[] { "c", "cpp" }); Assert.AreEqual(2, issues.Length, "Override test failed"); }
public void UseCase_EmptyString_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); // Empty string test string testString = ""; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(0, issues.Length, "Empty test failed"); }
public void UseCase_IgnoreRules_Test() { Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null); rules.AddDirectory(@"rules\custom", null); RuleProcessor processor = new RuleProcessor(rules) { EnableSuppressions = true }; // MD5CryptoServiceProvider test string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "MD5CryptoServiceProvider should be flagged"); Assert.AreEqual(15, issues[0].Index, "MD5CryptoServiceProvider invalid index"); Assert.AreEqual(24, issues[0].Length, "MD5CryptoServiceProvider invalid length "); Assert.AreEqual("DS168931", issues[0].Rule.Id, "MD5CryptoServiceProvider invalid rule"); // Ignore until test DateTime expirationDate = DateTime.Now.AddDays(5); testString = "requests.get('somelink', verify = False) #DevSkim: ignore DS130821 until {0:yyyy}-{0:MM}-{0:dd}"; issues = processor.Analyze(string.Format(testString, expirationDate), "python"); Assert.AreEqual(0, issues.Length, "Ignore until should not be flagged"); // Expired until test expirationDate = DateTime.Now; issues = processor.Analyze(string.Format(testString, expirationDate), "python"); Assert.AreEqual(1, issues.Length, "Expired until should be flagged"); // Ignore all until test expirationDate = DateTime.Now.AddDays(5); testString = "MD5 hash = new MD5.Create(); #DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}"; issues = processor.Analyze(string.Format(testString, expirationDate), "csharp"); Assert.AreEqual(0, issues.Length, "Ignore all until should not be flagged"); // Expired all test expirationDate = DateTime.Now; testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}"; issues = processor.Analyze(string.Format(testString, expirationDate), "csharp"); Assert.AreEqual(2, issues.Length, "Expired all should be flagged"); }
public void UseCase_IssueTwice_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); // Same issue twice test string testString = "MD5 hash = MD5.Create();"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "Same issue should be twice on line"); Assert.AreEqual(issues[0].Rule, issues[1].Rule, "Same issues should have sames rule IDs"); }
public void Conditions1Test() { RuleProcessor processor = new RuleProcessor(LoadRules(false)) { EnableSuppressions = true }; // http test string testString = "<h:table xmlns:h=\"http://www.w3.org/TR/html4/\">"; Issue[] issues = processor.Analyze(testString, "xml"); Assert.AreEqual(0, issues.Length, "http should NOT be flagged"); // http test testString = "<h:table src=\"http://www.w3.org/TR/html4/\">"; issues = processor.Analyze(testString, "xml"); Assert.AreEqual(1, issues.Length, "http should be flagged"); Assert.AreEqual(1, issues[0].StartLocation.Line, "http location line doesn't match"); Assert.AreEqual(14, issues[0].Boundary.Index, "http index doesn't match"); Assert.AreEqual(5, issues[0].Boundary.Length, "http length doesn't match"); Assert.AreEqual("DS137138", issues[0].Rule.Id, "http rule doesn't match"); }
public void Conditions2Test() { RuleProcessor processor = new RuleProcessor(LoadRules(false)) { EnableSuppressions = true }; // $POST test string testString = "require_once($_POST['t']);"; Issue[] issues = processor.Analyze(testString, "php"); Assert.AreEqual(1, issues.Length, "$_POST should be flagged"); Assert.AreEqual(1, issues[0].StartLocation.Line, "$_POST location line doesn't match"); Assert.AreEqual(0, issues[0].Boundary.Index, "$_POST index doesn't match"); Assert.AreEqual(19, issues[0].Boundary.Length, "$_POST length doesn't match"); Assert.AreEqual("DS181731", issues[0].Rule.Id, "$_POST rule doesn't match"); // $POST test testString = "echo(urlencode($_POST['data']);"; issues = processor.Analyze(testString, "php"); Assert.AreEqual(0, issues.Length, "$_POST should not be flagged"); }
public void UseCase_IgnoreAllRules_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = true }; // Ignore all until test DateTime expirationDate = DateTime.Now.AddDays(5); string testString = "encryption=false; MD5 hash = MD5.Create(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}"; Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "csharp"); Assert.AreEqual(3, issues.Length, "Ignore all should flag two infos"); }
public void UseCase_IgnoreAllRulesExpired_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = true }; // Expired all test DateTime expirationDate = DateTime.Now; string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}"; Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "csharp"); Assert.AreEqual(2, issues.Length, "Expired all should be flagged"); }
public void UseCase_IgnoreRulesExpired_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = true }; // Expired until test DateTime expirationDate = DateTime.Now; string testString = "requests.get('somelink', verify = False) #DevSkim: ignore DS126186 until {0:yyyy}-{0:MM}-{0:dd}"; Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "python"); Assert.AreEqual(1, issues.Length, "Expired until should be flagged"); Assert.AreEqual(false, issues[0].IsSuppressionInfo, "Expired until issue should NOT be info"); }
public void UseCase_IgnoreSuppression_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = false }; // MD5CryptoServiceProvider test string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged"); Assert.AreEqual(0, issues[1].Boundary.Index, "MD5CryptoServiceProvider invalid index"); Assert.AreEqual(3, issues[1].Boundary.Length, "MD5CryptoServiceProvider invalid length "); Assert.AreEqual("DS126858", issues[1].Rule.Id, "MD5CryptoServiceProvider invalid rule"); }
public void RuleInfoTest() { RuleSet ruleset = RuleSet.FromDirectory(Path.Combine("rules", "valid"), null); RuleProcessor processor = new RuleProcessor(ruleset); string testString = "strcpy(dest,src);"; Issue[] issues = processor.Analyze(testString, "cpp"); Assert.AreEqual(1, issues.Length, "strcpy should be flagged"); Rule r = issues[0].Rule; Assert.IsTrue(r.Description.Contains("strcpy"), "Invalid decription"); Assert.IsTrue(r.Name.Contains("strcpy"), "Invalid name"); Assert.IsTrue(r.Recommendation.Contains("strcpy_s"), "Invalid replacement"); Assert.IsTrue(r.RuleInfo.Contains(r.Id), "Invalid ruleinfo"); }
public void UseCase_ManualReview_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)); // QUICKFIX test processor.SeverityLevel |= Severity.ManualReview; string testString = "//QUICKFIX: fix this later"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(1, issues.Length, "QUICKFIX should be flagged"); Assert.AreEqual(2, issues[0].Boundary.Index, "QUICKFIX invalid index"); Assert.AreEqual(8, issues[0].Boundary.Length, "QUICKFIX invalid length "); Assert.AreEqual("DS276209", issues[0].Rule.Id, "QUICKFIX invalid rule"); Assert.AreEqual(0, issues[0].Rule.Fixes.Length, "QUICKFIX invalid Fixes"); Assert.AreEqual("my rules", issues[0].Rule.RuntimeTag, "QUICKFIX invalid tag"); }
public void RuleInfoTest() { Ruleset ruleset = Ruleset.FromDirectory(@"rules\valid", null); RuleProcessor processor = new RuleProcessor(ruleset); string testString = "strcpy(dest,src);"; Issue[] issues = processor.Analyze(testString, "cpp"); Assert.AreEqual(1, issues.Length, "strcpy should be flagged"); Rule r = issues[0].Rule; Assert.IsTrue(r.Description.Contains("strcpy"), "Invalid decription"); Assert.IsTrue(r.Source.Contains("dangerous_api.json"), "Invalid file"); Assert.IsTrue(r.Name.Contains("strcpy"), "Invalid name"); Assert.IsTrue(r.Replecement.Contains("strcpy_s"), "Invalid replacement"); Assert.IsTrue(r.RuleInfo.Contains(r.Id), "Invalid ruleinfo"); }
public void UseCase_IgnoreSuppression_Test() { Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null); rules.AddDirectory(@"rules\custom", null); RuleProcessor processor = new RuleProcessor(rules); processor.EnableSuppressions = false; // MD5CryptoServiceProvider test string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged"); Assert.AreEqual(0, issues[1].Index, "MD5CryptoServiceProvider invalid index"); Assert.AreEqual(3, issues[1].Length, "MD5CryptoServiceProvider invalid length "); Assert.AreEqual("DS126858", issues[1].Rule.Id, "MD5CryptoServiceProvider invalid rule"); }
public void LangugeSelectorTest() { RuleProcessor processor = new RuleProcessor(LoadRules(false)); string testString = "<package id=\"Microsoft.IdentityModel.Tokens\" version=\"5.1.0\""; string lang = Language.FromFileName("helloworld.klingon"); Assert.AreEqual(string.Empty, lang, "Klingon language should not be detected"); lang = Language.FromFileName(Path.Combine("project", "packages.config")); Issue[] issues = processor.Analyze(testString, lang); Assert.AreEqual(1, issues.Length, "There should be positive hit"); bool langExists = Language.GetNames().Contains("csharp"); Assert.IsTrue(langExists, "csharp should be in the collection"); langExists = Language.GetNames().Contains("klyngon"); Assert.IsFalse(langExists, "klingon should not be in the collection"); }
public void UseCase_IgnoreRules_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = true }; // MD5CryptoServiceProvider test string testString = "var a = 10;\nMD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858\nvar b = 20;"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged"); Assert.AreEqual(27, issues[0].Boundary.Index, "MD5CryptoServiceProvider invalid index"); Assert.AreEqual(24, issues[0].Boundary.Length, "MD5CryptoServiceProvider invalid length "); Assert.AreEqual(2, issues[0].StartLocation.Line, "MD5CryptoServiceProvider invalid start location line"); Assert.AreEqual(16, issues[0].StartLocation.Column, "MD5CryptoServiceProvider invalid start location column"); Assert.AreEqual(2, issues[0].EndLocation.Line, "MD5CryptoServiceProvider invalid end location line"); Assert.AreEqual(40, issues[0].EndLocation.Column, "MD5CryptoServiceProvider invalid end location column"); Assert.AreEqual("DS168931", issues[0].Rule.Id, "MD5CryptoServiceProvider invalid rule"); Assert.AreEqual(true, issues[1].IsSuppressionInfo, "MD5CryptoServiceProvider second issue should be info"); }
public void UseCase_IgnoreRules_Test() { RuleProcessor processor = new RuleProcessor(LoadRules(true)) { EnableSuppressions = true }; // MD5CryptoServiceProvider test string testString = "var a = 10;\nMD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858\nvar b = 20;"; Issue[] issues = processor.Analyze(testString, "csharp"); Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged"); Assert.IsTrue(issues.Any(x => x.Boundary.Index == 27), "MD5CryptoServiceProvider invalid index"); Assert.IsTrue(issues.Any(x => x.Boundary.Length == 24), "MD5CryptoServiceProvider invalid length "); Assert.IsTrue(issues.Any(x => x.StartLocation.Line == 2), "MD5CryptoServiceProvider invalid start location line"); Assert.IsTrue(issues.Any(x => x.StartLocation.Column == 16), "MD5CryptoServiceProvider invalid start location column"); Assert.IsTrue(issues.Any(x => x.EndLocation.Line == 2), "MD5CryptoServiceProvider invalid end location line"); Assert.IsTrue(issues.Any(x => x.EndLocation.Column == 40), "MD5CryptoServiceProvider invalid end location column"); Assert.IsTrue(issues.Any(x => x.Rule.Id == "DS168931"), "MD5CryptoServiceProvider invalid rule"); Assert.IsTrue(issues.Count(x => x.IsSuppressionInfo) == 1, "MD5CryptoServiceProvider second issue should be info"); }
public int RunFileEntries(IEnumerable <FileEntry> fileListing, StreamWriter?outputStreamWriter = null) { Verifier?verifier = null; if (_rulespath.Length > 0) { // Setup the rules verifier = new Verifier(_rulespath); if (!verifier.Verify()) { return((int)ExitCode.CriticalError); } if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules) { Debug.WriteLine("Error: No rules were loaded. "); return((int)ExitCode.CriticalError); } } RuleSet rules = new RuleSet(); if (verifier != null) { rules = verifier.CompiledRuleset; } if (!_ignoreDefaultRules) { Assembly?assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream? resource = assembly?.GetManifestResourceStream(filePath); if (resource is Stream) { using (StreamReader file = new StreamReader(resource)) { var rulesString = file.ReadToEnd(); rules.AddString(rulesString, filePath, null); } } } // Initialize the processor RuleProcessor processor = new RuleProcessor(rules); processor.EnableSuppressions = !_disableSuppression; if (_severities.Count() > 0) { processor.SeverityLevel = 0; foreach (string severityText in _severities) { Severity severity; if (ParseSeverity(severityText, out severity)) { processor.SeverityLevel |= severity; } else { Debug.WriteLine("Invalid severity: {0}", severityText); return((int)ExitCode.CriticalError); } } } Writer outputWriter = WriterFactory.GetWriter(string.IsNullOrEmpty(_fileFormat) ? string.IsNullOrEmpty(_outputFile) ? "_dummy" : "text" : _fileFormat, _outputFormat, (outputStreamWriter is null)?(string.IsNullOrEmpty(_outputFile) ? Console.Out : File.CreateText(_outputFile)):outputStreamWriter, (outputStreamWriter is null)?_outputFile:null); int filesAnalyzed = 0; int filesSkipped = 0; int filesAffected = 0; int issuesCount = 0; void parseFileEntry(FileEntry fileEntry) { string language = Language.FromFileName(fileEntry.FullPath); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { Interlocked.Increment(ref filesSkipped); } else { string fileText = string.Empty; try { using (StreamReader reader = new StreamReader(fileEntry.Content)) { fileText = reader.ReadToEnd(); } Interlocked.Increment(ref filesAnalyzed); } catch (Exception) { // Skip files we can't parse Interlocked.Increment(ref filesSkipped); return; } Issue[] issues = processor.Analyze(fileText, language); bool issuesFound = issues.Any(iss => !iss.IsSuppressionInfo) || _disableSuppression && issues.Any(); if (issuesFound) { Interlocked.Increment(ref filesAffected); Debug.WriteLine("file:{0}", fileEntry.FullPath); // Iterate through each issue foreach (Issue issue in issues) { if (!issue.IsSuppressionInfo || _disableSuppression) { Interlocked.Increment(ref issuesCount); Debug.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}", issue.StartLocation.Line, issue.StartLocation.Column, issue.EndLocation.Line, issue.EndLocation.Column, issue.Rule.Id, issue.Rule.Severity, issue.Rule.Name); IssueRecord record = new IssueRecord( Filename: fileEntry.FullPath, Filesize: fileText.Length, TextSample: fileText.Substring(issue.Boundary.Index, issue.Boundary.Length), Issue: issue, Language: language); outputWriter.WriteIssue(record); } } } } } //Iterate through all files if (_disableParallel) { foreach (var fileEntry in fileListing) { parseFileEntry(fileEntry); } } else { Parallel.ForEach(fileListing, parseFileEntry); } outputWriter.FlushAndClose(); Debug.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected); Debug.WriteLine("Files analyzed: {0}", filesAnalyzed); Debug.WriteLine("Files skipped: {0}", filesSkipped); return((int)ExitCode.NoIssues); }
/// <summary> /// Main WORKHORSE for analyzing file; called from file based or decompression functions /// </summary> /// <param name="filename"></param> /// <param name="fileText"></param> void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo) { #region minorRollupTrackingAndProgress WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace); _appProfile.MetaData.FilesAnalyzed++; int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped; int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100); //earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI if (percentCompleted > 100) { percentCompleted = 100; } else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output { WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false); } #endregion //process file against rules Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo); //if any matches found for this file... if (matches.Count() > 0) { _appProfile.MetaData.FilesAffected++; _appProfile.MetaData.TotalMatchesCount += matches.Count(); // Iterate through each match issue foreach (Issue match in matches) { WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace); //maintain a list of unique tags; multi-purpose but primarily for filtering -d option bool dupTagFound = false; foreach (string t in match.Rule.Tags) { dupTagFound = !_uniqueTagsControl.Add(t); } //save all unique dependencies even if Dependency tag pattern is not-unique var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase); String textMatch; if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v))) { textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name); } else { textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length); } //wrap rule issue result to add metadata MatchRecord record = new MatchRecord() { Filename = filePath, Language = languageInfo, Filesize = fileText.Length, TextSample = textMatch, Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line), Issue = match }; //preserve issue level characteristics as rolled up meta data of interest bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record); //bail after extracting any dependency unique items IF user requested if (_arg_outputUniqueTagsOnly && dupTagFound) { continue; } else if (addAsFeatureMatch) { _appProfile.MatchList.Add(record); } else { _appProfile.MetaData.TotalMatchesCount -= 1;//reduce e.g. tag counters only as per preferences file } } } else { WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace); } }
public int Run() { if (!Directory.Exists(_path) && !File.Exists(_path)) { Console.Error.WriteLine("Error: Not a valid file or directory {0}", _path); return((int)ExitCode.CriticalError); } Verifier verifier = null; if (_rulespath.Count() > 0) { // Setup the rules verifier = new Verifier(_rulespath); if (!verifier.Verify()) { return((int)ExitCode.CriticalError); } if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules) { Console.Error.WriteLine("Error: No rules were loaded. "); return((int)ExitCode.CriticalError); } } RuleSet rules = new RuleSet(); if (verifier != null) { rules = verifier.CompiledRuleset; } if (!_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string filePath = "Microsoft.DevSkim.CLI.Resources.devskim-rules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { rules.AddString(file.ReadToEnd(), filePath, null); } } // Initialize the processor RuleProcessor processor = new RuleProcessor(rules); if (_severities.Count() > 0) { processor.SeverityLevel = 0; foreach (string severityText in _severities) { Severity severity; if (ParseSeverity(severityText, out severity)) { processor.SeverityLevel |= severity; } else { Console.Error.WriteLine("Invalid severity: {0}", severityText); return((int)ExitCode.CriticalError); } } } Writer outputWriter = WriterFactory.GetWriter(_fileFormat, (string.IsNullOrEmpty(_outputFile)) ? null : "text", _outputFormat); if (string.IsNullOrEmpty(_outputFile)) { outputWriter.TextWriter = Console.Out; } else { outputWriter.TextWriter = File.CreateText(_outputFile); } int filesAnalyzed = 0; int filesSkipped = 0; int filesAffected = 0; int issuesCount = 0; // Iterate through all files foreach (string filename in Directory.EnumerateFiles(_path, "*.*", SearchOption.AllDirectories)) { string language = Language.FromFileName(filename); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { filesSkipped++; continue; } filesAnalyzed++; string fileText = File.ReadAllText(filename); Issue[] issues = processor.Analyze(fileText, language); if (issues.Count() > 0) { filesAffected++; issuesCount += issues.Count(); Console.Error.WriteLine("file:{0}", filename); // Iterate through each issue foreach (Issue issue in issues) { Console.Error.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}", issue.StartLocation.Line, issue.StartLocation.Column, issue.EndLocation.Line, issue.EndLocation.Column, issue.Rule.Id, issue.Rule.Severity, issue.Rule.Name); IssueRecord record = new IssueRecord() { Filename = filename, Filesize = fileText.Length, TextSample = fileText.Substring(issue.Boundary.Index, issue.Boundary.Length), Issue = issue }; outputWriter.WriteIssue(record); } Console.Error.WriteLine(); } } outputWriter.FlushAndClose(); Console.Error.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected); Console.Error.WriteLine("Files analyzed: {0}", filesAnalyzed); Console.Error.WriteLine("Files skipped: {0}", filesSkipped); return((int)ExitCode.NoIssues); }
/// <summary> /// Main WORKHORSE for analyzing file; called from file based or decompression functions /// </summary> /// <param name="filename"></param> /// <param name="fileText"></param> void ProcessInMemory(string filePath, string fileText) { #region quickvalidation if (fileText.Length > MAX_FILESIZE) { WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error); _appProfile.MetaData.FilesSkipped++; return; } if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v))) { WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error); _appProfile.MetaData.FilesSkipped++; return; } //determine if file is a compressed item to unpackage for processing string language = Language.FromFileName(filePath); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace); language = Path.GetFileName(filePath); _appProfile.MetaData.FilesSkipped++; return; } else { WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace); } #endregion #region minorRollupTrackingAndProgress _appProfile.MetaData.FilesAnalyzed++; _appProfile.MetaData.AddLanguage(language); _appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart()); LastUpdated = File.GetLastWriteTime(filePath); int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped; int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100); WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false); #endregion //process file against rules Issue[] matches = _rulesProcessor.Analyze(fileText, language); //if any matches found for this file... if (matches.Count() > 0) { _appProfile.MetaData.FilesAffected++; _appProfile.MetaData.TotalMatchesCount += matches.Count(); HashSet <string> uniqueTagsControl = new HashSet <string>(); // Iterate through each match issue foreach (Issue match in matches) { WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace); //maintain a list of unique tags; multi-purpose but primarily for filtering -u option bool dupTagFound = false; foreach (string t in match.Rule.Tags) { dupTagFound = !uniqueTagsControl.Add(t); } //save all unique dependendencies even if Dependency tag pattern is not-unique var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase); String textMatch; if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v))) { textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language); } else { textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length); } //wrap rule issue result to add metadata MatchRecord record = new MatchRecord() { Filename = filePath, Language = language, Filesize = fileText.Length, TextSample = textMatch, Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line), Issue = match }; //preserve issue level characteristics as rolled up meta data of interest bool valid = _appProfile.MetaData.AddStandardProperties(record); //bail after extracting any dependency unique items IF user requested if (_arg_outputUniqueTagsOnly && dupTagFound) { continue; } else if (valid) { _appProfile.MatchList.Add(record); } } } else { WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace); } }