public void ScopeTest()
{
RuleProcessor processor = new RuleProcessor(LoadRules(false))
{
EnableSuppressions = true
};
// Ignore inline comment
string testString = "var hash = MD5.Create(); // MD5 is not allowed";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "MD5 in inline comment should be ignored");
Assert.AreEqual(11, issues[0].Boundary.Index, "MD5 inline index is wrong");
// ignore multinline comment
testString = " /*\r\nMD5 is not allowed\r\n */ \r\nvar hash = MD5.Create();";
issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "MD5 in multi line comment should be ignored");
Assert.AreEqual(42, issues[0].Boundary.Index, "MD5 multi line index is wrong");
// TODO test
testString = "//TODO: fix it later";
processor.SeverityLevel |= Severity.ManualReview;
issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "TODO should be flagged");
}
public void UseCase_SeverityFilter_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
string testString = "eval(something)";
Issue[] issues = processor.Analyze(testString, "javascript");
Assert.AreEqual(0, issues.Length, "Manual Review should not be flagged");
processor.SeverityLevel |= Severity.ManualReview;
issues = processor.Analyze(testString, "javascript");
Assert.AreEqual(1, issues.Length, "Manual Review should be flagged");
}
public void UseCase_Normal_Test()
{
Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null);
rules.AddDirectory(@"rules\custom", "my rules");
RuleProcessor processor = new RuleProcessor(rules);
string lang = Language.FromFileName("testfilename.cpp");
string testString = "strcpy(dest,src);";
// strcpy test
Issue[] issues = processor.Analyze(testString, lang);
Assert.AreEqual(1, issues.Length, "strcpy should be flagged");
Assert.AreEqual(0, issues[0].Index, "strcpy invalid index");
Assert.AreEqual(16, issues[0].Length, "strcpy invalid length ");
Assert.AreEqual("DS185832", issues[0].Rule.Id, "strcpy invalid rule");
// Fix it test
Assert.AreNotEqual(issues[0].Rule.Fixes.Length, 0, "strcpy invalid Fixes");
CodeFix fix = issues[0].Rule.Fixes[0];
string fixedCode = RuleProcessor.Fix(testString, fix);
Assert.AreEqual("strcpy_s(dest, <size of dest>, src);", fixedCode, "strcpy invalid code fix");
Assert.IsTrue(fix.Name.Contains("Change to strcpy_s"), "strcpy wrong fix name");
// QUICKFIX test
processor.SeverityLevel |= Severity.ManualReview;
testString = "//QUICKFIX: fix this later";
issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "QUICKFIX should be flagged");
Assert.AreEqual(2, issues[0].Index, "QUICKFIX invalid index");
Assert.AreEqual(8, issues[0].Length, "QUICKFIX invalid length ");
Assert.AreEqual("DS276209", issues[0].Rule.Id, "QUICKFIX invalid rule");
Assert.AreEqual(0, issues[0].Rule.Fixes.Length, "QUICKFIX invalid Fixes");
Assert.AreEqual("my rules", issues[0].Rule.RuntimeTag, "QUICKFIX invalid tag");
// Same issue twice test
testString = "MD5 hash = MD5.Create();";
issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "Same issue should be twice on line");
Assert.AreEqual(issues[0].Rule, issues[1].Rule, "Same issues should have sames rule IDs");
// Overlaping issues
testString = " MD5 hash = new MD5CryptoServiceProvider();";
issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "Overlaping issue count doesn't add up");
// Override test
testString = "strncat(dest, \"this is also bad\", strlen(dest))";
issues = processor.Analyze(testString, new string[] { "c", "cpp" });
Assert.AreEqual(2, issues.Length, "Override test failed");
}
public void IsMatch_FalseTest()
{
RuleSet ruleset = RuleSet.FromDirectory(Path.Combine("rules", "valid"), null);
RuleProcessor processor = new RuleProcessor(ruleset);
string testString = "this is a test string";
// Normal functionality test
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(0, issues.Length, "Match.Success should be false");
// Non existent langugage
issues = processor.Analyze(testString, "");
Assert.AreEqual(0, issues.Length, "Match.Success should be false, when no language is passed");
}
public void UseCase_Normal_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
string lang = Language.FromFileName("testfilename.cpp");
string testString = "strcpy(dest,src);";
// strcpy test
Issue[] issues = processor.Analyze(testString, lang);
Assert.AreEqual(1, issues.Length, "strcpy should be flagged");
Assert.AreEqual(0, issues[0].Boundary.Index, "strcpy invalid index");
Assert.AreEqual(16, issues[0].Boundary.Length, "strcpy invalid length ");
Assert.AreEqual(1, issues[0].StartLocation.Line, "strcpy invalid start location line ");
Assert.AreEqual(1, issues[0].StartLocation.Column, "strcpy invalid start location column ");
Assert.AreEqual(1, issues[0].EndLocation.Line, "strcpy invalid end location line ");
Assert.AreEqual(16, issues[0].EndLocation.Column, "strcpy invalid end location column ");
Assert.AreEqual("DS185832", issues[0].Rule.Id, "strcpy invalid rule");
// Fix it test
Assert.AreNotEqual(issues[0].Rule.Fixes.Length, 0, "strcpy invalid Fixes");
CodeFix fix = issues[0].Rule.Fixes[0];
string fixedCode = RuleProcessor.Fix(testString, fix);
Assert.AreEqual("strcpy_s(dest, <size of dest>, src);", fixedCode, "strcpy invalid code fix");
Assert.IsTrue(fix.Name.Contains("Change to strcpy_s"), "strcpy wrong fix name");
}
public void UseCase_SeverityFilter_Test()
{
Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null);
rules.AddDirectory(@"rules\custom", null);
RuleProcessor processor = new RuleProcessor(rules);
string testString = "eval(something)";
Issue[] issues = processor.Analyze(testString, "javascript");
Assert.AreEqual(0, issues.Length, "Manual Review should not be flagged");
processor.SeverityLevel |= Severity.ManualReview;
issues = processor.Analyze(testString, "javascript");
Assert.AreEqual(1, issues.Length, "Manual Review should be flagged");
}
public void IsMatch_InvalidLanguageTest()
{
RuleProcessor processor = new RuleProcessor();
string testString = "this is a test string";
// Langugage is empty
Issue[] issues = processor.Analyze(testString, string.Empty);
}
public void IsMatch_InvalidInputTest()
{
RuleProcessor processor = new RuleProcessor();
// Langugage is null
Issue[] issues = processor.Analyze(null, "");
Assert.AreEqual(0, issues.Length, "Match.Success should be false");
}
public void UseCase_OverlapingIssues_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
// Overlaping issues
string testString = " MD5 hash = new MD5CryptoServiceProvider();";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "Overlaping issue count doesn't add up");
}
public void UseCase_OverrideTest_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
// Override test
string testString = "strncat(dest, \"this is also bad\", strlen(dest))";
Issue[] issues = processor.Analyze(testString, new string[] { "c", "cpp" });
Assert.AreEqual(2, issues.Length, "Override test failed");
}
public void UseCase_EmptyString_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
// Empty string test
string testString = "";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(0, issues.Length, "Empty test failed");
}
public void UseCase_IgnoreRules_Test()
{
Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null);
rules.AddDirectory(@"rules\custom", null);
RuleProcessor processor = new RuleProcessor(rules)
{
EnableSuppressions = true
};
// MD5CryptoServiceProvider test
string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "MD5CryptoServiceProvider should be flagged");
Assert.AreEqual(15, issues[0].Index, "MD5CryptoServiceProvider invalid index");
Assert.AreEqual(24, issues[0].Length, "MD5CryptoServiceProvider invalid length ");
Assert.AreEqual("DS168931", issues[0].Rule.Id, "MD5CryptoServiceProvider invalid rule");
// Ignore until test
DateTime expirationDate = DateTime.Now.AddDays(5);
testString = "requests.get('somelink', verify = False) #DevSkim: ignore DS130821 until {0:yyyy}-{0:MM}-{0:dd}";
issues = processor.Analyze(string.Format(testString, expirationDate), "python");
Assert.AreEqual(0, issues.Length, "Ignore until should not be flagged");
// Expired until test
expirationDate = DateTime.Now;
issues = processor.Analyze(string.Format(testString, expirationDate), "python");
Assert.AreEqual(1, issues.Length, "Expired until should be flagged");
// Ignore all until test
expirationDate = DateTime.Now.AddDays(5);
testString = "MD5 hash = new MD5.Create(); #DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}";
issues = processor.Analyze(string.Format(testString, expirationDate), "csharp");
Assert.AreEqual(0, issues.Length, "Ignore all until should not be flagged");
// Expired all test
expirationDate = DateTime.Now;
testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}";
issues = processor.Analyze(string.Format(testString, expirationDate), "csharp");
Assert.AreEqual(2, issues.Length, "Expired all should be flagged");
}
public void UseCase_IssueTwice_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
// Same issue twice test
string testString = "MD5 hash = MD5.Create();";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "Same issue should be twice on line");
Assert.AreEqual(issues[0].Rule, issues[1].Rule, "Same issues should have sames rule IDs");
}
public void Conditions1Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(false))
{
EnableSuppressions = true
};
// http test
string testString = "<h:table xmlns:h=\"http://www.w3.org/TR/html4/\">";
Issue[] issues = processor.Analyze(testString, "xml");
Assert.AreEqual(0, issues.Length, "http should NOT be flagged");
// http test
testString = "<h:table src=\"http://www.w3.org/TR/html4/\">";
issues = processor.Analyze(testString, "xml");
Assert.AreEqual(1, issues.Length, "http should be flagged");
Assert.AreEqual(1, issues[0].StartLocation.Line, "http location line doesn't match");
Assert.AreEqual(14, issues[0].Boundary.Index, "http index doesn't match");
Assert.AreEqual(5, issues[0].Boundary.Length, "http length doesn't match");
Assert.AreEqual("DS137138", issues[0].Rule.Id, "http rule doesn't match");
}
public void Conditions2Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(false))
{
EnableSuppressions = true
};
// $POST test
string testString = "require_once($_POST['t']);";
Issue[] issues = processor.Analyze(testString, "php");
Assert.AreEqual(1, issues.Length, "$_POST should be flagged");
Assert.AreEqual(1, issues[0].StartLocation.Line, "$_POST location line doesn't match");
Assert.AreEqual(0, issues[0].Boundary.Index, "$_POST index doesn't match");
Assert.AreEqual(19, issues[0].Boundary.Length, "$_POST length doesn't match");
Assert.AreEqual("DS181731", issues[0].Rule.Id, "$_POST rule doesn't match");
// $POST test
testString = "echo(urlencode($_POST['data']);";
issues = processor.Analyze(testString, "php");
Assert.AreEqual(0, issues.Length, "$_POST should not be flagged");
}
public void UseCase_IgnoreAllRules_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = true
};
// Ignore all until test
DateTime expirationDate = DateTime.Now.AddDays(5);
string testString = "encryption=false; MD5 hash = MD5.Create(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}";
Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "csharp");
Assert.AreEqual(3, issues.Length, "Ignore all should flag two infos");
}
public void UseCase_IgnoreAllRulesExpired_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = true
};
// Expired all test
DateTime expirationDate = DateTime.Now;
string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore all until {0:yyyy}-{0:MM}-{0:dd}";
Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "csharp");
Assert.AreEqual(2, issues.Length, "Expired all should be flagged");
}
public void UseCase_IgnoreRulesExpired_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = true
};
// Expired until test
DateTime expirationDate = DateTime.Now;
string testString = "requests.get('somelink', verify = False) #DevSkim: ignore DS126186 until {0:yyyy}-{0:MM}-{0:dd}";
Issue[] issues = processor.Analyze(string.Format(testString, expirationDate), "python");
Assert.AreEqual(1, issues.Length, "Expired until should be flagged");
Assert.AreEqual(false, issues[0].IsSuppressionInfo, "Expired until issue should NOT be info");
}
public void UseCase_IgnoreSuppression_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = false
};
// MD5CryptoServiceProvider test
string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged");
Assert.AreEqual(0, issues[1].Boundary.Index, "MD5CryptoServiceProvider invalid index");
Assert.AreEqual(3, issues[1].Boundary.Length, "MD5CryptoServiceProvider invalid length ");
Assert.AreEqual("DS126858", issues[1].Rule.Id, "MD5CryptoServiceProvider invalid rule");
}
public void RuleInfoTest()
{
RuleSet ruleset = RuleSet.FromDirectory(Path.Combine("rules", "valid"), null);
RuleProcessor processor = new RuleProcessor(ruleset);
string testString = "strcpy(dest,src);";
Issue[] issues = processor.Analyze(testString, "cpp");
Assert.AreEqual(1, issues.Length, "strcpy should be flagged");
Rule r = issues[0].Rule;
Assert.IsTrue(r.Description.Contains("strcpy"), "Invalid decription");
Assert.IsTrue(r.Name.Contains("strcpy"), "Invalid name");
Assert.IsTrue(r.Recommendation.Contains("strcpy_s"), "Invalid replacement");
Assert.IsTrue(r.RuleInfo.Contains(r.Id), "Invalid ruleinfo");
}
public void UseCase_ManualReview_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true));
// QUICKFIX test
processor.SeverityLevel |= Severity.ManualReview;
string testString = "//QUICKFIX: fix this later";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(1, issues.Length, "QUICKFIX should be flagged");
Assert.AreEqual(2, issues[0].Boundary.Index, "QUICKFIX invalid index");
Assert.AreEqual(8, issues[0].Boundary.Length, "QUICKFIX invalid length ");
Assert.AreEqual("DS276209", issues[0].Rule.Id, "QUICKFIX invalid rule");
Assert.AreEqual(0, issues[0].Rule.Fixes.Length, "QUICKFIX invalid Fixes");
Assert.AreEqual("my rules", issues[0].Rule.RuntimeTag, "QUICKFIX invalid tag");
}
public void RuleInfoTest()
{
Ruleset ruleset = Ruleset.FromDirectory(@"rules\valid", null);
RuleProcessor processor = new RuleProcessor(ruleset);
string testString = "strcpy(dest,src);";
Issue[] issues = processor.Analyze(testString, "cpp");
Assert.AreEqual(1, issues.Length, "strcpy should be flagged");
Rule r = issues[0].Rule;
Assert.IsTrue(r.Description.Contains("strcpy"), "Invalid decription");
Assert.IsTrue(r.Source.Contains("dangerous_api.json"), "Invalid file");
Assert.IsTrue(r.Name.Contains("strcpy"), "Invalid name");
Assert.IsTrue(r.Replecement.Contains("strcpy_s"), "Invalid replacement");
Assert.IsTrue(r.RuleInfo.Contains(r.Id), "Invalid ruleinfo");
}
public void UseCase_IgnoreSuppression_Test()
{
Ruleset rules = Ruleset.FromDirectory(@"rules\valid", null);
rules.AddDirectory(@"rules\custom", null);
RuleProcessor processor = new RuleProcessor(rules);
processor.EnableSuppressions = false;
// MD5CryptoServiceProvider test
string testString = "MD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged");
Assert.AreEqual(0, issues[1].Index, "MD5CryptoServiceProvider invalid index");
Assert.AreEqual(3, issues[1].Length, "MD5CryptoServiceProvider invalid length ");
Assert.AreEqual("DS126858", issues[1].Rule.Id, "MD5CryptoServiceProvider invalid rule");
}
public void LangugeSelectorTest()
{
RuleProcessor processor = new RuleProcessor(LoadRules(false));
string testString = "<package id=\"Microsoft.IdentityModel.Tokens\" version=\"5.1.0\"";
string lang = Language.FromFileName("helloworld.klingon");
Assert.AreEqual(string.Empty, lang, "Klingon language should not be detected");
lang = Language.FromFileName(Path.Combine("project", "packages.config"));
Issue[] issues = processor.Analyze(testString, lang);
Assert.AreEqual(1, issues.Length, "There should be positive hit");
bool langExists = Language.GetNames().Contains("csharp");
Assert.IsTrue(langExists, "csharp should be in the collection");
langExists = Language.GetNames().Contains("klyngon");
Assert.IsFalse(langExists, "klingon should not be in the collection");
}
public void UseCase_IgnoreRules_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = true
};
// MD5CryptoServiceProvider test
string testString = "var a = 10;\nMD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858\nvar b = 20;";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged");
Assert.AreEqual(27, issues[0].Boundary.Index, "MD5CryptoServiceProvider invalid index");
Assert.AreEqual(24, issues[0].Boundary.Length, "MD5CryptoServiceProvider invalid length ");
Assert.AreEqual(2, issues[0].StartLocation.Line, "MD5CryptoServiceProvider invalid start location line");
Assert.AreEqual(16, issues[0].StartLocation.Column, "MD5CryptoServiceProvider invalid start location column");
Assert.AreEqual(2, issues[0].EndLocation.Line, "MD5CryptoServiceProvider invalid end location line");
Assert.AreEqual(40, issues[0].EndLocation.Column, "MD5CryptoServiceProvider invalid end location column");
Assert.AreEqual("DS168931", issues[0].Rule.Id, "MD5CryptoServiceProvider invalid rule");
Assert.AreEqual(true, issues[1].IsSuppressionInfo, "MD5CryptoServiceProvider second issue should be info");
}
public void UseCase_IgnoreRules_Test()
{
RuleProcessor processor = new RuleProcessor(LoadRules(true))
{
EnableSuppressions = true
};
// MD5CryptoServiceProvider test
string testString = "var a = 10;\nMD5 hash = new MD5CryptoServiceProvider(); //DevSkim: ignore DS126858\nvar b = 20;";
Issue[] issues = processor.Analyze(testString, "csharp");
Assert.AreEqual(2, issues.Length, "MD5CryptoServiceProvider should be flagged");
Assert.IsTrue(issues.Any(x => x.Boundary.Index == 27), "MD5CryptoServiceProvider invalid index");
Assert.IsTrue(issues.Any(x => x.Boundary.Length == 24), "MD5CryptoServiceProvider invalid length ");
Assert.IsTrue(issues.Any(x => x.StartLocation.Line == 2), "MD5CryptoServiceProvider invalid start location line");
Assert.IsTrue(issues.Any(x => x.StartLocation.Column == 16), "MD5CryptoServiceProvider invalid start location column");
Assert.IsTrue(issues.Any(x => x.EndLocation.Line == 2), "MD5CryptoServiceProvider invalid end location line");
Assert.IsTrue(issues.Any(x => x.EndLocation.Column == 40), "MD5CryptoServiceProvider invalid end location column");
Assert.IsTrue(issues.Any(x => x.Rule.Id == "DS168931"), "MD5CryptoServiceProvider invalid rule");
Assert.IsTrue(issues.Count(x => x.IsSuppressionInfo) == 1, "MD5CryptoServiceProvider second issue should be info");
}
public int RunFileEntries(IEnumerable <FileEntry> fileListing, StreamWriter?outputStreamWriter = null)
{
Verifier?verifier = null;
if (_rulespath.Length > 0)
{
// Setup the rules
verifier = new Verifier(_rulespath);
if (!verifier.Verify())
{
return((int)ExitCode.CriticalError);
}
if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules)
{
Debug.WriteLine("Error: No rules were loaded. ");
return((int)ExitCode.CriticalError);
}
}
RuleSet rules = new RuleSet();
if (verifier != null)
{
rules = verifier.CompiledRuleset;
}
if (!_ignoreDefaultRules)
{
Assembly?assembly = Assembly.GetAssembly(typeof(Boundary));
string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json";
Stream? resource = assembly?.GetManifestResourceStream(filePath);
if (resource is Stream)
{
using (StreamReader file = new StreamReader(resource))
{
var rulesString = file.ReadToEnd();
rules.AddString(rulesString, filePath, null);
}
}
}
// Initialize the processor
RuleProcessor processor = new RuleProcessor(rules);
processor.EnableSuppressions = !_disableSuppression;
if (_severities.Count() > 0)
{
processor.SeverityLevel = 0;
foreach (string severityText in _severities)
{
Severity severity;
if (ParseSeverity(severityText, out severity))
{
processor.SeverityLevel |= severity;
}
else
{
Debug.WriteLine("Invalid severity: {0}", severityText);
return((int)ExitCode.CriticalError);
}
}
}
Writer outputWriter = WriterFactory.GetWriter(string.IsNullOrEmpty(_fileFormat) ? string.IsNullOrEmpty(_outputFile) ? "_dummy" : "text" : _fileFormat,
_outputFormat,
(outputStreamWriter is null)?(string.IsNullOrEmpty(_outputFile) ? Console.Out : File.CreateText(_outputFile)):outputStreamWriter,
(outputStreamWriter is null)?_outputFile:null);
int filesAnalyzed = 0;
int filesSkipped = 0;
int filesAffected = 0;
int issuesCount = 0;
void parseFileEntry(FileEntry fileEntry)
{
string language = Language.FromFileName(fileEntry.FullPath);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
Interlocked.Increment(ref filesSkipped);
}
else
{
string fileText = string.Empty;
try
{
using (StreamReader reader = new StreamReader(fileEntry.Content))
{
fileText = reader.ReadToEnd();
}
Interlocked.Increment(ref filesAnalyzed);
}
catch (Exception)
{
// Skip files we can't parse
Interlocked.Increment(ref filesSkipped);
return;
}
Issue[] issues = processor.Analyze(fileText, language);
bool issuesFound = issues.Any(iss => !iss.IsSuppressionInfo) || _disableSuppression && issues.Any();
if (issuesFound)
{
Interlocked.Increment(ref filesAffected);
Debug.WriteLine("file:{0}", fileEntry.FullPath);
// Iterate through each issue
foreach (Issue issue in issues)
{
if (!issue.IsSuppressionInfo || _disableSuppression)
{
Interlocked.Increment(ref issuesCount);
Debug.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}",
issue.StartLocation.Line,
issue.StartLocation.Column,
issue.EndLocation.Line,
issue.EndLocation.Column,
issue.Rule.Id,
issue.Rule.Severity,
issue.Rule.Name);
IssueRecord record = new IssueRecord(
Filename: fileEntry.FullPath,
Filesize: fileText.Length,
TextSample: fileText.Substring(issue.Boundary.Index, issue.Boundary.Length),
Issue: issue,
Language: language);
outputWriter.WriteIssue(record);
}
}
}
}
}
//Iterate through all files
if (_disableParallel)
{
foreach (var fileEntry in fileListing)
{
parseFileEntry(fileEntry);
}
}
else
{
Parallel.ForEach(fileListing, parseFileEntry);
}
outputWriter.FlushAndClose();
Debug.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected);
Debug.WriteLine("Files analyzed: {0}", filesAnalyzed);
Debug.WriteLine("Files skipped: {0}", filesSkipped);
return((int)ExitCode.NoIssues);
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
{
#region minorRollupTrackingAndProgress
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
_appProfile.MetaData.FilesAnalyzed++;
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
//earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
if (percentCompleted > 100)
{
percentCompleted = 100;
}
else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output
{
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
}
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -d option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !_uniqueTagsControl.Add(t);
}
//save all unique dependencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = languageInfo,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (addAsFeatureMatch)
{
_appProfile.MatchList.Add(record);
}
else
{
_appProfile.MetaData.TotalMatchesCount -= 1;//reduce e.g. tag counters only as per preferences file
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
public int Run()
{
if (!Directory.Exists(_path) && !File.Exists(_path))
{
Console.Error.WriteLine("Error: Not a valid file or directory {0}", _path);
return((int)ExitCode.CriticalError);
}
Verifier verifier = null;
if (_rulespath.Count() > 0)
{
// Setup the rules
verifier = new Verifier(_rulespath);
if (!verifier.Verify())
{
return((int)ExitCode.CriticalError);
}
if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules)
{
Console.Error.WriteLine("Error: No rules were loaded. ");
return((int)ExitCode.CriticalError);
}
}
RuleSet rules = new RuleSet();
if (verifier != null)
{
rules = verifier.CompiledRuleset;
}
if (!_ignoreDefaultRules)
{
Assembly assembly = Assembly.GetExecutingAssembly();
string filePath = "Microsoft.DevSkim.CLI.Resources.devskim-rules.json";
Stream resource = assembly.GetManifestResourceStream(filePath);
using (StreamReader file = new StreamReader(resource))
{
rules.AddString(file.ReadToEnd(), filePath, null);
}
}
// Initialize the processor
RuleProcessor processor = new RuleProcessor(rules);
if (_severities.Count() > 0)
{
processor.SeverityLevel = 0;
foreach (string severityText in _severities)
{
Severity severity;
if (ParseSeverity(severityText, out severity))
{
processor.SeverityLevel |= severity;
}
else
{
Console.Error.WriteLine("Invalid severity: {0}", severityText);
return((int)ExitCode.CriticalError);
}
}
}
Writer outputWriter = WriterFactory.GetWriter(_fileFormat,
(string.IsNullOrEmpty(_outputFile)) ? null : "text",
_outputFormat);
if (string.IsNullOrEmpty(_outputFile))
{
outputWriter.TextWriter = Console.Out;
}
else
{
outputWriter.TextWriter = File.CreateText(_outputFile);
}
int filesAnalyzed = 0;
int filesSkipped = 0;
int filesAffected = 0;
int issuesCount = 0;
// Iterate through all files
foreach (string filename in Directory.EnumerateFiles(_path, "*.*", SearchOption.AllDirectories))
{
string language = Language.FromFileName(filename);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
filesSkipped++;
continue;
}
filesAnalyzed++;
string fileText = File.ReadAllText(filename);
Issue[] issues = processor.Analyze(fileText, language);
if (issues.Count() > 0)
{
filesAffected++;
issuesCount += issues.Count();
Console.Error.WriteLine("file:{0}", filename);
// Iterate through each issue
foreach (Issue issue in issues)
{
Console.Error.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}",
issue.StartLocation.Line,
issue.StartLocation.Column,
issue.EndLocation.Line,
issue.EndLocation.Column,
issue.Rule.Id,
issue.Rule.Severity,
issue.Rule.Name);
IssueRecord record = new IssueRecord()
{
Filename = filename,
Filesize = fileText.Length,
TextSample = fileText.Substring(issue.Boundary.Index, issue.Boundary.Length),
Issue = issue
};
outputWriter.WriteIssue(record);
}
Console.Error.WriteLine();
}
}
outputWriter.FlushAndClose();
Console.Error.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected);
Console.Error.WriteLine("Files analyzed: {0}", filesAnalyzed);
Console.Error.WriteLine("Files skipped: {0}", filesSkipped);
return((int)ExitCode.NoIssues);
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
//determine if file is a compressed item to unpackage for processing
string language = Language.FromFileName(filePath);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
language = Path.GetFileName(filePath);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(language);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, language);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
HashSet <string> uniqueTagsControl = new HashSet <string>();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -u option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = language,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool valid = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (valid)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
