/// <summary> /// Reloads rules based on settings /// </summary> private void LoadRules() { Settings set = Settings.GetSettings(); try { Assembly assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); if (set.UseDefaultRules) { using (StreamReader file = new StreamReader(resource)) { ruleset.AddString(file.ReadToEnd(), filePath); } } } catch (Exception e) { Debug.WriteLine("Failed to load Default rules. {0}:{1}\n{2}", e.GetType(), e.Message, e.StackTrace); } try { if (set.UseCustomRules && Directory.Exists(set.CustomRulesPath)) { ruleset.AddDirectory(set.CustomRulesPath, "custom"); } } catch (Exception e) { Debug.WriteLine("Failed to load custom rules. {0}:{1}\n{2}", e.GetType(), e.Message, e.StackTrace); } processor.Rules = ruleset; processor.SeverityLevel = Severity.Critical; if (set.EnableImportantRules) { processor.SeverityLevel |= Severity.Important; } if (set.EnableModerateRules) { processor.SeverityLevel |= Severity.Moderate; } if (set.EnableBestPracticeRules) { processor.SeverityLevel |= Severity.BestPractice; } if (set.EnableManualReviewRules) { processor.SeverityLevel |= Severity.ManualReview; } }
/// <summary> /// Common method of retrieving rules from AppInspector.Commands manifest /// </summary> /// <param name="logger"></param> /// <returns></returns> public static RuleSet GetDefaultRuleSet(Logger logger = null) { RuleSet ruleSet = new RuleSet(logger); Assembly assembly = Assembly.GetExecutingAssembly(); string filePath = "Microsoft.ApplicationInspector.Commands.defaultRulesPkd.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { ruleSet.AddString(file.ReadToEnd(), filePath, null); } return(ruleSet); }
/// <summary> /// Reloads rules based on settings /// </summary> private void LoadRules() { Settings set = Settings.GetSettings(); ruleset = new RuleSet(); Assembly assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); if (set.UseDefaultRules) { using (StreamReader file = new StreamReader(resource)) { ruleset.AddString(file.ReadToEnd(), filePath); } } if (set.UseCustomRules) { ruleset.AddDirectory(set.CustomRulesPath, "custom"); } processor.Rules = ruleset; processor.SeverityLevel = Severity.Critical; if (set.EnableImportantRules) { processor.SeverityLevel |= Severity.Important; } if (set.EnableModerateRules) { processor.SeverityLevel |= Severity.Moderate; } if (set.EnableBestPracticeRules) { processor.SeverityLevel |= Severity.BestPractice; } if (set.EnableManualReviewRules) { processor.SeverityLevel |= Severity.ManualReview; } }
public void VerifyDefaultRules() { var rules = new RuleSet(); Assembly assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream resource = assembly?.GetManifestResourceStream(filePath); if (resource is Stream) { using StreamReader file = new StreamReader(resource); rules.AddString(file.ReadToEnd(), filePath, null); } var analyzer = new Analyzer(); analyzer.SetOperation(new ScopedRegexOperation(analyzer)); analyzer.SetOperation(new WithinOperation(analyzer)); Assert.IsFalse(analyzer.EnumerateRuleIssues(rules.GetAllOatRules()).Any()); }
void ConfigRules() { List <string> rulePaths = new List <string>(); if (!_arg_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string[] resourceName = assembly.GetManifestResourceNames(); string filePath = "Microsoft.ApplicationInspector.Commands.defaultRules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { _rules.AddString(file.ReadToEnd(), filePath, null); } } if (!string.IsNullOrEmpty(_arg_customRulesPath)) { rulePaths.Add(_arg_customRulesPath); } foreach (string rulePath in rulePaths) { if (Directory.Exists(rulePath)) { _rules.AddDirectory(rulePath); } else if (File.Exists(rulePath)) { _rules.AddFile(rulePath); } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } } //error check based on ruleset not path enumeration if (_rules.Count() == 0) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED)); } }
public int RunFileEntries(IEnumerable <FileEntry> fileListing, StreamWriter?outputStreamWriter = null) { Verifier?verifier = null; if (_rulespath.Length > 0) { // Setup the rules verifier = new Verifier(_rulespath); if (!verifier.Verify()) { return((int)ExitCode.CriticalError); } if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules) { Debug.WriteLine("Error: No rules were loaded. "); return((int)ExitCode.CriticalError); } } RuleSet rules = new RuleSet(); if (verifier != null) { rules = verifier.CompiledRuleset; } if (!_ignoreDefaultRules) { Assembly?assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream? resource = assembly?.GetManifestResourceStream(filePath); if (resource is Stream) { using (StreamReader file = new StreamReader(resource)) { var rulesString = file.ReadToEnd(); rules.AddString(rulesString, filePath, null); } } } // Initialize the processor RuleProcessor processor = new RuleProcessor(rules); processor.EnableSuppressions = !_disableSuppression; if (_severities.Count() > 0) { processor.SeverityLevel = 0; foreach (string severityText in _severities) { Severity severity; if (ParseSeverity(severityText, out severity)) { processor.SeverityLevel |= severity; } else { Debug.WriteLine("Invalid severity: {0}", severityText); return((int)ExitCode.CriticalError); } } } Writer outputWriter = WriterFactory.GetWriter(string.IsNullOrEmpty(_fileFormat) ? string.IsNullOrEmpty(_outputFile) ? "_dummy" : "text" : _fileFormat, _outputFormat, (outputStreamWriter is null)?(string.IsNullOrEmpty(_outputFile) ? Console.Out : File.CreateText(_outputFile)):outputStreamWriter, (outputStreamWriter is null)?_outputFile:null); int filesAnalyzed = 0; int filesSkipped = 0; int filesAffected = 0; int issuesCount = 0; void parseFileEntry(FileEntry fileEntry) { string language = Language.FromFileName(fileEntry.FullPath); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { Interlocked.Increment(ref filesSkipped); } else { string fileText = string.Empty; try { using (StreamReader reader = new StreamReader(fileEntry.Content)) { fileText = reader.ReadToEnd(); } Interlocked.Increment(ref filesAnalyzed); } catch (Exception) { // Skip files we can't parse Interlocked.Increment(ref filesSkipped); return; } Issue[] issues = processor.Analyze(fileText, language); bool issuesFound = issues.Any(iss => !iss.IsSuppressionInfo) || _disableSuppression && issues.Any(); if (issuesFound) { Interlocked.Increment(ref filesAffected); Debug.WriteLine("file:{0}", fileEntry.FullPath); // Iterate through each issue foreach (Issue issue in issues) { if (!issue.IsSuppressionInfo || _disableSuppression) { Interlocked.Increment(ref issuesCount); Debug.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}", issue.StartLocation.Line, issue.StartLocation.Column, issue.EndLocation.Line, issue.EndLocation.Column, issue.Rule.Id, issue.Rule.Severity, issue.Rule.Name); IssueRecord record = new IssueRecord( Filename: fileEntry.FullPath, Filesize: fileText.Length, TextSample: fileText.Substring(issue.Boundary.Index, issue.Boundary.Length), Issue: issue, Language: language); outputWriter.WriteIssue(record); } } } } } //Iterate through all files if (_disableParallel) { foreach (var fileEntry in fileListing) { parseFileEntry(fileEntry); } } else { Parallel.ForEach(fileListing, parseFileEntry); } outputWriter.FlushAndClose(); Debug.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected); Debug.WriteLine("Files analyzed: {0}", filesAnalyzed); Debug.WriteLine("Files skipped: {0}", filesSkipped); return((int)ExitCode.NoIssues); }
public int Run() { if (!Directory.Exists(_path) && !File.Exists(_path)) { Console.Error.WriteLine("Error: Not a valid file or directory {0}", _path); return((int)ExitCode.CriticalError); } Verifier verifier = null; if (_rulespath.Count() > 0) { // Setup the rules verifier = new Verifier(_rulespath); if (!verifier.Verify()) { return((int)ExitCode.CriticalError); } if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules) { Console.Error.WriteLine("Error: No rules were loaded. "); return((int)ExitCode.CriticalError); } } RuleSet rules = new RuleSet(); if (verifier != null) { rules = verifier.CompiledRuleset; } if (!_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string filePath = "Microsoft.DevSkim.CLI.Resources.devskim-rules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { rules.AddString(file.ReadToEnd(), filePath, null); } } // Initialize the processor RuleProcessor processor = new RuleProcessor(rules); if (_severities.Count() > 0) { processor.SeverityLevel = 0; foreach (string severityText in _severities) { Severity severity; if (ParseSeverity(severityText, out severity)) { processor.SeverityLevel |= severity; } else { Console.Error.WriteLine("Invalid severity: {0}", severityText); return((int)ExitCode.CriticalError); } } } Writer outputWriter = WriterFactory.GetWriter(_fileFormat, (string.IsNullOrEmpty(_outputFile)) ? null : "text", _outputFormat); if (string.IsNullOrEmpty(_outputFile)) { outputWriter.TextWriter = Console.Out; } else { outputWriter.TextWriter = File.CreateText(_outputFile); } int filesAnalyzed = 0; int filesSkipped = 0; int filesAffected = 0; int issuesCount = 0; // Iterate through all files foreach (string filename in Directory.EnumerateFiles(_path, "*.*", SearchOption.AllDirectories)) { string language = Language.FromFileName(filename); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { filesSkipped++; continue; } filesAnalyzed++; string fileText = File.ReadAllText(filename); Issue[] issues = processor.Analyze(fileText, language); if (issues.Count() > 0) { filesAffected++; issuesCount += issues.Count(); Console.Error.WriteLine("file:{0}", filename); // Iterate through each issue foreach (Issue issue in issues) { Console.Error.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}", issue.StartLocation.Line, issue.StartLocation.Column, issue.EndLocation.Line, issue.EndLocation.Column, issue.Rule.Id, issue.Rule.Severity, issue.Rule.Name); IssueRecord record = new IssueRecord() { Filename = filename, Filesize = fileText.Length, TextSample = fileText.Substring(issue.Boundary.Index, issue.Boundary.Length), Issue = issue }; outputWriter.WriteIssue(record); } Console.Error.WriteLine(); } } outputWriter.FlushAndClose(); Console.Error.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected); Console.Error.WriteLine("Files analyzed: {0}", filesAnalyzed); Console.Error.WriteLine("Files skipped: {0}", filesSkipped); return((int)ExitCode.NoIssues); }
/// <summary> /// Analyzes a directory of files. /// </summary> /// <param name="directory"> directory to analyze. </param> /// <returns> List of tags identified </returns> public async Task <List <IssueRecord> > AnalyzeDirectory(string directory) { Logger.Trace("AnalyzeDirectory({0})", directory); var analysisResults = new List <IssueRecord>(); RuleSet rules = new RuleSet(); if (Options["disable-default-rules"] is bool disableDefaultRules && !disableDefaultRules) { var assembly = Assembly.GetExecutingAssembly(); foreach (var resourceName in assembly.GetManifestResourceNames()) { if (resourceName.EndsWith(".json")) { try { var stream = assembly.GetManifestResourceStream(resourceName); using var resourceStream = new StreamReader(stream ?? new MemoryStream()); rules.AddString(resourceStream.ReadToEnd(), resourceName); } catch (Exception ex) { Logger.Warn(ex, "Error loading {0}: {1}", resourceName, ex.Message); } } } } if (Options["custom-rule-directory"] is string customDirectory) { rules.AddDirectory(customDirectory); } if (!rules.Any()) { Logger.Error("No rules were specified, unable to continue."); return(analysisResults); // empty } var processor = new RuleProcessor(rules) { EnableSuppressions = false, SeverityLevel = (Severity)31 }; string[] fileList; if (Directory.Exists(directory)) { fileList = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); } else if (File.Exists(directory)) { fileList = new string[] { directory }; } else { Logger.Warn("{0} is neither a directory nor a file.", directory); return(analysisResults); // empty } foreach (var filename in fileList) { Logger.Trace($"Processing {filename}"); // TODO: Make this more resilient if (IGNORE_FILES.Contains(Path.GetFileName(filename))) { Logger.Trace($"Ignoring {filename}"); continue; } byte[] fileContents; try { fileContents = File.ReadAllBytes(filename); } catch (Exception ex) { Logger.Trace(ex, "File {0} cannot be read, ignoring.", filename); continue; } var buffer = NormalizeFileContent(filename, fileContents); Logger.Debug("Normalization complete."); double MIN_CRYPTO_OP_DENSITY = 0.10; try { // TODO don't do this if we disassembled native code var cryptoOperationLikelihood = CalculateCryptoOpDensity(buffer); Logger.Debug("Cryptographic operation density for {0} was {1}", filename, cryptoOperationLikelihood); if (cryptoOperationLikelihood >= MIN_CRYPTO_OP_DENSITY) { analysisResults.Add(new IssueRecord( Filename: filename, Filesize: buffer.Length, TextSample: "n/a", Issue: new Issue( Boundary: new Boundary(), StartLocation: new Location(), EndLocation: new Location(), Rule: new Rule("Crypto Symbols") { Id = "_CRYPTO_DENSITY", Name = "Cryptographic symbols", Description = cryptoOperationLikelihood.ToString(), Tags = new string[] { "Cryptography.GenericImplementation.HighDensityOperators" } } ) )); } Logger.Debug($"Analyzing {filename}, length={buffer.Length}"); Issue[]? fileResults = null; var task = Task.Run(() => processor.Analyze(buffer, Language.FromFileName(filename))); if (task.Wait(TimeSpan.FromSeconds(2))) { fileResults = task.Result; } else { Logger.Debug("DevSkim operation timed out."); return(analysisResults); } Logger.Debug("Operation Complete: {0}", fileResults?.Length); foreach (var issue in fileResults ?? Array.Empty <Issue>()) { var fileContentArray = buffer.Split(new[] { Environment.NewLine }, StringSplitOptions.None); var excerpt = new List <string>(); var startLoc = Math.Max(issue.StartLocation.Line - 1, 0); var endLoc = Math.Min(issue.EndLocation.Line + 1, fileContentArray.Length - 1); for (int i = startLoc; i <= endLoc; i++) { excerpt.Add(fileContentArray[i].Trim()); } analysisResults.Add(new IssueRecord( Filename: filename, Filesize: buffer.Length, TextSample: issue.StartLocation.Line + " => " + string.Join(Environment.NewLine, excerpt), Issue: issue) ); } } catch (Exception ex) { Logger.Warn(ex, "Error analyzing {0}: {1}", filename, ex.Message); Logger.Warn(ex.StackTrace); } } return(analysisResults); }
public int Run() { if (_suppressError) { Console.SetError(StreamWriter.Null); } if (!Directory.Exists(_path) && !File.Exists(_path)) { Console.Error.WriteLine("Error: Not a valid file or directory {0}", _path); return((int)ExitCode.CriticalError); } Verifier?verifier = null; if (_rulespath.Count() > 0) { // Setup the rules verifier = new Verifier(_rulespath); if (!verifier.Verify()) { return((int)ExitCode.CriticalError); } if (verifier.CompiledRuleset.Count() == 0 && _ignoreDefaultRules) { Console.Error.WriteLine("Error: No rules were loaded. "); return((int)ExitCode.CriticalError); } } RuleSet rules = new RuleSet(); if (verifier != null) { rules = verifier.CompiledRuleset; } if (!_ignoreDefaultRules) { Assembly?assembly = Assembly.GetAssembly(typeof(Boundary)); string filePath = "Microsoft.DevSkim.Resources.devskim-rules.json"; Stream? resource = assembly?.GetManifestResourceStream(filePath); if (resource is Stream) { using (StreamReader file = new StreamReader(resource)) { rules.AddString(file.ReadToEnd(), filePath, null); } } } // Initialize the processor RuleProcessor processor = new RuleProcessor(rules); processor.EnableSuppressions = !_disableSuppression; if (_severities.Count() > 0) { processor.SeverityLevel = 0; foreach (string severityText in _severities) { Severity severity; if (ParseSeverity(severityText, out severity)) { processor.SeverityLevel |= severity; } else { Console.Error.WriteLine("Invalid severity: {0}", severityText); return((int)ExitCode.CriticalError); } } } Writer outputWriter = WriterFactory.GetWriter(string.IsNullOrEmpty(_fileFormat) ? string.IsNullOrEmpty(_outputFile) ? "_dummy" : "text" : _fileFormat, _outputFormat, (string.IsNullOrEmpty(_outputFile) ? Console.Out : File.CreateText(_outputFile)), _outputFile); int filesAnalyzed = 0; int filesSkipped = 0; int filesAffected = 0; int issuesCount = 0; // We can pass either a file or a directory; if it's a file, make an IEnumerable out of it. IEnumerable <FileEntry> fileListing; var extractor = new Extractor(new ExtractorOptions() { ExtractSelfOnFail = false }); if (!Directory.Exists(_path)) { fileListing = extractor.ExtractFile(_path); } else { fileListing = Directory.EnumerateFiles(_path, "*.*", SearchOption.AllDirectories).SelectMany(x => _crawlArchives ? extractor.ExtractFile(x) : FilenameToFileEntryArray(x)); } // Iterate through all files foreach (FileEntry fileEntry in fileListing) { string language = Language.FromFileName(fileEntry.FullPath); // Skip files written in unknown language if (string.IsNullOrEmpty(language)) { filesSkipped++; continue; } string fileText = string.Empty; try { using (StreamReader reader = new StreamReader(fileEntry.Content)) { fileText = reader.ReadToEnd(); } filesAnalyzed++; } catch (Exception) { // Skip files we can't parse filesSkipped++; continue; } Issue[] issues = processor.Analyze(fileText, language); bool issuesFound = issues.Any(iss => iss.IsSuppressionInfo == false) || _disableSuppression && issues.Count() > 0; if (issuesFound) { filesAffected++; Console.Error.WriteLine("file:{0}", fileEntry.FullPath); // Iterate through each issue foreach (Issue issue in issues) { if (!issue.IsSuppressionInfo || _disableSuppression) { issuesCount++; Console.Error.WriteLine("\tregion:{0},{1},{2},{3} - {4} [{5}] - {6}", issue.StartLocation.Line, issue.StartLocation.Column, issue.EndLocation.Line, issue.EndLocation.Column, issue.Rule.Id, issue.Rule.Severity, issue.Rule.Name); IssueRecord record = new IssueRecord( Filename: fileEntry.FullPath, Filesize: fileText.Length, TextSample: fileText.Substring(issue.Boundary.Index, issue.Boundary.Length), Issue: issue, Language: language); outputWriter.WriteIssue(record); } } Console.Error.WriteLine(); } } outputWriter.FlushAndClose(); Console.Error.WriteLine("Issues found: {0} in {1} files", issuesCount, filesAffected); Console.Error.WriteLine("Files analyzed: {0}", filesAnalyzed); Console.Error.WriteLine("Files skipped: {0}", filesSkipped); return((int)ExitCode.NoIssues); }
/// <summary> /// Add default and/or custom rules paths /// Iterate paths and add to ruleset /// </summary> void ConfigRules() { WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace); RuleSet rulesSet = new RuleSet(_arg_logger); List <string> rulePaths = new List <string>(); if (!_arg_ignoreDefaultRules) { Assembly assembly = Assembly.GetExecutingAssembly(); string[] resourceName = assembly.GetManifestResourceNames(); string filePath = "Microsoft.ApplicationInspector.Commands.defaultRules.json"; Stream resource = assembly.GetManifestResourceStream(filePath); using (StreamReader file = new StreamReader(resource)) { rulesSet.AddString(file.ReadToEnd(), filePath, null); } } if (!string.IsNullOrEmpty(_arg_customRulesPath)) { rulePaths.Add(_arg_customRulesPath); } foreach (string rulePath in rulePaths) { if (Directory.Exists(rulePath)) { rulesSet.AddDirectory(rulePath); } else if (File.Exists(rulePath)) { rulesSet.AddFile(rulePath); } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } } //error check based on ruleset not path enumeration if (rulesSet.Count() == 0) { throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED)); } //instantiate a RuleProcessor with the added rules and exception for dependency _rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, _arg_logger); if (_arg_outputUniqueTagsOnly) { List <TagException> tagExceptions; if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref))) { tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref))); string[] exceptions = new string[tagExceptions.Count]; for (int i = 0; i < tagExceptions.Count; i++) { exceptions[i] = tagExceptions[i].Tag; } _rulesProcessor.UniqueTagExceptions = exceptions; } } _appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly, _arg_autoBrowserOpen); _appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " + WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower(); }