public static IEnumerable <string> GetValidatorPaths(IEnumerable <string> searchDefinitionsPaths) { var validatorAssemblyPaths = new List <string>(); foreach (string searchDefinitionsPath in searchDefinitionsPaths) { var serializer = new JsonSerializer(); using var textReader = new StreamReader(searchDefinitionsPath); using var jsonReader = new JsonTextReader(textReader); SearchDefinitions definitions = serializer.Deserialize <SearchDefinitions>(jsonReader); if (!string.IsNullOrEmpty(definitions.ValidatorsAssemblyName)) { string validatorAssemblyPath = Path.GetDirectoryName(searchDefinitionsPath); validatorAssemblyPath = Path.Combine(validatorAssemblyPath, definitions.ValidatorsAssemblyName); validatorAssemblyPaths.Add(validatorAssemblyPath); } } if (validatorAssemblyPaths.Count == 0) { throw new InvalidOperationException( "No validator assembly paths could be retrieved from configured search definitions files."); } return(validatorAssemblyPaths); }
private static void ValidateSharedStringsExpansion(SearchDefinitions searchDefinitions) { foreach (SearchDefinition definition in searchDefinitions.Definitions) { ValidateSharedStringsExpansion(definition.FileNameDenyRegex); ValidateSharedStringsExpansion(definition.FileNameAllowRegex); foreach (MatchExpression matchExpression in definition.MatchExpressions) { ValidateSharedStringsExpansion(matchExpression.ContentsRegex); ValidateSharedStringsExpansion(matchExpression.FileNameDenyRegex); ValidateSharedStringsExpansion(matchExpression.FileNameAllowRegex); } } }
private SearchSkimmer CreateSkimmer( SearchDefinition definition, IRegex engine = null, ValidatorsCache validators = null, FileRegionsCache fileRegionsCache = null, IFileSystem fileSystem = null) { var definitions = new SearchDefinitions { Definitions = new List <SearchDefinition>(new[] { definition }), }; definitions = AnalyzeCommand.PushInheritedData(definitions, sharedStrings: null); return(new SearchSkimmer( engine: engine ?? RE2Regex.Instance, validators: validators, fileRegionsCache: fileRegionsCache ?? new FileRegionsCache(), definition: definitions.Definitions[0], fileSystem: fileSystem)); }
public static ISet <Skimmer <AnalyzeContext> > CreateSkimmersFromDefinitionsFiles( IFileSystem fileSystem, IEnumerable <string> searchDefinitionsPaths, IRegex engine = null) { engine ??= RE2Regex.Instance; var validators = new ValidatorsCache(); FileRegionsCache fileRegionsCache = FileRegionsCache.Instance; var skimmers = new HashSet <Skimmer <AnalyzeContext> >(); // TODO exception handling for bad search definitions files foreach (string searchDefinitionsPath in searchDefinitionsPaths) { string searchDefinitionsText = fileSystem.FileReadAllText(searchDefinitionsPath); SearchDefinitions definitions = JsonConvert.DeserializeObject <SearchDefinitions>(searchDefinitionsText); // This would skip files that does not look like rules. if (definitions == null || definitions.Definitions == null) { continue; } string validatorPath = null; string definitionsDirectory = Path.GetDirectoryName(searchDefinitionsPath); if (!string.IsNullOrEmpty(definitions.ValidatorsAssemblyName)) { // TODO File.Exists check? Logging if not locatable? validatorPath = Path.Combine(definitionsDirectory, definitions.ValidatorsAssemblyName); validators.ValidatorPaths.Add(validatorPath); } else { // If no explicit name of a validator binary was provided, // we look for one that lives alongside the definitions file. validatorPath = Path.GetFileNameWithoutExtension(searchDefinitionsPath) + ".dll"; validatorPath = Path.Combine(definitionsDirectory, validatorPath); if (File.Exists(validatorPath)) { validators.ValidatorPaths.Add(validatorPath); } } Dictionary <string, string> sharedStrings = null; if (!string.IsNullOrEmpty(definitions.SharedStringsFileName)) { string sharedStringsFullPath = Path.Combine(definitionsDirectory, definitions.SharedStringsFileName); sharedStrings = LoadSharedStrings(sharedStringsFullPath, fileSystem); } definitions = PushInheritedData(definitions, sharedStrings); foreach (SearchDefinition definition in definitions.Definitions) { Skimmer <AnalyzeContext> skimmer = skimmers.FirstOrDefault(skimmer => skimmer.Id == definition.Id); if (skimmer != null) { skimmers.Remove(skimmer); } skimmers.Add( new SearchSkimmer( engine: engine, validators: validators, fileRegionsCache: fileRegionsCache, definition)); const string singleSpace = " "; // Send no-op match operations through engine in order to drive caching of all regexes. if (definition.FileNameAllowRegex != null) { engine.Match(singleSpace, definition.FileNameAllowRegex, RegexDefaults.DefaultOptionsCaseSensitive); } foreach (MatchExpression matchExpression in definition.MatchExpressions) { if (!string.IsNullOrEmpty(matchExpression.FileNameAllowRegex)) { engine.Match(singleSpace, matchExpression.FileNameAllowRegex, RegexDefaults.DefaultOptionsCaseSensitive); } if (!string.IsNullOrEmpty(matchExpression.ContentsRegex)) { engine.Match(singleSpace, matchExpression.ContentsRegex, RegexDefaults.DefaultOptionsCaseSensitive); } } } } return(skimmers); }
internal static SearchDefinitions PushInheritedData(SearchDefinitions definitions, Dictionary <string, string> sharedStrings) { var idToExpressionsMap = new Dictionary <string, List <MatchExpression> >(); foreach (SearchDefinition definition in definitions.Definitions) { definition.FileNameDenyRegex = PushData(definition.FileNameDenyRegex, definition.SharedStrings, sharedStrings); definition.FileNameAllowRegex = PushData(definition.FileNameAllowRegex, definition.SharedStrings, sharedStrings); foreach (MatchExpression matchExpression in definition.MatchExpressions) { matchExpression.FileNameDenyRegex = PushData(matchExpression.FileNameDenyRegex, definition.SharedStrings, sharedStrings); matchExpression.FileNameDenyRegex ??= definition.FileNameDenyRegex; matchExpression.FileNameAllowRegex = PushData(matchExpression.FileNameAllowRegex, definition.SharedStrings, sharedStrings); matchExpression.FileNameAllowRegex ??= definition.FileNameAllowRegex; matchExpression.ContentsRegex = PushData(matchExpression.ContentsRegex, definition.SharedStrings, sharedStrings); matchExpression.Id ??= definition.Id; matchExpression.Name ??= definition.Name; matchExpression.Message ??= definition.Message; matchExpression.Description ??= definition.Description; if (matchExpression.Level == 0) { matchExpression.Level = definition.Level; } if (!idToExpressionsMap.TryGetValue(matchExpression.Id, out List <MatchExpression> cachedMatchExpressions)) { cachedMatchExpressions = idToExpressionsMap[matchExpression.Id] = new List <MatchExpression>(); } cachedMatchExpressions.Add(matchExpression); } } var searchDefinitions = new SearchDefinitions { Definitions = new List <SearchDefinition>(), }; foreach (KeyValuePair <string, List <MatchExpression> > kv in idToExpressionsMap) { string ruleId = kv.Key; List <MatchExpression> matchExpressions = kv.Value; var definition = new SearchDefinition { Id = matchExpressions[0].Id, Name = matchExpressions[0].Name, MatchExpressions = matchExpressions, Description = matchExpressions[0].Description, }; searchDefinitions.Definitions.Add(definition); } #if DEBUG ValidateSharedStringsExpansion(searchDefinitions); #endif return(searchDefinitions); }
private static void AnalyzeCommand(IRegex engine) { var definitions = new SearchDefinitions() { Definitions = new List <SearchDefinition>(new[] { new SearchDefinition() { Name = "MinimalRule", Id = "Test1002", Level = FailureLevel.Error, FileNameAllowRegex = "(?i)\\.test$", Message = "A problem occurred in '{0:scanTarget}'.", MatchExpressions = new List <MatchExpression>(new[] { new MatchExpression() { ContentsRegex = "foo", Fixes = new Dictionary <string, SimpleFix>() { { "convertToPublic", new SimpleFix() { Description = "Make class public.", Find = "foo", ReplaceWith = "bar" } } } } }) } }) }; string definitionsText = JsonConvert.SerializeObject(definitions); string searchDefinitionsPath = Guid.NewGuid().ToString(); var disabledSkimmers = new HashSet <string>(); var testLogger = new TestLogger(); var mockFileSystem = new Mock <IFileSystem>(); mockFileSystem.Setup(x => x.FileReadAllText(searchDefinitionsPath)).Returns(definitionsText); // Acquire skimmers for searchers ISet <Skimmer <AnalyzeContext> > skimmers = PatternMatcher.AnalyzeCommand.CreateSkimmersFromDefinitionsFiles( mockFileSystem.Object, new string[] { searchDefinitionsPath }, engine); string scanTargetFileName = Path.Combine(@"C:\", Guid.NewGuid().ToString() + ".test"); FlexString fileContents = "bar foo foo"; FlexString fixedFileContents = "bar bar bar"; var context = new AnalyzeContext() { TargetUri = new Uri(scanTargetFileName, UriKind.RelativeOrAbsolute), FileContents = fileContents, Logger = testLogger }; IEnumerable <Skimmer <AnalyzeContext> > applicableSkimmers = PatternMatcher.AnalyzeCommand.DetermineApplicabilityForTargetHelper(context, skimmers, disabledSkimmers); PatternMatcher.AnalyzeCommand.AnalyzeTargetHelper(context, applicableSkimmers, disabledSkimmers); testLogger.Results.Should().NotBeNull(); testLogger.Results.Count.Should().Be(2); foreach (Result result in testLogger.Results) { result.Level.Should().Be(FailureLevel.Error); } }