public static IEnumerable <string> GetValidatorPaths(IEnumerable <string> searchDefinitionsPaths)
{
var validatorAssemblyPaths = new List <string>();
foreach (string searchDefinitionsPath in searchDefinitionsPaths)
{
var serializer = new JsonSerializer();
using var textReader = new StreamReader(searchDefinitionsPath);
using var jsonReader = new JsonTextReader(textReader);
SearchDefinitions definitions = serializer.Deserialize <SearchDefinitions>(jsonReader);
if (!string.IsNullOrEmpty(definitions.ValidatorsAssemblyName))
{
string validatorAssemblyPath = Path.GetDirectoryName(searchDefinitionsPath);
validatorAssemblyPath = Path.Combine(validatorAssemblyPath, definitions.ValidatorsAssemblyName);
validatorAssemblyPaths.Add(validatorAssemblyPath);
}
}
if (validatorAssemblyPaths.Count == 0)
{
throw
new InvalidOperationException(
"No validator assembly paths could be retrieved from configured search definitions files.");
}
return(validatorAssemblyPaths);
}
private static void ValidateSharedStringsExpansion(SearchDefinitions searchDefinitions)
{
foreach (SearchDefinition definition in searchDefinitions.Definitions)
{
ValidateSharedStringsExpansion(definition.FileNameDenyRegex);
ValidateSharedStringsExpansion(definition.FileNameAllowRegex);
foreach (MatchExpression matchExpression in definition.MatchExpressions)
{
ValidateSharedStringsExpansion(matchExpression.ContentsRegex);
ValidateSharedStringsExpansion(matchExpression.FileNameDenyRegex);
ValidateSharedStringsExpansion(matchExpression.FileNameAllowRegex);
}
}
}
private SearchSkimmer CreateSkimmer(
SearchDefinition definition,
IRegex engine = null,
ValidatorsCache validators = null,
FileRegionsCache fileRegionsCache = null,
IFileSystem fileSystem = null)
{
var definitions = new SearchDefinitions
{
Definitions = new List <SearchDefinition>(new[] { definition }),
};
definitions = AnalyzeCommand.PushInheritedData(definitions, sharedStrings: null);
return(new SearchSkimmer(
engine: engine ?? RE2Regex.Instance,
validators: validators,
fileRegionsCache: fileRegionsCache ?? new FileRegionsCache(),
definition: definitions.Definitions[0],
fileSystem: fileSystem));
}
public static ISet <Skimmer <AnalyzeContext> > CreateSkimmersFromDefinitionsFiles(
IFileSystem fileSystem,
IEnumerable <string> searchDefinitionsPaths,
IRegex engine = null)
{
engine ??= RE2Regex.Instance;
var validators = new ValidatorsCache();
FileRegionsCache fileRegionsCache = FileRegionsCache.Instance;
var skimmers = new HashSet <Skimmer <AnalyzeContext> >();
// TODO exception handling for bad search definitions files
foreach (string searchDefinitionsPath in searchDefinitionsPaths)
{
string searchDefinitionsText =
fileSystem.FileReadAllText(searchDefinitionsPath);
SearchDefinitions definitions =
JsonConvert.DeserializeObject <SearchDefinitions>(searchDefinitionsText);
// This would skip files that does not look like rules.
if (definitions == null || definitions.Definitions == null)
{
continue;
}
string validatorPath = null;
string definitionsDirectory = Path.GetDirectoryName(searchDefinitionsPath);
if (!string.IsNullOrEmpty(definitions.ValidatorsAssemblyName))
{
// TODO File.Exists check? Logging if not locatable?
validatorPath = Path.Combine(definitionsDirectory, definitions.ValidatorsAssemblyName);
validators.ValidatorPaths.Add(validatorPath);
}
else
{
// If no explicit name of a validator binary was provided,
// we look for one that lives alongside the definitions file.
validatorPath = Path.GetFileNameWithoutExtension(searchDefinitionsPath) + ".dll";
validatorPath = Path.Combine(definitionsDirectory, validatorPath);
if (File.Exists(validatorPath))
{
validators.ValidatorPaths.Add(validatorPath);
}
}
Dictionary <string, string> sharedStrings = null;
if (!string.IsNullOrEmpty(definitions.SharedStringsFileName))
{
string sharedStringsFullPath = Path.Combine(definitionsDirectory, definitions.SharedStringsFileName);
sharedStrings = LoadSharedStrings(sharedStringsFullPath, fileSystem);
}
definitions = PushInheritedData(definitions, sharedStrings);
foreach (SearchDefinition definition in definitions.Definitions)
{
Skimmer <AnalyzeContext> skimmer = skimmers.FirstOrDefault(skimmer => skimmer.Id == definition.Id);
if (skimmer != null)
{
skimmers.Remove(skimmer);
}
skimmers.Add(
new SearchSkimmer(
engine: engine,
validators: validators,
fileRegionsCache: fileRegionsCache,
definition));
const string singleSpace = " ";
// Send no-op match operations through engine in order to drive caching of all regexes.
if (definition.FileNameAllowRegex != null)
{
engine.Match(singleSpace, definition.FileNameAllowRegex, RegexDefaults.DefaultOptionsCaseSensitive);
}
foreach (MatchExpression matchExpression in definition.MatchExpressions)
{
if (!string.IsNullOrEmpty(matchExpression.FileNameAllowRegex))
{
engine.Match(singleSpace, matchExpression.FileNameAllowRegex, RegexDefaults.DefaultOptionsCaseSensitive);
}
if (!string.IsNullOrEmpty(matchExpression.ContentsRegex))
{
engine.Match(singleSpace, matchExpression.ContentsRegex, RegexDefaults.DefaultOptionsCaseSensitive);
}
}
}
}
return(skimmers);
}
internal static SearchDefinitions PushInheritedData(SearchDefinitions definitions, Dictionary <string, string> sharedStrings)
{
var idToExpressionsMap = new Dictionary <string, List <MatchExpression> >();
foreach (SearchDefinition definition in definitions.Definitions)
{
definition.FileNameDenyRegex = PushData(definition.FileNameDenyRegex,
definition.SharedStrings,
sharedStrings);
definition.FileNameAllowRegex = PushData(definition.FileNameAllowRegex,
definition.SharedStrings,
sharedStrings);
foreach (MatchExpression matchExpression in definition.MatchExpressions)
{
matchExpression.FileNameDenyRegex = PushData(matchExpression.FileNameDenyRegex,
definition.SharedStrings,
sharedStrings);
matchExpression.FileNameDenyRegex ??= definition.FileNameDenyRegex;
matchExpression.FileNameAllowRegex = PushData(matchExpression.FileNameAllowRegex,
definition.SharedStrings,
sharedStrings);
matchExpression.FileNameAllowRegex ??= definition.FileNameAllowRegex;
matchExpression.ContentsRegex = PushData(matchExpression.ContentsRegex,
definition.SharedStrings,
sharedStrings);
matchExpression.Id ??= definition.Id;
matchExpression.Name ??= definition.Name;
matchExpression.Message ??= definition.Message;
matchExpression.Description ??= definition.Description;
if (matchExpression.Level == 0)
{
matchExpression.Level = definition.Level;
}
if (!idToExpressionsMap.TryGetValue(matchExpression.Id, out List <MatchExpression> cachedMatchExpressions))
{
cachedMatchExpressions = idToExpressionsMap[matchExpression.Id] = new List <MatchExpression>();
}
cachedMatchExpressions.Add(matchExpression);
}
}
var searchDefinitions = new SearchDefinitions
{
Definitions = new List <SearchDefinition>(),
};
foreach (KeyValuePair <string, List <MatchExpression> > kv in idToExpressionsMap)
{
string ruleId = kv.Key;
List <MatchExpression> matchExpressions = kv.Value;
var definition = new SearchDefinition
{
Id = matchExpressions[0].Id,
Name = matchExpressions[0].Name,
MatchExpressions = matchExpressions,
Description = matchExpressions[0].Description,
};
searchDefinitions.Definitions.Add(definition);
}
#if DEBUG
ValidateSharedStringsExpansion(searchDefinitions);
#endif
return(searchDefinitions);
}
private static void AnalyzeCommand(IRegex engine)
{
var definitions = new SearchDefinitions()
{
Definitions = new List <SearchDefinition>(new[]
{
new SearchDefinition()
{
Name = "MinimalRule", Id = "Test1002",
Level = FailureLevel.Error, FileNameAllowRegex = "(?i)\\.test$",
Message = "A problem occurred in '{0:scanTarget}'.",
MatchExpressions = new List <MatchExpression>(new[]
{
new MatchExpression()
{
ContentsRegex = "foo",
Fixes = new Dictionary <string, SimpleFix>()
{
{
"convertToPublic", new SimpleFix()
{
Description = "Make class public.",
Find = "foo",
ReplaceWith = "bar"
}
}
}
}
})
}
})
};
string definitionsText = JsonConvert.SerializeObject(definitions);
string searchDefinitionsPath = Guid.NewGuid().ToString();
var disabledSkimmers = new HashSet <string>();
var testLogger = new TestLogger();
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.FileReadAllText(searchDefinitionsPath)).Returns(definitionsText);
// Acquire skimmers for searchers
ISet <Skimmer <AnalyzeContext> > skimmers =
PatternMatcher.AnalyzeCommand.CreateSkimmersFromDefinitionsFiles(
mockFileSystem.Object,
new string[] { searchDefinitionsPath },
engine);
string scanTargetFileName = Path.Combine(@"C:\", Guid.NewGuid().ToString() + ".test");
FlexString fileContents = "bar foo foo";
FlexString fixedFileContents = "bar bar bar";
var context = new AnalyzeContext()
{
TargetUri = new Uri(scanTargetFileName, UriKind.RelativeOrAbsolute),
FileContents = fileContents,
Logger = testLogger
};
IEnumerable <Skimmer <AnalyzeContext> > applicableSkimmers = PatternMatcher.AnalyzeCommand.DetermineApplicabilityForTargetHelper(context, skimmers, disabledSkimmers);
PatternMatcher.AnalyzeCommand.AnalyzeTargetHelper(context, applicableSkimmers, disabledSkimmers);
testLogger.Results.Should().NotBeNull();
testLogger.Results.Count.Should().Be(2);
foreach (Result result in testLogger.Results)
{
result.Level.Should().Be(FailureLevel.Error);
}
}
