private AnalyzeContext CreateGuidMatchingSkimmer( string scanTargetExtension, ref SearchDefinition definition, out SearchSkimmer skimmer, string allowFileExtension = null, string denyFileExtension = null, ValidatorsCache validators = null) { MatchExpression expression = CreateGuidDetectingMatchExpression( denyFileExtension: denyFileExtension, allowFileExtension: allowFileExtension); definition ??= CreateDefaultSearchDefinition(expression); var logger = new TestLogger(); var context = new AnalyzeContext { TargetUri = new Uri($"file:///c:/{definition.Name}.{definition.FileNameAllowRegex}.{scanTargetExtension}"), FileContents = definition.Id, Logger = logger }; var mockFileSystem = new Mock <IFileSystem>(); mockFileSystem.Setup(x => x.FileReadAllText(context.TargetUri.LocalPath)).Returns(definition.Id); skimmer = CreateSkimmer( definition, validators: validators, fileSystem: mockFileSystem.Object); return(context); }
private Result ConstructResult( Uri targetUri, string ruleId, FailureLevel level, Region region, FlexMatch flexMatch, string fingerprint, MatchExpression matchExpression, IList <string> arguments) { var location = new Location() { PhysicalLocation = new PhysicalLocation { ArtifactLocation = new ArtifactLocation { Uri = targetUri, }, Region = region, }, }; Dictionary <string, string> fingerprints = BuildFingerprints(fingerprint); string messageId = matchExpression.SubId ?? "Default"; var result = new Result() { RuleId = ruleId, Level = level, Message = new Message() { Id = messageId, Arguments = arguments, }, Locations = new List <Location>(new[] { location }), Fingerprints = fingerprints, }; if (matchExpression.Fixes?.Count > 0) { // Build arguments that may be required for fix text. var argumentNameToValueMap = new Dictionary <string, string>(); foreach (KeyValuePair <string, int> kv in matchExpression.ArgumentNameToIndexMap) { argumentNameToValueMap["{" + kv.Key + "}"] = arguments[kv.Value]; } foreach (SimpleFix fix in matchExpression.Fixes.Values) { ExpandArguments(fix, argumentNameToValueMap); AddFixToResult(flexMatch, fix, result); } } return(result); }
private SearchDefinition CreateDefaultSearchDefinition(MatchExpression matchExpression) { return(new SearchDefinition() { FileNameAllowRegex = Guid.NewGuid().ToString(), Description = Guid.NewGuid().ToString(), Id = Guid.NewGuid().ToString(), Level = FailureLevel.Error, MatchExpressions = new List <MatchExpression> { matchExpression }, Message = Guid.NewGuid().ToString(), Name = Guid.NewGuid().ToString(), }); }
public void SearchSkimmer_NoDetectionWhenMatchIsEmpty() { var expression = new MatchExpression(); SearchDefinition definition = CreateDefaultSearchDefinition(expression); string scanTargetContents = definition.Id; var logger = new TestLogger(); var context = new AnalyzeContext { TargetUri = new Uri($"file:///c:/{definition.Name}.Fake.asc"), FileContents = $"{ definition.Id}", Logger = logger }; SearchSkimmer skimmer = CreateSkimmer(definition); skimmer.Analyze(context); logger.Results.Should().BeNull(); }
public void SearchSkimmer_DetectsFilePatternOnly() { string fileExtension = Guid.NewGuid().ToString(); MatchExpression expr = CreateFileDetectingMatchExpression(fileExtension: fileExtension); SearchDefinition definition = CreateDefaultSearchDefinition(expr); string scanTargetContents = definition.Id; var logger = new TestLogger(); var context = new AnalyzeContext { TargetUri = new Uri($"file:///c:/{definition.Name}.Fake.{fileExtension}"), FileContents = definition.Id, Logger = logger }; SearchSkimmer skimmer = CreateSkimmer(definition); skimmer.Analyze(context); ValidateResultsAgainstDefinition(logger.Results, definition, skimmer); }
public void SearchSkimmer_ValidatorResultsAreProperlyPrioritized() { string validatorAssemblyPath = $@"c:\{Guid.NewGuid()}.dll"; string scanTargetExtension = Guid.NewGuid().ToString(); var mockFileSystem = new Mock <IFileSystem>(); mockFileSystem.Setup(x => x.FileExists(validatorAssemblyPath)).Returns(true); mockFileSystem.Setup(x => x.AssemblyLoadFrom(validatorAssemblyPath)).Returns(this.GetType().Assembly); var validators = new ValidatorsCache( new string[] { validatorAssemblyPath }, fileSystem: mockFileSystem.Object); MatchExpression expression = CreateGuidDetectingMatchExpression( allowFileExtension: scanTargetExtension); expression.ContentsRegex = "TestRule"; SearchDefinition definition = CreateDefaultSearchDefinition(expression); // This Id will match us up with the TestRuleValidator type. definition.Id = "TestRule"; AnalyzeContext context = CreateGuidMatchingSkimmer( scanTargetExtension: scanTargetExtension, ref definition, out SearchSkimmer skimmer, validators: validators); skimmer.Analyze(context); //((TestLogger)context.Logger).Results.Should().BeNull(); }
public void SearchSkimmer_DetectsBase64EncodedPattern() { MatchExpression expr = CreateGuidDetectingMatchExpression(); SearchDefinition definition = CreateDefaultSearchDefinition(expr); string originalMessage = definition.Message; // We inject the well-known encoding name that reports with // 'plaintext' or 'base64-encoded' depending on how a match // was made. definition.Message = $"{{0:encoding}}:{definition.Message}"; string scanTargetContents = definition.Id; byte[] bytes = Encoding.UTF8.GetBytes(scanTargetContents); string base64Encoded = Convert.ToBase64String(bytes); var logger = new TestLogger(); var context = new AnalyzeContext { TargetUri = new Uri($"file:///c:/{definition.Name}.{definition.FileNameAllowRegex}"), FileContents = base64Encoded, Logger = logger }; SearchSkimmer skimmer = CreateSkimmer(definition); skimmer.Analyze(context); // Analyzing base64-encoded values with MatchLengthToDecode > 0 succeeds logger.Results.Count.Should().Be(1); logger.Results[0].RuleId.Should().Be(definition.Id); logger.Results[0].Level.Should().Be(definition.Level); logger.Results[0].GetMessageText(skimmer).Should().Be($"base64-encoded:{originalMessage}"); // Analyzing base64-encoded values with MatchLengthToDecode == 0 fails definition.MatchExpressions[0].MatchLengthToDecode = 0; logger.Results.Clear(); skimmer = CreateSkimmer(definition); skimmer.Analyze(context); logger.Results.Count.Should().Be(0); // Analyzing plaintext values with MatchLengthToDecode > 0 succeeds context.FileContents = scanTargetContents; logger.Results.Clear(); skimmer = CreateSkimmer(definition); skimmer.Analyze(context); // But we should see a change in encoding information in message. Note // that when emitting plaintext matches, we elide this information // entirely (i.e., we only explicitly report 'base64-encoded' and // report nothing for plaintext). logger.Results.Count.Should().Be(1); logger.Results[0].RuleId.Should().Be(definition.Id); logger.Results[0].Level.Should().Be(definition.Level); logger.Results[0].GetMessageText(skimmer).Should().Be($":{originalMessage}"); }
private void RunMatchExpressionForFileNameRegex(AnalyzeContext context, MatchExpression matchExpression, FailureLevel level) { ReportingDescriptor reportingDescriptor = this; string levelText = level.ToString(); string fingerprint = null; IDictionary <string, string> groups = new Dictionary <string, string>(); string filePath = context.TargetUri.LocalPath; string fingerprintText = null, validatorMessage = null; string validationPrefix = string.Empty, validationSuffix = string.Empty; if (_validators != null && matchExpression.IsValidatorEnabled) { Validation state = _validators.Validate(reportingDescriptor.Name, context.DynamicValidation, ref filePath, ref groups, ref levelText, ref fingerprintText, ref validatorMessage, out bool pluginSupportsDynamicValidation); if (!Enum.TryParse <FailureLevel>(levelText, out level)) { // An illegal failure level '{0}' was returned running check '{1}' against '{2}'. context.Logger.LogToolNotification( Errors.CreateNotification( context.TargetUri, "ERR998.ValidatorReturnedIllegalResultLevel", context.Rule.Id, FailureLevel.Error, exception: null, persistExceptionStack: false, messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalResultLevel, levelText, context.Rule.Id, context.TargetUri.GetFileName())); // If we don't understand the failure level, elevate it to error. level = FailureLevel.Error; } switch (state) { case Validation.NoMatch: { // The validator determined the match is a false positive. // i.e., it is not the kind of artifact we're looking for. // We should suspend processing and move to the next match. return; } case Validation.None: case Validation.ValidatorReturnedIllegalValidationState: { // An illegal state was returned running check '{0}' against '{1}' ({2}). context.Logger.LogToolNotification( Errors.CreateNotification( context.TargetUri, "ERR998.ValidatorReturnedIllegalValidationState", context.Rule.Id, FailureLevel.Error, exception: null, persistExceptionStack: false, messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalValidationState, context.Rule.Id, context.TargetUri.GetFileName(), validatorMessage)); level = FailureLevel.Error; return; } case Validation.Authorized: { level = FailureLevel.Error; // Contributes to building a message fragment such as: // 'SomeFile.txt' is an exposed SomeSecret file [...]. validationPrefix = "an exposed "; break; } case Validation.Expired: { level = FailureLevel.Warning; // Contributes to building a message fragment such as: // 'SomeFile.txt' contains an expired SomeApi token[...]. validationPrefix = "an expired "; break; } case Validation.PasswordProtected: { level = FailureLevel.Warning; // Contributes to building a message fragment such as: // 'SomeFile.txt' contains a password-protected SomeSecret file // which could be exfiltrated and potentially brute-forced offline. validationPrefix = "a password-protected "; validationSuffix = " which could be exfiltrated and potentially brute-forced offline"; break; } case Validation.UnknownHost: case Validation.Unauthorized: case Validation.InvalidForConsultedAuthorities: { throw new InvalidOperationException(); } case Validation.Unknown: { level = FailureLevel.Warning; validationPrefix = "an apparent "; if (!context.DynamicValidation) { if (pluginSupportsDynamicValidation) { // This indicates that dynamic validation was disabled but we // passed this result to a validator that could have performed // this work. validationSuffix = ". No validation occurred as it was not enabled. Pass '--dynamic-validation' on the command-line to validate this match"; } else { // No validation was requested. The plugin indicated // that is can't perform this work in any case. validationSuffix = string.Empty; } } else if (pluginSupportsDynamicValidation) { validationSuffix = ", the validity of which could not be determined by runtime analysis"; } else { // Validation was requested. But the plugin indicated // that it can't perform this work in any case. validationSuffix = string.Empty; } break; } case Validation.ValidatorNotFound: { // TODO: should we have an explicit indicator in // all cases that tells us whether this is an // expected condition or not? validationPrefix = "an apparent "; break; } default: { throw new InvalidOperationException($"Unrecognized validation value '{state}'."); } } } Dictionary <string, string> messageArguments = matchExpression.MessageArguments != null ? new Dictionary <string, string>(matchExpression.MessageArguments) : new Dictionary <string, string>(); messageArguments["validationPrefix"] = validationPrefix; messageArguments["validationSuffix"] = validationSuffix; IList <string> arguments = GetMessageArguments( match: null, matchExpression.ArgumentNameToIndexMap, context.TargetUri.LocalPath, validatorMessage: NormalizeValidatorMessage(validatorMessage), messageArguments); Result result = this.ConstructResult( context.TargetUri, reportingDescriptor.Id, level, region: null, flexMatch: null, fingerprint, matchExpression, arguments); context.Logger.Log(reportingDescriptor, result); }
private void RunMatchExpressionForContentsRegex( FlexMatch binary64DecodedMatch, AnalyzeContext context, MatchExpression matchExpression, FailureLevel level) { string filePath = context.TargetUri.GetFilePath(); string searchText = binary64DecodedMatch != null ? Decode(binary64DecodedMatch.Value) : context.FileContents; foreach (FlexMatch flexMatch in _engine.Matches(searchText, matchExpression.ContentsRegex)) { if (!flexMatch.Success) { continue; } ReportingDescriptor reportingDescriptor = this; Regex regex = CachedDotNetRegex.GetOrCreateRegex(matchExpression.ContentsRegex, RegexDefaults.DefaultOptionsCaseInsensitive); Match match = regex.Match(flexMatch.Value); string refinedMatchedPattern = match.Groups["refine"].Value; IDictionary <string, string> groups = match.Groups.CopyToDictionary(regex.GetGroupNames()); Debug.Assert(!groups.ContainsKey("scanTargetFullPath")); groups["scanTargetFullPath"] = filePath; if (matchExpression.Properties != null) { foreach (KeyValuePair <string, string> kv in matchExpression.Properties) { // We will never allow a group returned by a dynamically executing // regex to overwrite a static value in the match expression. This // allows the match expression to provide a default value that // may be replaced by the analysis. if (!groups.ContainsKey(kv.Key)) { groups[kv.Key] = kv.Value; } } } if (string.IsNullOrEmpty(refinedMatchedPattern)) { refinedMatchedPattern = flexMatch.Value; } string levelText = level.ToString(); Validation state = 0; string fingerprint = null; string validatorMessage = null; string validationPrefix = string.Empty; string validationSuffix = string.Empty; if (_validators != null && matchExpression.IsValidatorEnabled) { state = _validators.Validate(reportingDescriptor.Name, context.DynamicValidation, ref refinedMatchedPattern, ref groups, ref levelText, ref fingerprint, ref validatorMessage, out bool pluginSupportsDynamicValidation); if (!Enum.TryParse <FailureLevel>(levelText, out level)) { // An illegal failure level '{0}' was returned running check '{1}' against '{2}'. context.Logger.LogToolNotification( Errors.CreateNotification( context.TargetUri, "ERR998.ValidatorReturnedIllegalResultLevel", context.Rule.Id, FailureLevel.Error, exception: null, persistExceptionStack: false, messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalResultLevel, levelText, context.Rule.Id, context.TargetUri.GetFileName())); // If we don't understand the failure level, elevate it to error. level = FailureLevel.Error; } SetPropertiesBasedOnValidationState(state, context, ref level, ref validationPrefix, ref validationSuffix, ref validatorMessage, pluginSupportsDynamicValidation); if (state == Validation.None || state == Validation.NoMatch || state == Validation.ValidatorReturnedIllegalValidationState) { continue; } } // If we're matching against decoded contents, the region should // relate to the base64-encoded scan target content. We do use // the decoded content for the fingerprint, however. FlexMatch regionFlexMatch = binary64DecodedMatch ?? flexMatch; Region region = ConstructRegion(context, regionFlexMatch, refinedMatchedPattern); Dictionary <string, string> messageArguments = matchExpression.MessageArguments != null ? new Dictionary <string, string>(matchExpression.MessageArguments) : new Dictionary <string, string>(); messageArguments["encoding"] = binary64DecodedMatch != null ? "base64-encoded" : string.Empty; // We don't bother to report a value for plaintext content messageArguments["validationPrefix"] = validationPrefix; messageArguments["validationSuffix"] = validationSuffix; IList <string> arguments = GetMessageArguments(match, matchExpression.ArgumentNameToIndexMap, filePath, validatorMessage: NormalizeValidatorMessage(validatorMessage), messageArguments); Result result = ConstructResult(context.TargetUri, reportingDescriptor.Id, level, region, flexMatch, fingerprint, matchExpression, arguments); // This skimmer instance mutates its reporting descriptor state, // for example, the sub-id may change for every match // expression. We will therefore generate a snapshot of // current ReportingDescriptor state when logging. context.Logger.Log(reportingDescriptor, result); } }
private void RunMatchExpression(FlexMatch binary64DecodedMatch, AnalyzeContext context, MatchExpression matchExpression) { FailureLevel level = matchExpression.Level; if (!string.IsNullOrEmpty(matchExpression.ContentsRegex)) { RunMatchExpressionForContentsRegex(binary64DecodedMatch, context, matchExpression, level); } else if (!string.IsNullOrEmpty(matchExpression.FileNameAllowRegex)) { RunMatchExpressionForFileNameRegex(context, matchExpression, level); } else { // Both FileNameAllowRegex and ContentRegex are null or empty. } }