private AnalyzeContext CreateGuidMatchingSkimmer(
string scanTargetExtension,
ref SearchDefinition definition,
out SearchSkimmer skimmer,
string allowFileExtension = null,
string denyFileExtension = null,
ValidatorsCache validators = null)
{
MatchExpression expression =
CreateGuidDetectingMatchExpression(
denyFileExtension: denyFileExtension,
allowFileExtension: allowFileExtension);
definition ??= CreateDefaultSearchDefinition(expression);
var logger = new TestLogger();
var context = new AnalyzeContext
{
TargetUri = new Uri($"file:///c:/{definition.Name}.{definition.FileNameAllowRegex}.{scanTargetExtension}"),
FileContents = definition.Id,
Logger = logger
};
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.FileReadAllText(context.TargetUri.LocalPath)).Returns(definition.Id);
skimmer = CreateSkimmer(
definition,
validators: validators,
fileSystem: mockFileSystem.Object);
return(context);
}
private Result ConstructResult(
Uri targetUri,
string ruleId,
FailureLevel level,
Region region,
FlexMatch flexMatch,
string fingerprint,
MatchExpression matchExpression,
IList <string> arguments)
{
var location = new Location()
{
PhysicalLocation = new PhysicalLocation
{
ArtifactLocation = new ArtifactLocation
{
Uri = targetUri,
},
Region = region,
},
};
Dictionary <string, string> fingerprints = BuildFingerprints(fingerprint);
string messageId = matchExpression.SubId ?? "Default";
var result = new Result()
{
RuleId = ruleId,
Level = level,
Message = new Message()
{
Id = messageId,
Arguments = arguments,
},
Locations = new List <Location>(new[] { location }),
Fingerprints = fingerprints,
};
if (matchExpression.Fixes?.Count > 0)
{
// Build arguments that may be required for fix text.
var argumentNameToValueMap = new Dictionary <string, string>();
foreach (KeyValuePair <string, int> kv in matchExpression.ArgumentNameToIndexMap)
{
argumentNameToValueMap["{" + kv.Key + "}"] = arguments[kv.Value];
}
foreach (SimpleFix fix in matchExpression.Fixes.Values)
{
ExpandArguments(fix, argumentNameToValueMap);
AddFixToResult(flexMatch, fix, result);
}
}
return(result);
}
private SearchDefinition CreateDefaultSearchDefinition(MatchExpression matchExpression)
{
return(new SearchDefinition()
{
FileNameAllowRegex = Guid.NewGuid().ToString(),
Description = Guid.NewGuid().ToString(),
Id = Guid.NewGuid().ToString(),
Level = FailureLevel.Error,
MatchExpressions = new List <MatchExpression> {
matchExpression
},
Message = Guid.NewGuid().ToString(),
Name = Guid.NewGuid().ToString(),
});
}
public void SearchSkimmer_NoDetectionWhenMatchIsEmpty()
{
var expression = new MatchExpression();
SearchDefinition definition = CreateDefaultSearchDefinition(expression);
string scanTargetContents = definition.Id;
var logger = new TestLogger();
var context = new AnalyzeContext
{
TargetUri = new Uri($"file:///c:/{definition.Name}.Fake.asc"),
FileContents = $"{ definition.Id}",
Logger = logger
};
SearchSkimmer skimmer = CreateSkimmer(definition);
skimmer.Analyze(context);
logger.Results.Should().BeNull();
}
public void SearchSkimmer_DetectsFilePatternOnly()
{
string fileExtension = Guid.NewGuid().ToString();
MatchExpression expr = CreateFileDetectingMatchExpression(fileExtension: fileExtension);
SearchDefinition definition = CreateDefaultSearchDefinition(expr);
string scanTargetContents = definition.Id;
var logger = new TestLogger();
var context = new AnalyzeContext
{
TargetUri = new Uri($"file:///c:/{definition.Name}.Fake.{fileExtension}"),
FileContents = definition.Id,
Logger = logger
};
SearchSkimmer skimmer = CreateSkimmer(definition);
skimmer.Analyze(context);
ValidateResultsAgainstDefinition(logger.Results, definition, skimmer);
}
public void SearchSkimmer_ValidatorResultsAreProperlyPrioritized()
{
string validatorAssemblyPath = $@"c:\{Guid.NewGuid()}.dll";
string scanTargetExtension = Guid.NewGuid().ToString();
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.FileExists(validatorAssemblyPath)).Returns(true);
mockFileSystem.Setup(x => x.AssemblyLoadFrom(validatorAssemblyPath)).Returns(this.GetType().Assembly);
var validators = new ValidatorsCache(
new string[] { validatorAssemblyPath },
fileSystem: mockFileSystem.Object);
MatchExpression expression =
CreateGuidDetectingMatchExpression(
allowFileExtension: scanTargetExtension);
expression.ContentsRegex = "TestRule";
SearchDefinition definition = CreateDefaultSearchDefinition(expression);
// This Id will match us up with the TestRuleValidator type.
definition.Id = "TestRule";
AnalyzeContext context =
CreateGuidMatchingSkimmer(
scanTargetExtension: scanTargetExtension,
ref definition,
out SearchSkimmer skimmer,
validators: validators);
skimmer.Analyze(context);
//((TestLogger)context.Logger).Results.Should().BeNull();
}
public void SearchSkimmer_DetectsBase64EncodedPattern()
{
MatchExpression expr = CreateGuidDetectingMatchExpression();
SearchDefinition definition = CreateDefaultSearchDefinition(expr);
string originalMessage = definition.Message;
// We inject the well-known encoding name that reports with
// 'plaintext' or 'base64-encoded' depending on how a match
// was made.
definition.Message = $"{{0:encoding}}:{definition.Message}";
string scanTargetContents = definition.Id;
byte[] bytes = Encoding.UTF8.GetBytes(scanTargetContents);
string base64Encoded = Convert.ToBase64String(bytes);
var logger = new TestLogger();
var context = new AnalyzeContext
{
TargetUri = new Uri($"file:///c:/{definition.Name}.{definition.FileNameAllowRegex}"),
FileContents = base64Encoded,
Logger = logger
};
SearchSkimmer skimmer = CreateSkimmer(definition);
skimmer.Analyze(context);
// Analyzing base64-encoded values with MatchLengthToDecode > 0 succeeds
logger.Results.Count.Should().Be(1);
logger.Results[0].RuleId.Should().Be(definition.Id);
logger.Results[0].Level.Should().Be(definition.Level);
logger.Results[0].GetMessageText(skimmer).Should().Be($"base64-encoded:{originalMessage}");
// Analyzing base64-encoded values with MatchLengthToDecode == 0 fails
definition.MatchExpressions[0].MatchLengthToDecode = 0;
logger.Results.Clear();
skimmer = CreateSkimmer(definition);
skimmer.Analyze(context);
logger.Results.Count.Should().Be(0);
// Analyzing plaintext values with MatchLengthToDecode > 0 succeeds
context.FileContents = scanTargetContents;
logger.Results.Clear();
skimmer = CreateSkimmer(definition);
skimmer.Analyze(context);
// But we should see a change in encoding information in message. Note
// that when emitting plaintext matches, we elide this information
// entirely (i.e., we only explicitly report 'base64-encoded' and
// report nothing for plaintext).
logger.Results.Count.Should().Be(1);
logger.Results[0].RuleId.Should().Be(definition.Id);
logger.Results[0].Level.Should().Be(definition.Level);
logger.Results[0].GetMessageText(skimmer).Should().Be($":{originalMessage}");
}
private void RunMatchExpressionForFileNameRegex(AnalyzeContext context, MatchExpression matchExpression, FailureLevel level)
{
ReportingDescriptor reportingDescriptor = this;
string levelText = level.ToString();
string fingerprint = null;
IDictionary <string, string> groups = new Dictionary <string, string>();
string filePath = context.TargetUri.LocalPath;
string fingerprintText = null, validatorMessage = null;
string validationPrefix = string.Empty, validationSuffix = string.Empty;
if (_validators != null && matchExpression.IsValidatorEnabled)
{
Validation state = _validators.Validate(reportingDescriptor.Name,
context.DynamicValidation,
ref filePath,
ref groups,
ref levelText,
ref fingerprintText,
ref validatorMessage,
out bool pluginSupportsDynamicValidation);
if (!Enum.TryParse <FailureLevel>(levelText, out level))
{
// An illegal failure level '{0}' was returned running check '{1}' against '{2}'.
context.Logger.LogToolNotification(
Errors.CreateNotification(
context.TargetUri,
"ERR998.ValidatorReturnedIllegalResultLevel",
context.Rule.Id,
FailureLevel.Error,
exception: null,
persistExceptionStack: false,
messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalResultLevel,
levelText,
context.Rule.Id,
context.TargetUri.GetFileName()));
// If we don't understand the failure level, elevate it to error.
level = FailureLevel.Error;
}
switch (state)
{
case Validation.NoMatch:
{
// The validator determined the match is a false positive.
// i.e., it is not the kind of artifact we're looking for.
// We should suspend processing and move to the next match.
return;
}
case Validation.None:
case Validation.ValidatorReturnedIllegalValidationState:
{
// An illegal state was returned running check '{0}' against '{1}' ({2}).
context.Logger.LogToolNotification(
Errors.CreateNotification(
context.TargetUri,
"ERR998.ValidatorReturnedIllegalValidationState",
context.Rule.Id,
FailureLevel.Error,
exception: null,
persistExceptionStack: false,
messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalValidationState,
context.Rule.Id,
context.TargetUri.GetFileName(),
validatorMessage));
level = FailureLevel.Error;
return;
}
case Validation.Authorized:
{
level = FailureLevel.Error;
// Contributes to building a message fragment such as:
// 'SomeFile.txt' is an exposed SomeSecret file [...].
validationPrefix = "an exposed ";
break;
}
case Validation.Expired:
{
level = FailureLevel.Warning;
// Contributes to building a message fragment such as:
// 'SomeFile.txt' contains an expired SomeApi token[...].
validationPrefix = "an expired ";
break;
}
case Validation.PasswordProtected:
{
level = FailureLevel.Warning;
// Contributes to building a message fragment such as:
// 'SomeFile.txt' contains a password-protected SomeSecret file
// which could be exfiltrated and potentially brute-forced offline.
validationPrefix = "a password-protected ";
validationSuffix = " which could be exfiltrated and potentially brute-forced offline";
break;
}
case Validation.UnknownHost:
case Validation.Unauthorized:
case Validation.InvalidForConsultedAuthorities:
{
throw new InvalidOperationException();
}
case Validation.Unknown:
{
level = FailureLevel.Warning;
validationPrefix = "an apparent ";
if (!context.DynamicValidation)
{
if (pluginSupportsDynamicValidation)
{
// This indicates that dynamic validation was disabled but we
// passed this result to a validator that could have performed
// this work.
validationSuffix = ". No validation occurred as it was not enabled. Pass '--dynamic-validation' on the command-line to validate this match";
}
else
{
// No validation was requested. The plugin indicated
// that is can't perform this work in any case.
validationSuffix = string.Empty;
}
}
else if (pluginSupportsDynamicValidation)
{
validationSuffix = ", the validity of which could not be determined by runtime analysis";
}
else
{
// Validation was requested. But the plugin indicated
// that it can't perform this work in any case.
validationSuffix = string.Empty;
}
break;
}
case Validation.ValidatorNotFound:
{
// TODO: should we have an explicit indicator in
// all cases that tells us whether this is an
// expected condition or not?
validationPrefix = "an apparent ";
break;
}
default:
{
throw new InvalidOperationException($"Unrecognized validation value '{state}'.");
}
}
}
Dictionary <string, string> messageArguments =
matchExpression.MessageArguments != null ?
new Dictionary <string, string>(matchExpression.MessageArguments) :
new Dictionary <string, string>();
messageArguments["validationPrefix"] = validationPrefix;
messageArguments["validationSuffix"] = validationSuffix;
IList <string> arguments = GetMessageArguments(
match: null,
matchExpression.ArgumentNameToIndexMap,
context.TargetUri.LocalPath,
validatorMessage: NormalizeValidatorMessage(validatorMessage),
messageArguments);
Result result = this.ConstructResult(
context.TargetUri,
reportingDescriptor.Id,
level,
region: null,
flexMatch: null,
fingerprint,
matchExpression,
arguments);
context.Logger.Log(reportingDescriptor, result);
}
private void RunMatchExpressionForContentsRegex(
FlexMatch binary64DecodedMatch,
AnalyzeContext context,
MatchExpression matchExpression,
FailureLevel level)
{
string filePath = context.TargetUri.GetFilePath();
string searchText = binary64DecodedMatch != null
? Decode(binary64DecodedMatch.Value)
: context.FileContents;
foreach (FlexMatch flexMatch in _engine.Matches(searchText, matchExpression.ContentsRegex))
{
if (!flexMatch.Success)
{
continue;
}
ReportingDescriptor reportingDescriptor = this;
Regex regex =
CachedDotNetRegex.GetOrCreateRegex(matchExpression.ContentsRegex,
RegexDefaults.DefaultOptionsCaseInsensitive);
Match match = regex.Match(flexMatch.Value);
string refinedMatchedPattern = match.Groups["refine"].Value;
IDictionary <string, string> groups = match.Groups.CopyToDictionary(regex.GetGroupNames());
Debug.Assert(!groups.ContainsKey("scanTargetFullPath"));
groups["scanTargetFullPath"] = filePath;
if (matchExpression.Properties != null)
{
foreach (KeyValuePair <string, string> kv in matchExpression.Properties)
{
// We will never allow a group returned by a dynamically executing
// regex to overwrite a static value in the match expression. This
// allows the match expression to provide a default value that
// may be replaced by the analysis.
if (!groups.ContainsKey(kv.Key))
{
groups[kv.Key] = kv.Value;
}
}
}
if (string.IsNullOrEmpty(refinedMatchedPattern))
{
refinedMatchedPattern = flexMatch.Value;
}
string levelText = level.ToString();
Validation state = 0;
string fingerprint = null;
string validatorMessage = null;
string validationPrefix = string.Empty;
string validationSuffix = string.Empty;
if (_validators != null && matchExpression.IsValidatorEnabled)
{
state = _validators.Validate(reportingDescriptor.Name,
context.DynamicValidation,
ref refinedMatchedPattern,
ref groups,
ref levelText,
ref fingerprint,
ref validatorMessage,
out bool pluginSupportsDynamicValidation);
if (!Enum.TryParse <FailureLevel>(levelText, out level))
{
// An illegal failure level '{0}' was returned running check '{1}' against '{2}'.
context.Logger.LogToolNotification(
Errors.CreateNotification(
context.TargetUri,
"ERR998.ValidatorReturnedIllegalResultLevel",
context.Rule.Id,
FailureLevel.Error,
exception: null,
persistExceptionStack: false,
messageFormat: SpamResources.ERR998_ValidatorReturnedIllegalResultLevel,
levelText,
context.Rule.Id,
context.TargetUri.GetFileName()));
// If we don't understand the failure level, elevate it to error.
level = FailureLevel.Error;
}
SetPropertiesBasedOnValidationState(state,
context,
ref level,
ref validationPrefix,
ref validationSuffix,
ref validatorMessage,
pluginSupportsDynamicValidation);
if (state == Validation.None ||
state == Validation.NoMatch ||
state == Validation.ValidatorReturnedIllegalValidationState)
{
continue;
}
}
// If we're matching against decoded contents, the region should
// relate to the base64-encoded scan target content. We do use
// the decoded content for the fingerprint, however.
FlexMatch regionFlexMatch = binary64DecodedMatch ?? flexMatch;
Region region = ConstructRegion(context, regionFlexMatch, refinedMatchedPattern);
Dictionary <string, string> messageArguments = matchExpression.MessageArguments != null ?
new Dictionary <string, string>(matchExpression.MessageArguments) :
new Dictionary <string, string>();
messageArguments["encoding"] = binary64DecodedMatch != null ?
"base64-encoded" :
string.Empty; // We don't bother to report a value for plaintext content
messageArguments["validationPrefix"] = validationPrefix;
messageArguments["validationSuffix"] = validationSuffix;
IList <string> arguments = GetMessageArguments(match,
matchExpression.ArgumentNameToIndexMap,
filePath,
validatorMessage: NormalizeValidatorMessage(validatorMessage),
messageArguments);
Result result = ConstructResult(context.TargetUri,
reportingDescriptor.Id,
level,
region,
flexMatch,
fingerprint,
matchExpression,
arguments);
// This skimmer instance mutates its reporting descriptor state,
// for example, the sub-id may change for every match
// expression. We will therefore generate a snapshot of
// current ReportingDescriptor state when logging.
context.Logger.Log(reportingDescriptor, result);
}
}
private void RunMatchExpression(FlexMatch binary64DecodedMatch, AnalyzeContext context, MatchExpression matchExpression)
{
FailureLevel level = matchExpression.Level;
if (!string.IsNullOrEmpty(matchExpression.ContentsRegex))
{
RunMatchExpressionForContentsRegex(binary64DecodedMatch, context, matchExpression, level);
}
else if (!string.IsNullOrEmpty(matchExpression.FileNameAllowRegex))
{
RunMatchExpressionForFileNameRegex(context, matchExpression, level);
}
else
{
// Both FileNameAllowRegex and ContentRegex are null or empty.
}
}