private void VerifyFail(
IBinarySkimmer skimmer,
IEnumerable<string> additionalTestFiles = null,
bool useDefaultPolicy = false)
{
Verify(skimmer, additionalTestFiles, useDefaultPolicy, expectToPass: false);
}
private void VerifyFail(
IBinarySkimmer skimmer,
IEnumerable <string> additionalTestFiles = null,
bool useDefaultPolicy = false)
{
Verify(skimmer, additionalTestFiles, useDefaultPolicy, expectToPass: false);
}
private void VerifyThrows <ExceptionType>(
IBinarySkimmer skimmer,
IEnumerable <string> additionalTestFiles = null,
bool useDefaultPolicy = false) where ExceptionType : Exception
{
var targets = new List <string>();
string ruleName = skimmer.GetType().Name;
string baseFilesDirectory = ruleName;
baseFilesDirectory = Path.Combine(Environment.CurrentDirectory, "FunctionalTestsData", baseFilesDirectory);
string[] testFilesDirectories =
new string[]
{
Path.Combine(baseFilesDirectory, "Pass"),
Path.Combine(baseFilesDirectory, "Fail"),
Path.Combine(baseFilesDirectory, "NotApplicable")
};
foreach (var testDirectory in testFilesDirectories)
{
if (Directory.Exists(testDirectory))
{
foreach (string target in Directory.GetFiles(testDirectory, "*", SearchOption.AllDirectories))
{
targets.Add(target);
}
}
}
var context = new BinaryAnalyzerContext();
var logger = new TestMessageLogger();
context.Logger = logger;
PropertiesDictionary policy = null;
if (useDefaultPolicy)
{
policy = new PropertiesDictionary();
}
context.Policy = policy;
skimmer.Initialize(context);
foreach (string target in targets)
{
context = CreateContext(logger, policy, target);
context.Rule = skimmer;
if (skimmer.CanAnalyze(context, out string reasonForNotAnalyzing) != AnalysisApplicability.ApplicableToSpecifiedTarget)
{
continue;
}
Assert.Throws <ExceptionType>(() => skimmer.Analyze(context));
}
}
private void Verify(
IBinarySkimmer skimmer,
IEnumerable <string> additionalTestFiles,
bool useDefaultPolicy,
bool expectToPass)
{
var targets = new List <string>();
string ruleName = skimmer.GetType().Name;
string testFilesDirectory = ruleName;
testFilesDirectory = Path.Combine(Environment.CurrentDirectory, "FunctionalTestsData", testFilesDirectory);
testFilesDirectory = Path.Combine(testFilesDirectory, expectToPass ? "Pass" : "Fail");
Assert.True(Directory.Exists(testFilesDirectory));
foreach (string target in Directory.GetFiles(testFilesDirectory, "*", SearchOption.AllDirectories))
{
if (AnalyzeCommand.ValidAnalysisFileExtensions.Contains(Path.GetExtension(target)))
{
targets.Add(target);
}
}
if (additionalTestFiles != null)
{
foreach (string additionalTestFile in additionalTestFiles)
{
targets.Add(additionalTestFile);
}
}
var context = new BinaryAnalyzerContext();
var logger = new TestMessageLogger();
context.Logger = logger;
PropertyBag policy = null;
if (useDefaultPolicy)
{
policy = new PropertyBag();
}
context.Policy = policy;
skimmer.Initialize(context);
foreach (string target in targets)
{
PE pe = new PE(target);
if (!pe.IsPEFile)
{
continue;
}
context = CreateContext(logger, policy, target);
context.Rule = skimmer;
string reasonForNotAnalyzing;
if (skimmer.CanAnalyze(context, out reasonForNotAnalyzing) != AnalysisApplicability.ApplicableToSpecifiedTarget)
{
continue;
}
skimmer.Analyze(context);
}
HashSet <string> expected = expectToPass ? logger.PassTargets : logger.FailTargets;
HashSet <string> other = expectToPass ? logger.FailTargets : logger.PassTargets;
HashSet <string> configErrors = logger.ConfigurationErrorTargets;
string expectedText = expectToPass ? "success" : "failure";
string actualText = expectToPass ? "failed" : "succeeded";
var sb = new StringBuilder();
foreach (string target in targets)
{
if (expected.Contains(target))
{
expected.Remove(target);
continue;
}
bool missingEntirely = !other.Contains(target);
if (missingEntirely && !expectToPass && target.Contains("Pdb"))
{
// Missing pdbs provoke configuration errors;
if (configErrors.Contains(target))
{
missingEntirely = false;
configErrors.Remove(target);
continue;
}
}
if (missingEntirely)
{
sb.AppendLine("Expected '" + ruleName + "' " + expectedText + " but saw no result at all for file: " + Path.GetFileName(target));
}
else
{
other.Remove(target);
sb.AppendLine("Expected '" + ruleName + "' " + expectedText + " but check " + actualText + " for: " + Path.GetFileName(target));
}
}
if (sb.Length > 0)
{
_testOutputHelper.WriteLine(sb.ToString());
}
Assert.Equal(0, sb.Length);
Assert.Equal(0, expected.Count);
Assert.Equal(0, other.Count);
}
private void VerifyNotApplicable(
IBinarySkimmer skimmer,
HashSet <string> notApplicableConditions,
AnalysisApplicability expectedApplicability = AnalysisApplicability.NotApplicableToSpecifiedTarget)
{
string ruleName = skimmer.GetType().Name;
string testFilesDirectory = ruleName;
testFilesDirectory = Path.Combine(Environment.CurrentDirectory, "FunctionalTestsData", testFilesDirectory);
testFilesDirectory = Path.Combine(testFilesDirectory, "NotApplicable");
var context = new BinaryAnalyzerContext();
HashSet <string> targets = GetTestFilesMatchingConditions(notApplicableConditions);
if (Directory.Exists(testFilesDirectory))
{
foreach (string target in Directory.GetFiles(testFilesDirectory, "*", SearchOption.AllDirectories))
{
if (AnalyzeCommand.ValidAnalysisFileExtensions.Contains(Path.GetExtension(target)))
{
targets.Add(target);
}
}
}
var logger = new TestMessageLogger();
context.Logger = logger;
var sb = new StringBuilder();
foreach (string target in targets)
{
string extension = Path.GetExtension(target);
if (!AnalyzeCommand.ValidAnalysisFileExtensions.Contains(extension))
{
Assert.True(false, "Test file with unexpected extension encountered: " + target);
}
context = CreateContext(logger, null, target);
if (!context.PE.IsPEFile)
{
continue;
}
context.Rule = skimmer;
string reasonForNotAnalyzing;
AnalysisApplicability applicability;
applicability = skimmer.CanAnalyze(context, out reasonForNotAnalyzing);
if (applicability != expectedApplicability)
{
sb.AppendLine("CanAnalyze did not indicate target was invalid for analysis (return was " +
applicability + "): " +
Path.GetFileName(target));
continue;
}
}
if (sb.Length > 0)
{
_testOutputHelper.WriteLine(sb.ToString());
}
Assert.Equal(0, sb.Length);
}
private void LogUnhandledRuleExceptionAnalyzingTarget(HashSet <string> disabledSkimmers, BinaryAnalyzerContext context, IBinarySkimmer skimmer, Exception ex)
{
string ruleName = context.Rule.Name;
// An unhandled exception was encountered analyzing '{0}' for check '{1}',
// which has been disabled for the remainder of the analysis.The
// exception may have resulted from a problem related to parsing
// image metadata and not specific to the rule, however.
// Exception information:
// {2}
context.Rule = ErrorRules.UnhandledRuleException;
context.Logger.Log(MessageKind.InternalError,
context,
string.Format(DriverResources.UnhandledRuleExceptionAnalyzingTarget,
context.Uri.LocalPath,
ruleName,
ex.ToString()));
if (disabledSkimmers != null)
{
disabledSkimmers.Add(skimmer.Id);
}
RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerAnalyze;
}
private void LogUnhandledRuleExceptionAssessingTargetApplicability(HashSet <string> disabledSkimmers, BinaryAnalyzerContext context, IBinarySkimmer skimmer, Exception ex)
{
string ruleName = context.Rule.Name;
// An unhandled exception was raised attempting to determine whether '{0}'
// is a valid analysis target for check '{1}' (which has been disabled
// for the remainder of the analysis). The exception may have resulted
// from a problem related to parsing image metadata and not specific to
// the rule, however. Exception information:
// {2}
context.Rule = ErrorRules.UnhandledRuleException;
context.Logger.Log(MessageKind.InternalError,
context,
string.Format(DriverResources.UnhandledExceptionCheckingApplicability,
context.Uri.LocalPath,
ruleName,
ex.ToString()));
if (disabledSkimmers != null)
{
disabledSkimmers.Add(skimmer.Id);
}
RuntimeErrors |= RuntimeConditions.ExceptionRaisedInSkimmerCanAnalyze;
}
private void LogUnhandledExceptionInitializingRule(BinaryAnalyzerContext context, IBinarySkimmer skimmer, Exception ex)
{
string ruleName = context.Rule.Name;
// An unhandled exception was encountered initializing check '{0}', which
// has been disabled for the remainder of the analysis. Exception information:
// {1}
context.Rule = ErrorRules.UnhandledRuleException;
context.Logger.Log(MessageKind.InternalError,
context,
string.Format(DriverResources.UnhandledExceptionInitializingRule,
ruleName,
ex.ToString()));
RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerInitialize;
}
private void Verify(
IBinarySkimmer skimmer,
IEnumerable<string> additionalTestFiles,
bool useDefaultPolicy,
bool expectToPass)
{
var targets = new List<string>();
string ruleName = skimmer.GetType().Name;
string testFilesDirectory = ruleName;
testFilesDirectory = Path.Combine(Environment.CurrentDirectory, "FunctionalTestsData", testFilesDirectory);
testFilesDirectory = Path.Combine(testFilesDirectory, expectToPass ? "Pass" : "Fail");
Assert.True(Directory.Exists(testFilesDirectory));
foreach (string target in Directory.GetFiles(testFilesDirectory, "*", SearchOption.AllDirectories))
{
if (AnalyzeCommand.ValidAnalysisFileExtensions.Contains(Path.GetExtension(target)))
{
targets.Add(target);
}
}
if (additionalTestFiles != null)
{
foreach (string additionalTestFile in additionalTestFiles)
{
targets.Add(additionalTestFile);
}
}
var context = new BinaryAnalyzerContext();
var logger = new TestMessageLogger();
context.Logger = logger;
PropertyBag policy = null;
if (useDefaultPolicy)
{
policy = new PropertyBag();
}
context.Policy = policy;
skimmer.Initialize(context);
foreach (string target in targets)
{
PE pe = new PE(target);
if (!pe.IsPEFile) { continue; }
context = CreateContext(logger, policy, target);
context.Rule = skimmer;
string reasonForNotAnalyzing;
if (skimmer.CanAnalyze(context, out reasonForNotAnalyzing) != AnalysisApplicability.ApplicableToSpecifiedTarget)
{
continue;
}
skimmer.Analyze(context);
}
HashSet<string> expected = expectToPass ? logger.PassTargets : logger.FailTargets;
HashSet<string> other = expectToPass ? logger.FailTargets : logger.PassTargets;
HashSet<string> configErrors = logger.ConfigurationErrorTargets;
string expectedText = expectToPass ? "success" : "failure";
string actualText = expectToPass ? "failed" : "succeeded";
var sb = new StringBuilder();
foreach (string target in targets)
{
if (expected.Contains(target))
{
expected.Remove(target);
continue;
}
bool missingEntirely = !other.Contains(target);
if (missingEntirely && !expectToPass && target.Contains("Pdb"))
{
// Missing pdbs provoke configuration errors;
if (configErrors.Contains(target))
{
missingEntirely = false;
configErrors.Remove(target);
continue;
}
}
if (missingEntirely)
{
sb.AppendLine("Expected '" + ruleName + "' " + expectedText + " but saw no result at all for file: " + Path.GetFileName(target));
}
else
{
other.Remove(target);
sb.AppendLine("Expected '" + ruleName + "' " + expectedText + " but check " + actualText + " for: " + Path.GetFileName(target));
}
}
if (sb.Length > 0)
{
_testOutputHelper.WriteLine(sb.ToString());
}
Assert.Equal(0, sb.Length);
Assert.Equal(0, expected.Count);
Assert.Equal(0, other.Count);
}
private void VerifyPass(IBinarySkimmer skimmer, bool useDefaultPolicy = false)
{
Verify(skimmer, null, useDefaultPolicy, expectToPass: true);
}
private void VerifyNotApplicable(
IBinarySkimmer skimmer,
HashSet<string> notApplicableConditions,
AnalysisApplicability expectedApplicability = AnalysisApplicability.NotApplicableToSpecifiedTarget)
{
string ruleName = skimmer.GetType().Name;
string testFilesDirectory = ruleName;
testFilesDirectory = Path.Combine(Environment.CurrentDirectory, "FunctionalTestsData", testFilesDirectory);
testFilesDirectory = Path.Combine(testFilesDirectory, "NotApplicable");
var context = new BinaryAnalyzerContext();
HashSet<string> targets = GetTestFilesMatchingConditions(notApplicableConditions);
if (Directory.Exists(testFilesDirectory))
{
foreach (string target in Directory.GetFiles(testFilesDirectory, "*", SearchOption.AllDirectories))
{
if (AnalyzeCommand.ValidAnalysisFileExtensions.Contains(Path.GetExtension(target)))
{
targets.Add(target);
}
}
}
var logger = new TestMessageLogger();
context.Logger = logger;
var sb = new StringBuilder();
foreach (string target in targets)
{
string extension = Path.GetExtension(target);
if (!AnalyzeCommand.ValidAnalysisFileExtensions.Contains(extension))
{
Assert.True(false, "Test file with unexpected extension encountered: " + target);
}
context = CreateContext(logger, null, target);
if (!context.PE.IsPEFile) { continue; }
context.Rule = skimmer;
string reasonForNotAnalyzing;
AnalysisApplicability applicability;
applicability = skimmer.CanAnalyze(context, out reasonForNotAnalyzing);
if (applicability != expectedApplicability)
{
sb.AppendLine("CanAnalyze did not indicate target was invalid for analysis (return was " +
applicability + "): " +
Path.GetFileName(target));
continue;
}
}
if (sb.Length > 0)
{
_testOutputHelper.WriteLine(sb.ToString());
}
Assert.Equal(0, sb.Length);
}
private void VerifyPass(IBinarySkimmer skimmer, bool useDefaultPolicy = false)
{
Verify(skimmer, null, useDefaultPolicy, expectToPass: true);
}