public void ValidateCommand_AcceptsTargetFileWithSpaceInName()
{
// Here's the space:
const string LogFileDirectoryWithSpace = @"c:\Users\John Smith\logs";
string logFilePath = Path.Combine(LogFileDirectoryWithSpace, LogFileName);
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.DirectoryExists(LogFileDirectoryWithSpace)).Returns(true);
mockFileSystem.Setup(x => x.GetDirectoriesInDirectory(It.IsAny <string>())).Returns(new string[0]);
mockFileSystem.Setup(x => x.GetFilesInDirectory(LogFileDirectoryWithSpace, LogFileName)).Returns(new string[] { logFilePath });
mockFileSystem.Setup(x => x.ReadAllText(logFilePath)).Returns(TransformCommandTests.MinimalCurrentV2Text);
mockFileSystem.Setup(x => x.ReadAllText(SchemaFilePath)).Returns(SchemaFileContents);
var validateCommand = new ValidateCommand(mockFileSystem.Object);
var options = new ValidateOptions
{
SchemaFilePath = SchemaFilePath,
TargetFileSpecifiers = new string[] { logFilePath }
};
int returnCode = validateCommand.Run(options);
returnCode.Should().Be(0);
}
public void ValidateCommand_WhenOutputFileIsPresentAndForceOptionIsAbsent_DoesNotOverwriteOutputFile()
{
string logFilePath = Path.Combine(LogFileDirectory, LogFileName);
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.FileExists(OutputFilePath)).Returns(true);
mockFileSystem.Setup(x => x.DirectoryExists(LogFileDirectory)).Returns(true);
mockFileSystem.Setup(x => x.GetDirectoriesInDirectory(It.IsAny <string>())).Returns(new string[0]);
mockFileSystem.Setup(x => x.GetFilesInDirectory(LogFileDirectory, LogFileName)).Returns(new string[] { logFilePath });
mockFileSystem.Setup(x => x.ReadAllText(logFilePath)).Returns(TransformCommandTests.MinimalCurrentV2Text);
mockFileSystem.Setup(x => x.ReadAllText(SchemaFilePath)).Returns(SchemaFileContents);
var validateCommand = new ValidateCommand(mockFileSystem.Object);
var options = new ValidateOptions
{
SchemaFilePath = SchemaFilePath,
TargetFileSpecifiers = new string[] { logFilePath },
OutputFilePath = OutputFilePath
};
int returnCode = validateCommand.Run(options);
returnCode.Should().Be(1);
validateCommand.ExecutionException.Should().BeOfType <ExitApplicationException <ExitReason> >();
}
private void Verify(string testFileName)
{
string testDirectory = Path.Combine(Environment.CurrentDirectory, TestDataDirectory);
string testFilePath = Path.Combine(TestDataDirectory, testFileName);
string expectedFilePath = MakeExpectedFilePath(testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(testDirectory, testFileName);
var validateOptions = new ValidateOptions
{
TargetFileSpecifiers = new[] { testFilePath },
OutputFilePath = actualFilePath,
SchemaFilePath = JsonSchemaFile,
Quiet = true
};
new ValidateCommand().Run(validateOptions);
string actualLogContents = File.ReadAllText(actualFilePath);
string expectedLogContents = File.ReadAllText(expectedFilePath);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
}
public void ValidateCommand_AcceptsTargetFileWithSpaceInName()
{
// Here's the space:
const string LogFileDirectoryWithSpace = @"c:\Users\John Smith\logs";
string logFilePath = Path.Combine(LogFileDirectoryWithSpace, LogFileName);
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.DirectoryExists(LogFileDirectoryWithSpace)).Returns(true);
mockFileSystem.Setup(x => x.DirectoryEnumerateFiles(It.IsAny <string>())).Returns(new string[0]);
mockFileSystem.Setup(x => x.DirectoryGetFiles(LogFileDirectoryWithSpace, LogFileName)).Returns(new string[] { logFilePath });
mockFileSystem.Setup(x => x.FileReadAllText(logFilePath)).Returns(RewriteCommandTests.MinimalCurrentV2Text);
mockFileSystem.Setup(x => x.FileReadAllText(SchemaFilePath)).Returns(SchemaFileContents);
var validateCommand = new ValidateCommand(mockFileSystem.Object);
var options = new ValidateOptions
{
SchemaFilePath = SchemaFilePath,
TargetFileSpecifiers = new string[] { logFilePath },
Kind = new List <ResultKind> {
ResultKind.Fail
},
Level = new List <FailureLevel> {
FailureLevel.Warning, FailureLevel.Error
}
};
int returnCode = validateCommand.Run(options);
returnCode.Should().Be(0);
}
private static SarifLog ExecuteTest(string path, string outputPath, string configuration = null)
{
var options = new ValidateOptions
{
TargetFileSpecifiers = new string[] { path },
OutputFilePath = outputPath,
Force = true,
ConfigurationFilePath = configuration
};
// Verify command returned success
int returnCode = new ValidateCommand().Run(options);
returnCode.Should().Be(0);
return(SarifLog.Load(outputPath));
}
public void OptionsInterpretter_CorrectlyAddsAdditiveEnvVars()
{
var mockEnvironmentVariableGetter = new Mock <IEnvironmentVariableGetter>();
mockEnvironmentVariableGetter.Setup(x => x.GetEnvironmentVariable("SARIF_DATATOINSERT_ADDITION")).Returns("TextFiles;BinaryFiles;");
// Deliberately more delimeters than needed
mockEnvironmentVariableGetter.Setup(x => x.GetEnvironmentVariable("SARIF_DATATOREMOVE_ADDITION")).Returns("ComprehensiveRegionProperties;RegionSnippets");
mockEnvironmentVariableGetter.Setup(x => x.GetEnvironmentVariable("SARIF_KIND_ADDITION")).Returns("Informational;");
// Deliberately more delimeters than needed
mockEnvironmentVariableGetter.Setup(x => x.GetEnvironmentVariable("SARIF_LEVEL_ADDITION")).Returns("Note");
OptionsInterpretter optionsInterpretter = new OptionsInterpretter(mockEnvironmentVariableGetter.Object);
ValidateOptions analyzeOptionsBase = new ValidateOptions
{
DataToInsert = new List <OptionallyEmittedData> {
OptionallyEmittedData.Hashes, OptionallyEmittedData.EnvironmentVariables
},
DataToRemove = new List <OptionallyEmittedData> {
OptionallyEmittedData.VersionControlDetails
},
Kind = new List <ResultKind> {
ResultKind.Fail
},
Level = new List <FailureLevel> {
FailureLevel.Error, FailureLevel.Warning
}
};
optionsInterpretter.ConsumeEnvVarsAndInterpretOptions(analyzeOptionsBase);
analyzeOptionsBase.DataToInsert.Should().Contain(OptionallyEmittedData.TextFiles);
analyzeOptionsBase.DataToInsert.Should().Contain(OptionallyEmittedData.BinaryFiles);
analyzeOptionsBase.DataToRemove.Should().Contain(OptionallyEmittedData.ComprehensiveRegionProperties);
analyzeOptionsBase.DataToRemove.Should().Contain(OptionallyEmittedData.RegionSnippets);
analyzeOptionsBase.Kind.Should().Contain(ResultKind.Informational);
analyzeOptionsBase.Level.Should().Contain(FailureLevel.Note);
}
private static SarifLog ExecuteTest(string path, string outputPath, string configuration = null)
{
var options = new ValidateOptions
{
TargetFileSpecifiers = new string[] { path },
OutputFilePath = outputPath,
Force = true,
ConfigurationFilePath = configuration,
Kind = new List <ResultKind> {
ResultKind.Fail
},
Level = new List <FailureLevel> {
FailureLevel.Warning, FailureLevel.Error
}
};
// Verify command returned success
int returnCode = new ValidateCommand().Run(options);
returnCode.Should().Be(0);
return(SarifLog.Load(outputPath));
}
private void Verify(string testFileName, bool disablePreleaseCompatibilityTransform = false)
{
try
{
ValidateCommand.s_DisablePrereleaseCompatibilityTransform = disablePreleaseCompatibilityTransform;
string testDirectory = Path.Combine(Environment.CurrentDirectory, TestDataDirectory);
string testFilePath = Path.Combine(TestDataDirectory, testFileName);
string expectedFilePath = MakeExpectedFilePath(testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(testDirectory, testFileName);
var validateOptions = new ValidateOptions
{
SarifOutputVersion = SarifVersion.Current,
TargetFileSpecifiers = new[] { testFilePath },
OutputFilePath = actualFilePath,
SchemaFilePath = JsonSchemaFile,
Quiet = true
};
new ValidateCommand().Run(validateOptions);
string actualLogContents = File.ReadAllText(actualFilePath);
string expectedLogContents = File.ReadAllText(expectedFilePath);
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(expectedLogContents, formatting: Newtonsoft.Json.Formatting.None, out expectedLogContents);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
}
finally
{
ValidateCommand.s_DisablePrereleaseCompatibilityTransform = false;
}
}
public void ConsumeEnvVarsAndInterpretOptions(ValidateOptions validateOptions)
{
ConsumeEnvVarsAndInterpretOptions((AnalyzeOptionsBase)validateOptions);
}
protected override string ConstructTestOutputFromInputResource(string inputResourceName, object parameter)
{
string v2LogText = GetResourceText(inputResourceName);
string inputLogDirectory = this.OutputFolderPath;
string inputLogFileName = Path.GetFileName(inputResourceName);
string inputLogFilePath = Path.Combine(this.OutputFolderPath, inputLogFileName);
string actualLogFilePath = Guid.NewGuid().ToString();
string ruleUnderTest = Path.GetFileNameWithoutExtension(inputLogFilePath).Split('.')[1];
// All SARIF rule prefixes require update to current release.
// All rules with JSON prefix are low level syntax/deserialization checks.
// We can't transform these test inputs as that operation fixes up errors in the file.
// Also, don't transform the tests for SARIF1011 or SARIF2008, because these rules
// examine the actual contents of the $schema property.
string[] shouldNotTransform = { "SARIF1011", "SARIF2008" };
bool updateInputsToCurrentSarif = IsSarifRule(ruleUnderTest) &&
!shouldNotTransform.Contains(ruleUnderTest);
var validateOptions = new ValidateOptions
{
SarifOutputVersion = SarifVersion.Current,
TargetFileSpecifiers = new[] { inputLogFilePath },
OutputFilePath = actualLogFilePath,
Quiet = true,
UpdateInputsToCurrentSarif = updateInputsToCurrentSarif,
PrettyPrint = true,
Optimize = true,
Kind = new List <ResultKind> {
ResultKind.Fail
},
Level = new List <FailureLevel> {
FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note, FailureLevel.None
},
};
var mockFileSystem = new Mock <IFileSystem>();
mockFileSystem.Setup(x => x.DirectoryExists(inputLogDirectory)).Returns(true);
mockFileSystem.Setup(x => x.DirectoryGetDirectories(It.IsAny <string>())).Returns(new string[0]);
mockFileSystem.Setup(x => x.DirectoryGetFiles(inputLogDirectory, inputLogFileName)).Returns(new string[] { inputLogFilePath });
mockFileSystem.Setup(x => x.FileReadAllText(inputLogFilePath)).Returns(v2LogText);
mockFileSystem.Setup(x => x.FileReadAllText(It.IsNotIn <string>(inputLogFilePath))).Returns <string>(path => File.ReadAllText(path));
mockFileSystem.Setup(x => x.FileWriteAllText(It.IsAny <string>(), It.IsAny <string>()));
// Some rules are disabled by default, so create a configuration file that explicitly
// enables the rule under test.
using (TempFile configFile = CreateTempConfigFile(ruleUnderTest, parameter))
{
validateOptions.ConfigurationFilePath = configFile.Name;
mockFileSystem.Setup(x => x.FileExists(validateOptions.ConfigurationFilePath)).Returns(true);
var validateCommand = new ValidateCommand(mockFileSystem.Object);
int returnCode = validateCommand.Run(validateOptions);
if (validateCommand.ExecutionException != null)
{
Console.WriteLine(validateCommand.ExecutionException.ToString());
}
returnCode.Should().Be(0);
}
string actualLogFileContents = File.ReadAllText(actualLogFilePath);
SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(actualLogFileContents);
Run run = actualLog.Runs[0];
// First, we'll strip any validation results that don't originate with the rule under test.
// But leave the results that originate from JSchema! Also, be careful because output files
// from "valid" test cases don't have any results.
run.Results = run.Results
?.Where(r => IsRelevant(r.RuleId, ruleUnderTest))
?.ToList();
// Next, remove any rule metadata for those rules. The output files from "valid" test
// cases don't have any rules.
run.Tool.Driver.Rules = run.Tool.Driver.Rules
?.Where(r => IsRelevant(r.Id, ruleUnderTest))
?.ToList();
// Since there's only one rule in the metadata, the ruleIndex for all remaining results
// must be 0.
foreach (Result result in run.Results)
{
result.RuleIndex = 0;
}
// Next, we'll remove non-deterministic information, most notably, timestamps emitted for the invocation data.
var removeTimestampsVisitor = new RemoveOptionalDataVisitor(OptionallyEmittedData.NondeterministicProperties);
removeTimestampsVisitor.Visit(actualLog);
// Finally, we'll elide non-deterministic build root details
var rebaseUrisVisitor = new RebaseUriVisitor("TEST_DIR", new Uri(inputLogDirectory));
rebaseUrisVisitor.Visit(actualLog);
// There are differences in log file output depending on whether we are invoking xunit
// from within Visual Studio or at the command-line via xunit.exe. We elide these differences.
ToolComponent driver = actualLog.Runs[0].Tool.Driver;
driver.Name = "SARIF Functional Testing";
driver.Version = null;
driver.FullName = null;
driver.SemanticVersion = null;
driver.DottedQuadFileVersion = null;
driver.Product = null;
driver.Organization = null;
driver.Properties?.Clear();
actualLog.Runs[0].OriginalUriBaseIds = null;
return(JsonConvert.SerializeObject(actualLog, Formatting.Indented));
}
