public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
/// <summary>
/// Creates an output directory.
/// </summary>
protected ValueTuple <DirectoryArtifact, ReadOnlyArray <FileArtifactWithAttributes> > CreateOutputDirectory(
AbsolutePath rootPath = default(AbsolutePath),
RelativePath relativePathToDirectory = default(RelativePath),
RelativePath[] relativePathToMembers = null)
{
var directory = CreateDirectory(rootPath, relativePathToDirectory);
ReadOnlyArray <FileArtifactWithAttributes> members;
if (relativePathToMembers == null || relativePathToMembers.Length == 0)
{
members = ReadOnlyArray <FileArtifactWithAttributes> .Empty;
}
else
{
var fullMemberNames = new FileArtifactWithAttributes[relativePathToMembers.Length];
for (int i = 0; i < fullMemberNames.Length; ++i)
{
fullMemberNames[i] = FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(directory.Path.Combine(Context.PathTable, relativePathToMembers[i])), FileExistence.Required);
}
members = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(fullMemberNames);
}
return(directory, members);
}
public async Task CorrelateMoveFileAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
WriteFile(pathTable, sourceFile, "content");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: "CorrelateMoveFile",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
FileArtifactWithAttributes.Create(
FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressDest,
sourceFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressSource));
}
}
public void FileExistenceStaysTheSameWithCallToCreateNextWrittenVersion(FileExistence fileExistence)
{
var pathTable = new PathTable();
AbsolutePath filePath = AbsolutePath.Create(pathTable, A("t", "file1.txt"));
var artifact = FileArtifactWithAttributes.Create(FileArtifact.CreateSourceFile(filePath), fileExistence);
var anotherArtifact = artifact.CreateNextWrittenVersion();
XAssert.AreEqual(artifact.FileExistence, anotherArtifact.FileExistence,
"FileExistence should be the same after call to CreateNextWrittenVersion");
}
private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath)
{
// Arguments to create an output file
var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) });
var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
return(pip);
}
public void TestSerialization(FileExistence fileExistence)
{
var pathTable = new PathTable();
var fileArtifact = FileArtifactWithAttributes.Create(FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable,
A("c", "foo.txt"))), fileExistence);
HasTheSamePathAndExistence(fileArtifact, CloneViaSerialization(fileArtifact));
// Write count is not affected by serialization/deserialization
HasTheSamePathAndExistence(fileArtifact, CloneViaSerialization(fileArtifact.CreateNextWrittenVersion()));
}
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\");
var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);
var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
});
var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
new[]
{
new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
}
);
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
var pipExecutionResult = await RunProcess(context, pip);
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);
var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);
// Make sure outputs got redirected based on the configured isolation level
switch (containerIsolationLevel)
{
case ContainerIsolationLevel.IsolateOutputFiles:
XAssert.IsTrue(File.Exists(redirectedFile));
break;
case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
break;
}
}
}
public void TestTemporaryAndRequiredOutputArtifacts()
{
var pathTable = new PathTable();
AbsolutePath filePath = AbsolutePath.Create(pathTable, A("t", "file1.txt"));
var temporaryArtifact = FileArtifactWithAttributes.Create(FileArtifact.CreateSourceFile(filePath), FileExistence.Temporary).CreateNextWrittenVersion();
var requiredArtifact = FileArtifactWithAttributes.Create(FileArtifact.CreateSourceFile(filePath), FileExistence.Required).CreateNextWrittenVersion();
var optionalArtifact = FileArtifactWithAttributes.Create(FileArtifact.CreateSourceFile(filePath), FileExistence.Optional).CreateNextWrittenVersion();
XAssert.IsTrue(temporaryArtifact.IsOutputFile, "Instance should be output");
XAssert.IsTrue(requiredArtifact.IsOutputFile, "Instance should be output");
XAssert.IsTrue(optionalArtifact.IsOutputFile, "Instance should be output");
XAssert.IsTrue(temporaryArtifact.IsTemporaryOutputFile, "Temporary artifact should be temporary");
XAssert.IsFalse(requiredArtifact.IsTemporaryOutputFile, "Required artifact is not temporary");
XAssert.IsFalse(optionalArtifact.IsTemporaryOutputFile, "Optional artifact is not temporary");
XAssert.IsFalse(temporaryArtifact.IsRequiredOutputFile, "Temporary artifact is not required");
XAssert.IsTrue(requiredArtifact.IsRequiredOutputFile, "Required artifact is required");
XAssert.IsFalse(optionalArtifact.IsRequiredOutputFile, "Optional artifact is not required");
}
public void FileArtifactEquality()
{
var pathTable = new PathTable();
FileArtifact file1 = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, A("t", "file1.txt")));
FileArtifact file2 = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, A("t", "file2.txt")));
StructTester.TestEquality(
baseValue: FileArtifactWithAttributes.Create(file1, FileExistence.Required),
equalValue: FileArtifactWithAttributes.Create(file1, FileExistence.Required),
notEqualValues: new[]
{
FileArtifactWithAttributes.Create(file1, FileExistence.Optional),
FileArtifactWithAttributes.Create(file1, FileExistence.Optional).CreateNextWrittenVersion(),
FileArtifactWithAttributes.Create(file2, FileExistence.Temporary),
FileArtifactWithAttributes.Create(file2, FileExistence.Temporary).CreateNextWrittenVersion(),
FileArtifactWithAttributes.Create(file1, FileExistence.Required).CreateNextWrittenVersion(),
FileArtifactWithAttributes.Create(file1, FileExistence.Required).CreateNextWrittenVersion().CreateNextWrittenVersion(),
FileArtifactWithAttributes.Create(file1.CreateNextWrittenVersion(), FileExistence.Required)
},
eq: (left, right) => left == right,
neq: (left, right) => left != right,
skipHashCodeForNotEqualValues: true);
}
public async Task DynamicDirectoryMaterializationDoesNotDeleteDeclaredFiles()
{
var harness = CreateDefaultHarness();
harness.Seal();
var pathTable = harness.Environment.Context.PathTable;
DirectoryArtifact dynamicOutputDirectory = CreateDirectory();
FileArtifact explicitDeclaredOutputFile = CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "expOut.txt");
string explicitNestedDeclaredOutputFileContents = "This is a nested declared file dependency";
var nestedDirectory = dynamicOutputDirectory.Path.Combine(pathTable, "nested");
FileArtifact explicitNestedDeclaredOutputFile = CreateOutputFile(rootPath: nestedDirectory, fileName: "expNestOut.txt");
string explicitDeclaredOutputFileContents = "This is a declared file dependency";
FileArtifact dynamicOutputFile = CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "dynout.txt");
string dynamicOutputFileContents = "This is a dynamic file dependency";
await harness.StoreAndReportStringContent(explicitDeclaredOutputFileContents, explicitDeclaredOutputFile);
await harness.StoreAndReportStringContent(explicitNestedDeclaredOutputFileContents, explicitNestedDeclaredOutputFile);
await harness.StoreAndReportStringContent(dynamicOutputFileContents, dynamicOutputFile);
const string PriorOutputText = "Prior output";
harness.WriteText(dynamicOutputFile, PriorOutputText);
harness.WriteText(explicitDeclaredOutputFile, PriorOutputText);
harness.WriteText(explicitNestedDeclaredOutputFile, PriorOutputText);
// Add some extraneous files/directories to be removed
const string RemovedFileText = "Removed file text";
var removedNestedDirectory = dynamicOutputDirectory.Path.Combine(pathTable, "removedNested");
harness.WriteText(CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "removedRootFile.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "removedRootFile2.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: nestedDirectory, fileName: "nestedremoved.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: nestedDirectory, fileName: "nestedremoved1.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: removedNestedDirectory, fileName: "removedNestedFile.txt"), RemovedFileText);
Directory.CreateDirectory(dynamicOutputDirectory.Path.Combine(pathTable, "removedEmptyNested").ToString(pathTable));
Directory.CreateDirectory(dynamicOutputDirectory.Path.Combine(pathTable, "removedEmptyNested3").ToString(pathTable));
harness.Environment.RegisterDynamicOutputDirectory(dynamicOutputDirectory);
var dynamicDirectoryContents = new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile, dynamicOutputFile };
// Report files for directory
harness.FileContentManager.ReportDynamicDirectoryContents(
dynamicOutputDirectory,
dynamicDirectoryContents.Select(fa => FileArtifactWithAttributes.Create(fa, FileExistence.Required)),
PipOutputOrigin.NotMaterialized);
var producer = CreateCmdProcess(
dependencies: new FileArtifact[0],
outputs: new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile },
directoryOutputs: new[] { dynamicOutputDirectory });
var fileConsumer = CreateCmdProcess(
dependencies: new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile },
outputs: new[] { CreateOutputFile() });
var directoryConsumer = CreateCmdProcess(
dependencies: new FileArtifact[0],
directoryDependencies: new[] { dynamicOutputDirectory },
outputs: new[] { CreateOutputFile() });
// First materialize the files by materializing inputs for the pip which consumes the files directly
var fileMaterializationResult = await harness.FileContentManager.TryMaterializeDependenciesAsync(fileConsumer, harness.UntrackedOpContext);
Assert.True(fileMaterializationResult);
// Verify that only the explicit declared dependencies have their contents materialized
harness.VerifyContent(explicitDeclaredOutputFile, explicitDeclaredOutputFileContents);
harness.VerifyContent(explicitNestedDeclaredOutputFile, explicitNestedDeclaredOutputFileContents);
// The dynamic file shouldn't be materialized at this point
harness.VerifyContent(dynamicOutputFile, PriorOutputText);
// Modify the explicit outputs so that we can check later that materialization did not
// delete and re-materialize these files
const string ModifiedExplicitOutputText = "Modified explicit output";
harness.WriteText(explicitDeclaredOutputFile, ModifiedExplicitOutputText);
harness.WriteText(explicitNestedDeclaredOutputFile, ModifiedExplicitOutputText);
// Now materialize the directory
var directoryMaterializationResult = await harness.FileContentManager.TryMaterializeDependenciesAsync(directoryConsumer, harness.UntrackedOpContext);
Assert.True(directoryMaterializationResult);
var filesAfterMaterialization = new HashSet <string>(Directory.GetFiles(dynamicOutputDirectory.Path.ToString(pathTable), "*.*", SearchOption.AllDirectories), OperatingSystemHelper.PathComparer);
HashSet <string> dynamicDirectoryContentPaths = new HashSet <string>(OperatingSystemHelper.PathComparer);
dynamicDirectoryContentPaths.UnionWith(dynamicDirectoryContents.Select(f => f.Path.ToString(pathTable)));
// Check that the dynamic directory contents are the only remaining files
Assert.Subset(dynamicDirectoryContentPaths, filesAfterMaterialization);
Assert.Superset(dynamicDirectoryContentPaths, filesAfterMaterialization);
var directoriesAfterMaterialization = new HashSet <string>(Directory.GetDirectories(dynamicOutputDirectory.Path.ToString(pathTable), "*.*", SearchOption.AllDirectories), OperatingSystemHelper.PathComparer);
HashSet <string> dynamicDirectorySubDirectoryPaths = new HashSet <string>(OperatingSystemHelper.PathComparer);
dynamicDirectorySubDirectoryPaths.UnionWith(dynamicDirectoryContents.Select(f => f.Path.GetParent(pathTable).ToString(pathTable)));
// Don't count the root directory
dynamicDirectorySubDirectoryPaths.Remove(dynamicOutputDirectory.Path.ToString(pathTable));
// Check that the dynamic directory contents parent directories are the only remaining directories
Assert.Subset(dynamicDirectorySubDirectoryPaths, directoriesAfterMaterialization);
Assert.Superset(dynamicDirectorySubDirectoryPaths, directoriesAfterMaterialization);
XAssert.IsTrue(fileMaterializationResult);
// Verify all files have the expected contents
// Dynamic file should have the content for the cache
// Explicit dependencies should have the modified explicit content because
// the file content manager skips content which has been materialized already so
// the later modifications should go unnoticed
harness.VerifyContent(explicitDeclaredOutputFile, ModifiedExplicitOutputText);
harness.VerifyContent(explicitNestedDeclaredOutputFile, ModifiedExplicitOutputText);
harness.VerifyContent(dynamicOutputFile, dynamicOutputFileContents);
}