/// <summary>
/// Merges the outputs of the given redirected process to its original location based on configured policies
/// </summary>
public Task <bool> MergeOutputsIfNeededAsync(Process process, ContainerConfiguration containerConfiguration, PipExecutionContext pipExecutionContext, IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites)
{
Contract.Requires(containerConfiguration != null);
Contract.Requires(pipExecutionContext != null);
if (!process.NeedsToRunInContainer)
{
return(Task.FromResult(true));
}
return(Task.Run(
() =>
{
try
{
using (var createdDirectoriesWrapper = Pools.AbsolutePathSetPool.GetInstance())
{
HashSet <AbsolutePath> cretedDirectories = createdDirectoriesWrapper.Instance;
var result = HardlinkAllDeclaredOutputs(process, containerConfiguration, pipExecutionContext, cretedDirectories);
if (result != MergeResult.Success)
{
return false;
}
result = HardlinkOpaqueDirectories(process, containerConfiguration, pipExecutionContext, sharedDynamicWrites, cretedDirectories);
if (result != MergeResult.Success)
{
return false;
}
}
}
catch (BuildXLException ex)
{
// The operation above may throw, and in that case we don't want to propagate the exception, but grab the error message
// and interpret it as a merge failure
Tracing.Logger.Log.FailedToMergeOutputsToOriginalLocation(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), ex.Message);
return false;
}
return true;
}));
}
private MergeResult HardlinkAllDeclaredOutputs(Process process, ContainerConfiguration containerConfiguration, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
{
if (!process.ContainerIsolationLevel.IsolateOutputFiles())
{
return(MergeResult.Success);
}
foreach (FileArtifactWithAttributes fileOutput in process.FileOutputs)
{
// If the output is not there, just continue. Checking all outputs are present already happened, so this means an optional output.
// If the output is a WCI reparse point or tombstone file, continue as well. This means the output was either not actually generated (but a file with that name was read) or the file was generated and later deleted.
string sourcePath = GetRedirectedOutputPathForDeclaredOutput(containerConfiguration, pipExecutionContext, fileOutput.Path);
var sourcePathExists = FileUtilities.Exists(sourcePath);
if (!sourcePathExists || FileUtilities.IsWciReparseArtifact(sourcePath))
{
// If the declared output is a WCI artifact, that means it is not a real output. If we reached this point it is because the sandbox process executor determined this output
// was an optional one, and the content was actually not produced. Since when storing the process into the cache we will try to use the redirected file to retrieve its content,
// make sure we remove the WCI artifact, so the file actually appears as absent
// Observe this is not really needed for the case of opaque directories, since the content that gets reported is based on the existence of the file in its original location, which
// is not hardlinked if the file is a WCI artifact. We could remove them in the opaque case as well, just for a matter of symetry, but since this may have perf implications, we are not
// doing that right now.
if (sourcePathExists)
{
FileUtilities.DeleteFile(sourcePath);
}
continue;
}
string destinationPath = fileOutput.Path.ToString(m_pathTable);
var mergeResult = TryCreateHardlinkForOutput(fileOutput.Path.Expand(m_pathTable), fileOutput.RewriteCount, sourcePath, process, pipExecutionContext, createdDirectories);
if (mergeResult != MergeResult.Success)
{
return(mergeResult);
}
}
return(MergeResult.Success);
}
private MergeResult HardlinkOpaqueDirectories(
Process process,
ContainerConfiguration containerConfiguration,
PipExecutionContext pipExecutionContext,
IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites,
HashSet <AbsolutePath> createdDirectories)
{
bool isolateSharedOpaques = process.ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories();
bool isolateExclusiveOpaques = process.ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories();
// Shortcut the iteration of output directories are not isolated at all
if (!isolateExclusiveOpaques && !isolateSharedOpaques)
{
return(MergeResult.Success);
}
foreach (DirectoryArtifact directoryOutput in process.DirectoryOutputs)
{
if (directoryOutput.IsSharedOpaque && isolateSharedOpaques)
{
AbsolutePath redirectedDirectory = GetRedirectedDirectoryForOutputContainer(containerConfiguration, directoryOutput.Path).Path;
// Here we don't need to check for WCI reparse points. We know those outputs are there based on what detours is saying.
var sharedOpaqueContent = sharedDynamicWrites[directoryOutput.Path];
foreach (AbsolutePath sharedOpaqueFile in sharedOpaqueContent)
{
string sourcePath = sharedOpaqueFile.Relocate(m_pathTable, directoryOutput.Path, redirectedDirectory).ToString(m_pathTable);
// The file may not exist because the pip could have created it but later deleted it
if (!FileUtilities.Exists(sourcePath))
{
continue;
}
ExpandedAbsolutePath destinationPath = sharedOpaqueFile.Expand(m_pathTable);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(destinationPath, rewriteCount: 1, sourcePath, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
else if (!directoryOutput.IsSharedOpaque && isolateExclusiveOpaques)
{
// We need to enumerate to discover the content of an exclusive opaque, and also skip the potential reparse points
// TODO: Enumeration will happen again when the file content manager tries to discover the content of the exclusive opaque. Consider doing this only once instead.
// An output directory should only have one redirected path
ExpandedAbsolutePath redirectedDirectory = containerConfiguration.OriginalDirectories[directoryOutput.Path].Single();
foreach (string exclusiveOpaqueFile in Directory.EnumerateFiles(redirectedDirectory.ExpandedPath, "*", SearchOption.AllDirectories))
{
if (FileUtilities.IsWciReparsePoint(exclusiveOpaqueFile))
{
continue;
}
AbsolutePath exclusiveOpaqueFilePath = AbsolutePath.Create(m_pathTable, exclusiveOpaqueFile);
AbsolutePath outputFile = exclusiveOpaqueFilePath.Relocate(m_pathTable, redirectedDirectory.Path, directoryOutput.Path);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(outputFile.Expand(m_pathTable), rewriteCount: 1, exclusiveOpaqueFile, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
}
return(MergeResult.Success);
}
/// <summary>
/// Returns the redirected output corresponding to the given original declared output file
/// </summary>
/// <remarks>
/// The provided original output is assumed to be part of the container configuration
/// </remarks>
public AbsolutePath GetRedirectedDeclaredOutputFile(AbsolutePath originalOutput, ContainerConfiguration containerConfiguration)
{
if (!TryGetRedirectedDeclaredOutputFile(originalOutput, containerConfiguration, out var redirectedOutputFile))
{
Contract.Assert(false, $"A redirected directory for '{originalOutput.GetParent(m_pathTable).ToString(m_pathTable)}' should be present in the container configuration");
}
return(redirectedOutputFile);
}
/// <summary>
/// Tries to return the redirected output corresponding to the given original opaque directory root and opaque output
/// </summary>
public bool TryGetRedirectedOpaqueFile(AbsolutePath originalOutput, AbsolutePath sharedOpaqueRoot, ContainerConfiguration containerConfiguration, out AbsolutePath redirectedOutputFile)
{
if (!containerConfiguration.OriginalDirectories.TryGetValue(sharedOpaqueRoot, out var redirectedDirectories))
{
redirectedOutputFile = AbsolutePath.Invalid;
return(false);
}
var redirectedDirectory = redirectedDirectories.Single();
redirectedOutputFile = originalOutput.Relocate(m_pathTable, sharedOpaqueRoot, redirectedDirectory.Path);
return(true);
}
