private bool TryResolveSymlinkedPath(string path, out ExpandedAbsolutePath expandedFinalPath)
{
if (!FileUtilities.TryGetFinalPathNameByPath(path, out string finalPathAsString, out _, volumeGuidPath: false))
{
// If the final path cannot be resolved (most common cause is file not found), we stay with the original path
expandedFinalPath = default;
return(false);
}
if (m_directoryTranslator != null)
{
finalPathAsString = m_directoryTranslator.Translate(finalPathAsString);
}
// We want to compare the final path with the parsed path, so let's go through the path table
// to fully canonicalize it
var success = AbsolutePath.TryCreate(m_context.PathTable, finalPathAsString, out var finalPath);
if (!success)
{
Contract.Assume(false, $"The result of GetFinalPathNameByPath should always be a path we can parse. Original path is '{path}', final path is '{finalPathAsString}'.");
}
expandedFinalPath = ExpandedAbsolutePath.CreateUnsafe(finalPath, finalPathAsString);
return(true);
}
/// <inheritdoc />
public Task <Possible <Unit, Failure> > TryMaterializeAsync(
FileRealizationMode fileRealizationMode,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
return(Task.Run(
() =>
{
lock (m_lock)
{
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
if ((entry.Sites & CacheSites.Local) == 0)
{
return new Failure <string>("Content is available in 'remote' cache but is not local. Load it locally first with TryLoadAvailableContentAsync.");
}
string expandedPath = path.ExpandedPath;
var result = ExceptionUtilities.HandleRecoverableIOExceptionAsync(
async() => await PlaceFileInternalAsync(expandedPath, contentHash, fileRealizationMode),
ex => { throw new BuildXLException("Failed to materialize content (content found, but couldn't write it)", ex); });
m_pathRealizationModes[expandedPath] = fileRealizationMode;
return result.Result;
}
else
{
return new Failure <string>("Content not found (locally or remotely). Store it first with TryStoreAsync.");
}
}
}));
}
/// <inheritdoc />
public async Task <Possible <Unit, Failure> > TryStoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
Possible <ContentHash, Failure> maybeStored = await Helpers.RetryOnFailureAsync(
async lastAttempt =>
{
return(await TryStoreAsync(fileRealizationModes, path));
});
return(maybeStored.Then <Unit>(
cacheReportedHash =>
{
if (cacheReportedHash == contentHash)
{
return Unit.Void;
}
else
{
return new Failure <string>(
string.Format(
CultureInfo.InvariantCulture,
"Stored content had an unexpected hash. (expected: {0}; actual: {1})",
contentHash,
cacheReportedHash));
}
}));
}
/// <summary>
/// Adds synthetic accesses for all intermediate directory symlinks for the given access path
/// </summary>
/// <remarks>
/// TODO: This function is only adding accesses for symlinks in the given path, so if those suymlinks point to other symlinks,
/// those are not added. So the result is not completely sound, consider doing multi-hop resolution.
/// </remarks>
public void AddAccessesForIntermediateSymlinks(FileAccessManifest manifest, ReportedFileAccess access, AbsolutePath accessPath, Dictionary <AbsolutePath, CompactSet <ReportedFileAccess> > accessesByPath)
{
Contract.Requires(accessPath.IsValid);
AbsolutePath currentPath = accessPath.GetParent(m_context.PathTable);
while (currentPath.IsValid)
{
// If we the current path is resolved and its resolved path is the same, then we know there are no more symlinks
// in the path. Shorcut the search.
if (m_resolvedPathCache.TryGetValue(currentPath, out var resolvedPath) && currentPath == resolvedPath)
{
return;
}
bool isDirSymlink = IsDirectorySymlinkOrJunctionWithCache(ExpandedAbsolutePath.CreateUnsafe(currentPath, currentPath.ToString(m_context.PathTable)));
if (isDirSymlink)
{
accessesByPath.TryGetValue(currentPath, out CompactSet <ReportedFileAccess> existingAccessesToPath);
var generatedProbe = GenerateProbeForPath(manifest, currentPath, access);
accessesByPath[currentPath] = existingAccessesToPath.Add(generatedProbe);
}
currentPath = currentPath.GetParent(m_context.PathTable);
}
}
/// <inheritdoc />
public async Task <Possible <Unit, Failure> > TryMaterializeAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
// TODO: The cache should be able to do this itself, preferably sharing the same code.
// When placing content, we may be replacing output that has been hardlinked elsewhere.
// Deleting links requires some care and magic, e.g. if a process has the file mapped.
// Correspondingly, IArtifactContentCache prescribes that materialization always produces a 'new' file.
Possible <Unit, Failure> result = Unit.Void;
var placeResult = await Helpers.RetryOnFailureAsync(
async lastAttempt =>
{
result = await TryMaterializeCoreAsync(fileRealizationModes, path, contentHash);
if (!result.Succeeded)
{
throw new BuildXLException(
I($"Failed to materialize '{path.ExpandedPath}': {result.Failure.DescribeIncludingInnerFailures()}"),
ExceptionRootCause.Unknown);
}
return(result.Succeeded);
});
Contract.Assert(placeResult || !result.Succeeded);
return(result);
}
private async Task <Possible <Unit, Failure> > TryMaterializeCoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
FileToDelete fileToDelete = FileToDelete.Create(path.ExpandedPath);
string pathForCache = GetExpandedPathForCache(path);
FileToDelete fileForCacheToDelete = (string.IsNullOrEmpty(pathForCache) ||
string.Equals(path.ExpandedPath, pathForCache, OperatingSystemHelper.PathComparison))
? FileToDelete.Invalid
: FileToDelete.Create(pathForCache);
if (!m_replaceExistingFileOnMaterialization)
{
// BuildXL controls the file deletion if the place file mode is FailIfExists.
var mayBeDelete = fileToDelete.TryDelete();
// The file materialization below can fail if fileToDelete and fileForCacheToDelete
// point to different object files. One can think that fileForCacheToDelete should
// be deleted as well by adding the following expression in the above statement:
//
// .Then(r => fileForCacheToDelete.IsValid ? fileForCacheToDelete.TryDelete() : r);
//
// However, this deletion masks a possibly serious underlying issue because we expect
// both fileToDelete and fileForCacheToDelete point to the same object file.
if (!mayBeDelete.Succeeded)
{
return(new FailToDeleteForMaterializationFailure(mayBeDelete.Failure));
}
}
if (fileRealizationModes.AllowVirtualization)
{
pathForCache = AddVfsSuffix(pathForCache);
}
Possible <ICacheSession, Failure> maybeOpen = m_cache.Get(nameof(TryMaterializeAsync));
Possible <string, Failure> maybePlaced = await PerformArtifactCacheOperationAsync(
() => maybeOpen.ThenAsync(cache => cache.ProduceFileAsync(
new CasHash(new global::BuildXL.Cache.Interfaces.Hash(contentHash)),
pathForCache,
GetFileStateForRealizationMode(fileRealizationModes))),
nameof(TryMaterializeAsync));
if (!maybePlaced.Succeeded && maybePlaced.Failure.DescribeIncludingInnerFailures().Contains("File exists at destination"))
{
string diagnostic = await fileToDelete.GetDiagnosticsAsync();
diagnostic = fileForCacheToDelete.IsValid
? diagnostic + Environment.NewLine + (await fileForCacheToDelete.GetDiagnosticsAsync())
: diagnostic;
return(maybePlaced.Failure.Annotate(diagnostic));
}
return(maybePlaced.Then(p => Unit.Void));
}
/// <inheritdoc />
public Task <Possible <Unit, Failure> > TryMaterializeAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash,
CancellationToken cancellationToken = default)
{
return(Task.Run <Possible <Unit, Failure> >(
() =>
{
lock (m_lock)
{
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
if ((entry.Sites & CacheSites.Local) == 0)
{
return new Failure <string>("Content is available in 'remote' cache but is not local. Load it locally first with TryLoadAvailableContentAsync.");
}
string expandedPath = path.ExpandedPath;
// IArtifactContentCache prescribes that materialization always produces a 'new' file.
var mayBeDelete = FileUtilities.TryDeletePathIfExists(expandedPath);
if (!mayBeDelete.Succeeded)
{
return new FailToDeleteForMaterializationFailure(mayBeDelete.Failure);
}
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[expandedPath] = fileRealizationModes;
}
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(expandedPath));
File.WriteAllBytes(expandedPath, entry.Content);
},
ex => { throw new BuildXLException("Failed to materialize content (content found, but couldn't write it)", ex); });
return Unit.Void;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
else
{
return new Failure <string>("Content not found (locally or remotely). Store it first with TryStoreAsync.");
}
}
}));
}
/// <inheritdoc />
public Task <Possible <ContentHash, Failure> > TryStoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path)
{
return(TryStoreInternalAsync(
path,
fileRealizationModes,
knownContentHash: null));
}
/// <inheritdoc />
public async Task <Possible <Unit, Failure> > TryStoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
Possible <ContentHash, Failure> maybeStored = await TryStoreInternalAsync(
path,
fileRealizationModes,
knownContentHash : contentHash);
return(maybeStored.Then(hash => Unit.Void));
}
/// <inheritdoc />
public async Task <Possible <ContentHash, Failure> > TryStoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path)
{
string expandedPath = GetExpandedPathForCache(path);
Possible <ICacheSession, Failure> maybeOpen = m_cache.Get(nameof(TryStoreAsync));
Possible <CasHash, Failure> maybeStored = await maybeOpen
.ThenAsync(cache => cache.AddToCasAsync(expandedPath, GetFileStateForRealizationMode(fileRealizationModes)));
return(maybeStored.Then <ContentHash>(c => c.ToContentHash()));
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationModes,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() => { return File.ReadAllBytes(path.ExpandedPath); },
ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); });
ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes);
if (knownContentHash.HasValue && contentHash != knownContentHash.Value)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})"));
}
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
// We assume that stores of content already present somewhere still cause replication
// to both the local and remote sites. See class remarks.
entry.Sites |= CacheSites.LocalAndRemote;
return contentHash;
}
else
{
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes;
}
// We assume that stored content is instantly and magically replicated to some remote place.
// See class remarks.
m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote);
return contentHash;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
}
}));
}
private void VerifyExpandedPathForCacheEquals(CacheCoreArtifactContentCache cache,
ExpandedAbsolutePath originalRoot,
ExpandedAbsolutePath expectedRoot)
{
var pathTable = Context.PathTable;
string fileName = "bits.txt";
var originalRootExpandedPath = AbsolutePath.Create(pathTable, R(originalRoot.ExpandedPath, fileName)).Expand(pathTable);
var expectedRootExpandedPath = AbsolutePath.Create(pathTable, R(expectedRoot.ExpandedPath, fileName)).Expand(pathTable);
Assert.Equal(
expectedRootExpandedPath.ExpandedPath,
cache.GetExpandedPathForCache(originalRootExpandedPath));
}
private bool IsDirectorySymlinkOrJunctionWithCache(ExpandedAbsolutePath path)
{
if (m_symlinkCache.TryGet(path.Path) is var cachedResult && cachedResult.IsFound)
{
return(cachedResult.Item.Value);
}
var result = FileUtilities.IsDirectorySymlinkOrJunction(path.ExpandedPath);
m_symlinkCache.TryAdd(path.Path, result);
return(result);
}
/// <summary>
/// Tries to store symlink file to cache.
/// </summary>
public static async Task <Possible <ContentHash> > TryStoreToCacheAsync(
LoggingContext loggingContext,
IArtifactContentCache cache,
ExpandedAbsolutePath symlinkFile)
{
var possibleStore = await cache.TryStoreAsync(FileRealizationMode.HardLinkOrCopy, symlinkFile);
if (!possibleStore.Succeeded)
{
Tracing.Logger.Log.FailedStoreSymlinkFileToCache(loggingContext, symlinkFile.ExpandedPath, possibleStore.Failure.DescribeIncludingInnerFailures());
return(possibleStore.Failure);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file '{symlinkFile}' with hash '{possibleStore.Result}'."));
return(possibleStore.Result);
}
/// <inheritdoc />
public async Task <Possible <Unit, Failure> > TryMaterializeAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
// TODO: The cache should be able to do this itself, preferably sharing the same code.
// When placing content, we may be replacing output that has been hardlinked elsewhere.
// Deleting links requires some care and magic, e.g. if a process has the file mapped.
// Correspondingly, IArtifactContentCache prescribes that materialization always produces a 'new' file.
var placeResult = await Helpers.RetryOnFailureAsync(
async lastAttempt =>
{
return(await TryMaterializeCoreAsync(fileRealizationModes, path, contentHash));
});
return(placeResult);
}
private bool ShouldNotResolveLastSegment(ExpandedAbsolutePath accessPath, ReportedFileOperation operation, FlagsAndAttributes flagsAndAttributes, out bool isReparsePoint)
{
isReparsePoint = IsDirectorySymlinkOrJunctionWithCache(accessPath);
// The following operations act on the reparse point directly, not on the target
// TODO: the logic is not perfect but good enough. The detours implementation will replace
// this eventually
var operationOnReparsePoint =
(operation == ReportedFileOperation.CreateHardLinkSource ||
operation == ReportedFileOperation.CreateSymbolicLinkSource ||
operation == ReportedFileOperation.GetFileAttributes ||
operation == ReportedFileOperation.GetFileAttributesEx ||
operation == ReportedFileOperation.DeleteFile ||
(operation == ReportedFileOperation.CreateFile && (flagsAndAttributes & FlagsAndAttributes.FILE_FLAG_OPEN_REPARSE_POINT) != 0) ||
(operation == ReportedFileOperation.NtCreateFile && (flagsAndAttributes & FlagsAndAttributes.FILE_FLAG_OPEN_REPARSE_POINT) != 0) ||
(operation == ReportedFileOperation.ZwCreateFile && (flagsAndAttributes & FlagsAndAttributes.FILE_FLAG_OPEN_REPARSE_POINT) != 0) ||
(operation == ReportedFileOperation.ZwOpenFile && (flagsAndAttributes & FlagsAndAttributes.FILE_FLAG_OPEN_REPARSE_POINT) != 0));
return(operationOnReparsePoint && isReparsePoint);
}
/// <summary>
/// Gets the expanded path for the cache to use when putting/placing file. This allows
/// the cache to pick a correct CAS root if CAS roots exist on various drives which
/// would allow it to use a hardlink rather than copy if the appropriate CAS root is chosen.
/// </summary>
public string GetExpandedPathForCache(ExpandedAbsolutePath path)
{
if (m_rootTranslator == null)
{
return(path.ExpandedPath);
}
else
{
var translatedPath = m_rootTranslator.Translate(path.ExpandedPath);
if (translatedPath[0].ToUpperInvariantFast() == path.ExpandedPath[0].ToUpperInvariantFast() &&
translatedPath.Length > path.ExpandedPath.Length)
{
// The path root did not change as a result of path translation and its longer
// so , just return the original path since the cache only cares about the root
// when deciding a particular CAS to hardlink from to avoid MAX_PATH issues
return(path.ExpandedPath);
}
return(translatedPath);
}
}
/// <inheritdoc />
public async Task <Possible <ContentHash, Failure> > TryStoreAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path)
{
string expandedPath = GetExpandedPathForCache(path);
Possible <ICacheSession, Failure> maybeOpen = m_cache.Get(nameof(TryStoreAsync));
Possible <CasHash, Failure> maybeStored = await PerformArtifactCacheOperationAsync(
() => maybeOpen.ThenAsync(
async cache =>
{
var result = await Helpers.RetryOnFailureAsync(
async lastAttempt =>
{
return(await cache.AddToCasAsync(expandedPath, GetFileStateForRealizationMode(fileRealizationModes)));
});
return(result);
}),
nameof(TryStoreAsync));
return(maybeStored.Then <ContentHash>(c => c.ToContentHash()));
}
private AbsolutePath ResolvePathWithCache(
AbsolutePath accessPath,
[CanBeNull] string accessPathAsString,
ReportedFileOperation operation,
FlagsAndAttributes flagsAndAttributes,
[CanBeNull] out string resolvedPathAsString,
out bool isDirectoryReparsePoint)
{
accessPathAsString ??= accessPath.ToString(m_context.PathTable);
// If the final segment is a directory reparse point and the operation acts on it, don't resolve it
if (ShouldNotResolveLastSegment(ExpandedAbsolutePath.CreateUnsafe(accessPath, accessPathAsString), operation, flagsAndAttributes, out isDirectoryReparsePoint))
{
AbsolutePath resolvedParentPath = ResolvePathWithCache(accessPath.GetParent(m_context.PathTable), accessPathAsString: null, operation, flagsAndAttributes, out string resolvedParentPathAsString, out _);
PathAtom name = accessPath.GetName(m_context.PathTable);
resolvedPathAsString = resolvedParentPathAsString != null?Path.Combine(resolvedParentPathAsString, name.ToString(m_context.StringTable)) : null;
return(resolvedParentPath.Combine(m_context.PathTable, name));
}
// Check the cache
var cachedResult = m_resolvedPathCache.TryGet(accessPath);
if (cachedResult.IsFound)
{
// Observe we don't cache expanded paths since that might cause the majority of all the paths of a build to live
// in memory for the whole execution time. So in case of a cache hit, we just return the absolute path. The expanded
// path needs to be reconstructed as needed.
resolvedPathAsString = null;
return(cachedResult.Item.Value);
}
// If not found and the path is not a reparse point, check the cache for its parent.
// Many files may have the same parent, and since the path is not a reparse point we know that
// resolve(parent\segment) == resolve(parent)\segment
var parentPath = accessPath.GetParent(m_context.PathTable);
if (parentPath.IsValid && !isDirectoryReparsePoint && m_resolvedPathCache.TryGet(parentPath) is var cachedParent && cachedParent.IsFound)
{
var lastFragment = accessPath.GetName(m_context.PathTable);
resolvedPathAsString = null;
AbsolutePath cachedParentPath = cachedParent.Item.Value;
return(cachedParentPath.IsValid ?
cachedParentPath.Combine(m_context.PathTable, lastFragment) :
AbsolutePath.Create(m_context.PathTable, lastFragment.ToString(m_context.StringTable)));
}
// The cache didn't have it, so let's resolve it
if (!TryResolveSymlinkedPath(accessPathAsString, out ExpandedAbsolutePath resolvedExpandedPath))
{
// If we cannot get the final path (e.g. file not found causes this), then we assume the path
// is already canonicalized
// Observe we cannot update the cache since the path was not resolved.
// TODO: we could consider always trying to resolve the parent
resolvedPathAsString = accessPathAsString;
return(accessPath);
}
// Update the cache
m_resolvedPathCache.TryAdd(accessPath, resolvedExpandedPath.Path);
// If the path is not a reparse point, update the parent as well
if (!isDirectoryReparsePoint && parentPath.IsValid)
{
// Observe we may be storing an invalid path if the resolved path does not have a parent.
m_resolvedPathCache.TryAdd(parentPath, resolvedExpandedPath.Path.GetParent(m_context.PathTable));
}
// If the resolved path is the same as the access path, then we know its last fragment is not a reparse point.
// So we might just update the symlink cache as well and hopefully speed up subsequent requests to generate read accesses
// for intermediate symlink dirs
if (accessPath == resolvedExpandedPath.Path)
{
while (parentPath.IsValid && m_symlinkCache.TryAdd(parentPath, false))
{
parentPath = parentPath.GetParent(m_context.PathTable);
}
}
resolvedPathAsString = resolvedExpandedPath.ExpandedPath;
return(resolvedExpandedPath.Path);
}
public Task <Possible <ContentHash, Failure> > TryStoreAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path)
{
return(this.m_cache.TryStoreAsync(fileRealizationModes, path));
}
public Task <Possible <Unit, Failure> > TryMaterializeAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, ContentHash contentHash)
{
return(this.m_cache.TryMaterializeAsync(fileRealizationModes, path, contentHash));
}
public Possible <PathExistence, Failure> TryProbeAndTrackPathForExistence(ExpandedAbsolutePath path, bool?isReadOnly = default)
{
AssertAdd(ProbePaths, path.Path);
AssertAdd(AllProbePaths, path.Path);
if (m_files.Contains(path.Path))
{
return(PathExistence.ExistsAsFile);
}
if (m_directories.ContainsKey(path.Path))
{
return(PathExistence.ExistsAsDirectory);
}
return(PathExistence.Nonexistent);
}
private MergeResult TryCreateHardlinkForOutput(ExpandedAbsolutePath fileOutput, int rewriteCount, string sourcePath, Process process, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
{
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out bool isDisallowed, out bool shouldDelete))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
if (shouldDelete)
{
FileUtilities.DeleteFile(fileOutput.ExpandedPath, waitUntilDeletionFinished: true);
}
else
{
// If we need to delete the target file, the directory is already created. So only do this otherwise.
// Make sure the destination directory exists before trying to create the hardlink
var containingDirectory = fileOutput.Path.GetParent(m_pathTable);
if (!createdDirectories.Contains(containingDirectory))
{
string destinationDirectory = containingDirectory.ToString(m_pathTable);
if (!FileUtilities.DirectoryExistsNoFollow(destinationDirectory))
{
FileUtilities.CreateDirectory(destinationDirectory);
}
createdDirectories.Add(containingDirectory);
}
}
var createHardlinkStatus = FileUtilities.TryCreateHardLink(fileOutput.ExpandedPath, sourcePath);
if (createHardlinkStatus != CreateHardLinkStatus.Success)
{
// Maybe somebody else raced and successfully created the hardlink since the last time we checked,
// so let's check again before failing
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out isDisallowed, out bool _))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
Tracing.Logger.Log.FailedToCreateHardlinkOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath, createHardlinkStatus.ToString());
return(MergeResult.Failure);
}
return(MergeResult.Success);
}
private MergeResult HardlinkOpaqueDirectories(
Process process,
ContainerConfiguration containerConfiguration,
PipExecutionContext pipExecutionContext,
IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites,
HashSet <AbsolutePath> createdDirectories)
{
bool isolateSharedOpaques = process.ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories();
bool isolateExclusiveOpaques = process.ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories();
// Shortcut the iteration of output directories are not isolated at all
if (!isolateExclusiveOpaques && !isolateSharedOpaques)
{
return(MergeResult.Success);
}
foreach (DirectoryArtifact directoryOutput in process.DirectoryOutputs)
{
if (directoryOutput.IsSharedOpaque && isolateSharedOpaques)
{
AbsolutePath redirectedDirectory = GetRedirectedDirectoryForOutputContainer(containerConfiguration, directoryOutput.Path).Path;
// Here we don't need to check for WCI reparse points. We know those outputs are there based on what detours is saying.
var sharedOpaqueContent = sharedDynamicWrites[directoryOutput.Path];
foreach (AbsolutePath sharedOpaqueFile in sharedOpaqueContent)
{
string sourcePath = sharedOpaqueFile.Relocate(m_pathTable, directoryOutput.Path, redirectedDirectory).ToString(m_pathTable);
// The file may not exist because the pip could have created it but later deleted it
if (!FileUtilities.Exists(sourcePath))
{
continue;
}
ExpandedAbsolutePath destinationPath = sharedOpaqueFile.Expand(m_pathTable);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(destinationPath, rewriteCount: 1, sourcePath, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
else if (!directoryOutput.IsSharedOpaque && isolateExclusiveOpaques)
{
// We need to enumerate to discover the content of an exclusive opaque, and also skip the potential reparse points
// TODO: Enumeration will happen again when the file content manager tries to discover the content of the exclusive opaque. Consider doing this only once instead.
// An output directory should only have one redirected path
ExpandedAbsolutePath redirectedDirectory = containerConfiguration.OriginalDirectories[directoryOutput.Path].Single();
foreach (string exclusiveOpaqueFile in Directory.EnumerateFiles(redirectedDirectory.ExpandedPath, "*", SearchOption.AllDirectories))
{
if (FileUtilities.IsWciReparsePoint(exclusiveOpaqueFile))
{
continue;
}
AbsolutePath exclusiveOpaqueFilePath = AbsolutePath.Create(m_pathTable, exclusiveOpaqueFile);
AbsolutePath outputFile = exclusiveOpaqueFilePath.Relocate(m_pathTable, redirectedDirectory.Path, directoryOutput.Path);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(outputFile.Expand(m_pathTable), rewriteCount: 1, exclusiveOpaqueFile, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
}
return(MergeResult.Success);
}
/// <summary>
/// Store the fingerprint store directory to the cache
/// </summary>
/// <remark>
/// The order of storing should be:
/// 1. The actual content(files) of the fingerprint store
/// 2. The metadata(descriptor) of the content.
/// 3. Publich the cache entry of the fingerprint store.
/// </remark>
public static async Task <Possible <long> > TrySaveFingerprintStoreAsync(
this EngineCache cache,
LoggingContext loggingContext,
AbsolutePath path,
PathTable pathTable,
string key,
string environment)
{
var fingerprint = ComputeFingerprint(pathTable, key, environment);
var pathStr = path.ToString(pathTable);
BoxRef <long> size = 0;
SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8);
var tasks = new List <Task <Possible <StringKeyedHash, Failure> > >();
FileUtilities.EnumerateDirectoryEntries(pathStr, (name, attr) =>
{
var task = Task.Run(async() =>
{
using (await concurrencyLimiter.AcquireAsync())
{
var filePath = path.Combine(pathTable, name);
ExpandedAbsolutePath expandedFilePath = filePath.Expand(pathTable);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
expandedFilePath);
var result = storeResult.Then(result => new StringKeyedHash()
{
Key = path.ExpandRelative(pathTable, filePath),
ContentHash = result.ToBondContentHash()
});
string message = I($"Saving fingerprint store to cache: Success='{result.Succeeded}', FilePath='{expandedFilePath}'");
if (result.Succeeded)
{
Interlocked.Add(ref size.Value, new FileInfo(filePath.ToString(pathTable)).Length);
message += I($", Key='{result.Result.Key}', Hash='{result.Result.ContentHash.ToContentHash()}'");
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, message);
return(result);
}
});
tasks.Add(task);
});
var storedFiles = await Task.WhenAll(tasks);
if (storedFiles.Length == 0 || size.Value == 0)
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Empty fingerprint store is not saved."));
return(0);
}
var failure = storedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault();
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{failure == null}', FileCount={storedFiles.Length} Size={size.Value}"));
if (failure != null)
{
return(failure);
}
PackageDownloadDescriptor descriptor = new PackageDownloadDescriptor()
{
TraceInfo = loggingContext.Session.Environment,
FriendlyName = nameof(FingerprintStore),
Contents = storedFiles.Select(p => p.Result).ToList()
};
var storeDescriptorResult = await cache.ArtifactContentCache.TrySerializeAndStoreContent(descriptor);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store descriptor to cache: Success='{storeDescriptorResult.Succeeded}'"));
if (!storeDescriptorResult.Succeeded)
{
return(storeDescriptorResult.Failure);
}
var associatedFileHashes = descriptor.Contents.Select(s => s.ContentHash.ToContentHash()).ToArray().ToReadOnlyArray().GetSubView(0);
var cacheEntry = new CacheEntry(storeDescriptorResult.Result, null, associatedFileHashes);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, fingerprint, cacheEntry);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Publishing fingerprint store to cache: Fingerprint='{fingerprint}' Hash={storeDescriptorResult.Result}"));
return(size.Value);
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationModes,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
var expandedPath = path.ExpandedPath;
using (FileStream fileStream = new FileStream(expandedPath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (BinaryReader binaryReader = new BinaryReader(fileStream))
{
// This will work with files up to 2GB in length, due to the 'int' API signature
return binaryReader.ReadBytes((int)new FileInfo(expandedPath).Length);
}
},
ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); });
ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes);
if (knownContentHash.HasValue && contentHash != knownContentHash.Value)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})"));
}
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
// We assume that stores of content already present somewhere still cause replication
// to both the local and remote sites. See class remarks.
entry.Sites |= CacheSites.LocalAndRemote;
return contentHash;
}
else
{
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes;
}
// We assume that stored content is instantly and magically replicated to some remote place.
// See class remarks.
m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote);
return contentHash;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
}
}));
}
/// <summary>
/// Adds a root translation
/// </summary>
public void AddTranslation(ExpandedAbsolutePath sourcePath, ExpandedAbsolutePath targetPath)
{
AddTranslation(sourcePath.ExpandedPath, targetPath.ExpandedPath);
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationMode,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
string expandedPath = path.ExpandedPath;
ContentHash contentHash;
if (knownContentHash.HasValue)
{
contentHash = knownContentHash.Value;
var mayBeHash = TryHashFile(expandedPath);
if (!mayBeHash.Succeeded)
{
return mayBeHash.Failure;
}
if (contentHash != mayBeHash.Result)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {contentHash}; actual: {mayBeHash.Result})"));
}
}
else
{
var maybeHash = TryHashFile(expandedPath);
if (!maybeHash.Succeeded)
{
return maybeHash.Failure;
}
contentHash = maybeHash.Result;
}
string localPath = GetLocalPath(contentHash);
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
EnsureLocal(contentHash);
EnsureRemote(contentHash);
return ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >(
() =>
{
var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode);
if (!putFile.Succeeded)
{
return putFile.Failure;
}
return contentHash;
},
ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex));
}
else
{
var result = ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >(
() =>
{
var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode);
if (!putFile.Succeeded)
{
return putFile.Failure;
}
return contentHash;
},
ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex));
m_content.Add(contentHash, new CacheEntry(CacheSites.Local));
EnsureRemote(contentHash);
m_pathRealizationModes[expandedPath] = fileRealizationMode;
return result;
}
}
}));
}
/// <inheritdoc />
public Task <Possible <Unit, Failure> > TryMaterializeAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, ContentHash contentHash, CancellationToken cancellationToken)
{
return(m_innerCache.TryMaterializeAsync(fileRealizationModes, path, contentHash, cancellationToken));
}
/// <inheritdoc />
public Task <Possible <ContentHash, Failure> > TryStoreAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, StoreArtifactOptions options)
{
return(m_innerCache.TryStoreAsync(fileRealizationModes, path, options));
}
