/// <summary>
/// Gets the existence of a path for the given file system view mode
/// </summary>
public Possible <PathExistence> GetExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default, bool cachePathExistence = true)
{
PathExistence existence;
if (TryGetKnownPathExistence(path, mode, out existence))
{
return(existence);
}
if (mode == FileSystemViewMode.Real)
{
// Compute and cache the real file system existence so subsequent calls
// do not have to query file system
if (cachePathExistence)
{
return(ComputeAndAddCacheRealFileSystemExistence(path, mode, isReadOnly));
}
var possibleExistence = FileUtilities.TryProbePathExistence(path.Expand(PathTable).ExpandedPath, followSymlink: false);
return(possibleExistence.Succeeded ? new Possible <PathExistence>(possibleExistence.Result) : new Possible <PathExistence>(possibleExistence.Failure));
}
else
{
// All graph filesystem path existences are statically known so just return NonExistent
// if not found in the existence cache or underlying graph filesystem view
return(PathExistence.Nonexistent);
}
}
private void CreateMappingForOutputs(Process process, out MultiValueDictionary <AbsolutePath, ExpandedAbsolutePath> originalDirectories, out MultiValueDictionary <ExpandedAbsolutePath, ExpandedAbsolutePath> redirectedDirectories)
{
// Collect all predicted outputs (directories and files) for the given process
var directories = CollectAllOutputDirectories(m_pathTable, process);
// In order to keep the filter configuration to its minimum, let's remove directories that are nested within each other
var dedupDirectories = AbsolutePathUtilities.CollapseDirectories(directories, m_pathTable, out var originalToCollapsedMapping);
var stringTable = m_pathTable.StringTable;
var reserveFoldersResolver = new ReserveFoldersResolver(new object());
originalDirectories = new MultiValueDictionary <AbsolutePath, ExpandedAbsolutePath>(originalToCollapsedMapping.Count);
redirectedDirectories = new MultiValueDictionary <ExpandedAbsolutePath, ExpandedAbsolutePath>(dedupDirectories.Count, ExpandedAbsolutePathEqualityComparer.Instance);
// Map from original dedup directories to unique redirected directories
var uniqueRedirectedDirectories = new Dictionary <AbsolutePath, ExpandedAbsolutePath>(dedupDirectories.Count);
foreach (var kvp in originalToCollapsedMapping)
{
AbsolutePath originalDirectory = kvp.Key;
AbsolutePath originalCollapsedDirectory = kvp.Value;
if (!uniqueRedirectedDirectories.TryGetValue(originalCollapsedDirectory, out var uniqueRedirectedDirectory))
{
uniqueRedirectedDirectory = GetUniqueRedirectedDirectory(process, ref reserveFoldersResolver, originalCollapsedDirectory).Expand(m_pathTable);
uniqueRedirectedDirectories.Add(originalCollapsedDirectory, uniqueRedirectedDirectory);
redirectedDirectories.Add(uniqueRedirectedDirectory, originalCollapsedDirectory.Expand(m_pathTable));
}
// Let's reconstruct the redirected directory
var redirectedDirectory = originalDirectory.Relocate(m_pathTable, originalCollapsedDirectory, uniqueRedirectedDirectory.Path);
originalDirectories.Add(originalDirectory, redirectedDirectory.Expand(m_pathTable));
}
}
private MergeResult HardlinkOpaqueDirectories(
Process process,
ContainerConfiguration containerConfiguration,
PipExecutionContext pipExecutionContext,
IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites,
HashSet <AbsolutePath> createdDirectories)
{
bool isolateSharedOpaques = process.ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories();
bool isolateExclusiveOpaques = process.ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories();
// Shortcut the iteration of output directories are not isolated at all
if (!isolateExclusiveOpaques && !isolateSharedOpaques)
{
return(MergeResult.Success);
}
foreach (DirectoryArtifact directoryOutput in process.DirectoryOutputs)
{
if (directoryOutput.IsSharedOpaque && isolateSharedOpaques)
{
AbsolutePath redirectedDirectory = GetRedirectedDirectoryForOutputContainer(containerConfiguration, directoryOutput.Path).Path;
// Here we don't need to check for WCI reparse points. We know those outputs are there based on what detours is saying.
var sharedOpaqueContent = sharedDynamicWrites[directoryOutput.Path];
foreach (AbsolutePath sharedOpaqueFile in sharedOpaqueContent)
{
string sourcePath = sharedOpaqueFile.Relocate(m_pathTable, directoryOutput.Path, redirectedDirectory).ToString(m_pathTable);
// The file may not exist because the pip could have created it but later deleted it
if (!FileUtilities.Exists(sourcePath))
{
continue;
}
ExpandedAbsolutePath destinationPath = sharedOpaqueFile.Expand(m_pathTable);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(destinationPath, rewriteCount: 1, sourcePath, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
else if (!directoryOutput.IsSharedOpaque && isolateExclusiveOpaques)
{
// We need to enumerate to discover the content of an exclusive opaque, and also skip the potential reparse points
// TODO: Enumeration will happen again when the file content manager tries to discover the content of the exclusive opaque. Consider doing this only once instead.
// An output directory should only have one redirected path
ExpandedAbsolutePath redirectedDirectory = containerConfiguration.OriginalDirectories[directoryOutput.Path].Single();
foreach (string exclusiveOpaqueFile in Directory.EnumerateFiles(redirectedDirectory.ExpandedPath, "*", SearchOption.AllDirectories))
{
if (FileUtilities.IsWciReparsePoint(exclusiveOpaqueFile))
{
continue;
}
AbsolutePath exclusiveOpaqueFilePath = AbsolutePath.Create(m_pathTable, exclusiveOpaqueFile);
AbsolutePath outputFile = exclusiveOpaqueFilePath.Relocate(m_pathTable, redirectedDirectory.Path, directoryOutput.Path);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(outputFile.Expand(m_pathTable), rewriteCount: 1, exclusiveOpaqueFile, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
}
return(MergeResult.Success);
}
/// <summary>
/// Computes the existence of a path when not cached. For graph file systems the existence state of all paths
/// is statically known, so non-existent is returned here as not finding a cached path means it does not appear
/// in graph file system.
/// </summary>
private Possible <PathExistence> ComputeAndAddCacheRealFileSystemExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default)
{
Contract.Requires(mode == FileSystemViewMode.Real);
// Optimization. Check if the path can be determined to not exist based on a parent path without
// checking file system
if (m_inferNonExistenceBasedOnParentPathInRealFileSystem &&
TryInferNonExistenceBasedOnParentPathInRealFileSystem(path, out var trackedParentPath, out var intermediateParentPath) &&
TrackRealFileSystemAbsentChildPath(trackedParentPath, descendantPath: path))
{
return(PathExistence.Nonexistent);
}
// TODO: Some kind of strategy to trigger enumerating directories with commonly probed members
// TODO: Perhaps probabilistically enumerate based on hash of path and some counter
var possibleExistence = LocalDiskFileSystem.TryProbeAndTrackPathForExistence(path.Expand(PathTable), isReadOnly);
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemDiskProbes);
if (possibleExistence.Succeeded)
{
GetOrAddExistence(path, mode, possibleExistence.Result);
}
return(possibleExistence);
}
/// <summary>
/// Queries the existence and members of a directory in the specified file system mode
/// </summary>
public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
{
FileSystemEntry entry;
PathExistence existence;
if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
{
if (existence == PathExistence.Nonexistent)
{
return(existence);
}
if (existence == PathExistence.ExistsAsFile)
{
bool isDirectorySymlinkOrJunction = false;
if (entry.HasFlag(FileSystemEntryFlags.CheckedIsDirectorySymlink))
{
isDirectorySymlinkOrJunction = entry.HasFlag(FileSystemEntryFlags.IsDirectorySymlink);
}
else
{
isDirectorySymlinkOrJunction = FileUtilities.IsDirectorySymlinkOrJunction(path.ToString(PathTable));
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(
FileSystemEntryFlags.CheckedIsDirectorySymlink
| (isDirectorySymlinkOrJunction ? FileSystemEntryFlags.IsDirectorySymlink : FileSystemEntryFlags.None)));
}
if (!isDirectorySymlinkOrJunction)
{
return(existence);
}
}
// For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
// For real file system, this same is true if the directory has already been enumerated
if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
{
foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
{
var childPath = new AbsolutePath(childPathValue);
PathExistence childExistence;
if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
{
var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
handleEntry(entryName, childPath, childExistence);
}
}
return(existence);
}
}
if (mode == FileSystemViewMode.Real)
{
var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
{
// Reparse points are always treated as files. Otherwise, honor the directory attribute to determine
// existence
var childExistence = (entryAttributes & FileAttributes.ReparsePoint) != 0 ?
PathExistence.ExistsAsFile :
(entryAttributes & FileAttributes.Directory) != 0 ?
PathExistence.ExistsAsDirectory :
PathExistence.ExistsAsFile;
var childPath = path.Combine(PathTable, entryName);
childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);
// NOTE: Because we are caching file system state in memory, it is possible that the existence state of
// files does not match the state from the file system.
if (childExistence != PathExistence.Nonexistent)
{
handleEntry(entryName, childPath, childExistence);
}
});
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);
if (cachePathExistence)
{
Possible <PathExistence> possibleExistence;
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(
path,
handleDirectoryEntry,
// This method is called during observed input processing. Currently, we simply include all entries.
// TODO: In the future, we may want to restrict it based on pip's untracked scopes/paths.
shouldIncludeEntry: null /* include all entries */);
}
if (possibleExistence.Succeeded)
{
existence = GetOrAddExistence(path, mode, possibleExistence.Result);
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));
return(existence);
}
return(possibleExistence);
}
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
path.Expand(PathTable).ExpandedPath,
handleEntry: handleDirectoryEntry);
return(possibleFingerprintResult.Succeeded
? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
: new Possible <PathExistence>(possibleFingerprintResult.Failure));
}
}
else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
return(PathExistence.ExistsAsFile);
}
return(PathExistence.Nonexistent);
}
/// <summary>
/// Queries the existence and members of a directory in the specified file system mode
/// </summary>
public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
{
FileSystemEntry entry;
PathExistence existence;
if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
{
if (existence != PathExistence.ExistsAsDirectory)
{
return(existence);
}
// For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
// For real file system, this same is true if the directory has already been enumerated
if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
{
foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
{
var childPath = new AbsolutePath(childPathValue);
PathExistence childExistence;
if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
{
var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
handleEntry(entryName, childPath, childExistence);
}
}
return(existence);
}
}
if (mode == FileSystemViewMode.Real)
{
var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
{
var childExistence = (entryAttributes & FileAttributes.Directory) != 0 ? PathExistence.ExistsAsDirectory : PathExistence.ExistsAsFile;
var childPath = path.Combine(PathTable, entryName);
childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);
// NOTE: Because we are caching file system state in memory, it is possible that the existence state of
// files does not match the state from the file system.
if (childExistence != PathExistence.Nonexistent)
{
handleEntry(entryName, childPath, childExistence);
}
});
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);
if (cachePathExistence)
{
Possible <PathExistence> possibleExistence;
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(path, handleDirectoryEntry);
}
if (possibleExistence.Succeeded)
{
existence = GetOrAddExistence(path, mode, possibleExistence.Result);
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));
return(existence);
}
return(possibleExistence);
}
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
path.Expand(PathTable).ExpandedPath,
handleEntry: handleDirectoryEntry);
return(possibleFingerprintResult.Succeeded
? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
: new Possible <PathExistence>(possibleFingerprintResult.Failure));
}
}
else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
return(PathExistence.ExistsAsFile);
}
return(PathExistence.Nonexistent);
}