/// <summary>
/// Gets the existence of a path for the given file system view mode
/// </summary>
public Possible <PathExistence> GetExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default, bool cachePathExistence = true)
{
PathExistence existence;
if (TryGetKnownPathExistence(path, mode, out existence))
{
return(existence);
}
if (mode == FileSystemViewMode.Real)
{
// Compute and cache the real file system existence so subsequent calls
// do not have to query file system
if (cachePathExistence)
{
return(ComputeAndAddCacheRealFileSystemExistence(path, mode, isReadOnly));
}
var possibleExistence = FileUtilities.TryProbePathExistence(path.Expand(PathTable).ExpandedPath, followSymlink: false);
return(possibleExistence.Succeeded ? new Possible <PathExistence>(possibleExistence.Result) : new Possible <PathExistence>(possibleExistence.Failure));
}
else
{
// All graph filesystem path existences are statically known so just return NonExistent
// if not found in the existence cache or underlying graph filesystem view
return(PathExistence.Nonexistent);
}
}
public void VerifyGraphExistence(string fullPath, FileSystemViewMode mode, PathExistence existence)
{
Contract.Assert(mode != FileSystemViewMode.Real);
// Graph queries should never probe the file system
VerifyExistence(fullPath, mode, existence, shouldBeProbed: false);
var path = GetPath(fullPath);
if (existence == PathExistence.ExistsAsFile)
{
// File should be queried from underlying pip graph file system
Assert.True(TryGetLatestFileArtifactPaths.Contains(path) && m_latestWriteCountByPath.ContainsKey(path));
}
else
{
// Directory and non-existent paths will query underlying pip graph file system but will not be found
Assert.True(TryGetLatestFileArtifactPaths.Contains(path));
if (mode == FileSystemViewMode.FullGraph)
{
Assert.False(m_latestWriteCountByPath.ContainsKey(path));
}
else
{
Assert.True(!m_latestWriteCountByPath.ContainsKey(path) || m_latestWriteCountByPath[path] == 0);
}
}
}
public void VerifyExistence(string fullPath, FileSystemViewMode mode, PathExistence existence, bool shouldBeProbed)
{
ProbePaths.Clear();
var path = GetPath(fullPath);
var actualExistence = FileSystemView.GetExistence(path, mode);
Assert.Equal(existence, actualExistence.Result);
Assert.Equal(shouldBeProbed, ProbePaths.Contains(path));
}
public void ExpectEnumerationMembers(string path, FileSystemViewMode mode, params FileOrDirectoryArtifact[] expectedMembers)
{
Dictionary <AbsolutePath, PathExistence> expectations = expectedMembers.ToDictionary(a => a.Path, a => a.IsFile ? PathExistence.ExistsAsFile : PathExistence.ExistsAsDirectory);
AssertSuccess(
FileSystemView.TryEnumerateDirectory(
GetPath(path),
mode,
(memberName, memberPath, existence) =>
{
Assert.True(expectations.Remove(memberPath), "Path not found in expected member paths");
})
);
Assert.True(expectations.Count == 0, "Expected path not encountered during enumeration");
}
/// <summary>
/// Gets the existence of a path for the given file system view mode if set
/// </summary>
private bool TryGetKnownPathExistence(AbsolutePath path, FileSystemViewMode mode, out PathExistence existence)
{
if (mode == FileSystemViewMode.FullGraph || mode == FileSystemViewMode.Output)
{
if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
existence = PathExistence.ExistsAsFile;
return(true);
}
}
// For graph file systems, we query the path existence cache for existence of directories. Files are tracked
// by the pip graph. For dynamic files, we need to check PathExistenceCache.
existence = PathExistence.Nonexistent;
FileSystemEntry entry;
return(PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence));
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
/// <remarks>
/// If flags are provided, those are added to potentially existing flags
/// </remarks>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, out bool added, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
{
PathExistence result = existence;
var originalPath = path;
while (path.IsValid)
{
var getOrAddResult = PathExistenceCache.GetOrAdd(path, FileSystemEntry.Create(mode, existence).SetFlag(flags));
if (getOrAddResult.IsFound)
{
var currentExistence = getOrAddResult.Item.Value.GetExistence(mode);
var currentFlags = getOrAddResult.Item.Value.Flags;
if (currentExistence != null && currentFlags.HasFlag(flags))
{
if (originalPath != path)
{
// Parent entry with existence already exists, just return the result
added = true;
return(result);
}
else
{
// Entry with existence already exists, just return the value
added = false;
return(currentExistence.Value);
}
}
// found entry for the 'path', but it does not contain existence for the specified 'mode'
var updateResult = PathExistenceCache.AddOrUpdate(path, (mode, existence),
(key, data) => FileSystemEntry.Create(data.mode, data.existence).SetFlag(flags),
(key, data, oldValue) => oldValue.TryUpdate(data.mode, data.existence).SetFlag(flags));
// the same entry might be updated concurrently; check whether we lost the race, and if we did, stop further processing
currentExistence = updateResult.OldItem.Value.GetExistence(mode);
currentFlags = updateResult.OldItem.Value.Flags;
if (currentExistence != null && currentFlags.HasFlag(flags))
{
if (originalPath != path)
{
// Parent entry with existence already exists, just return the result
added = true;
return(result);
}
else
{
// Entry with existence already exists, just return the value
added = false;
return(currentExistence.Value);
}
}
}
// Only register parents as directories if path exists AND updateParents=true
if (existence == PathExistence.Nonexistent || !updateParents)
{
break;
}
// Set ancestor paths existence to directory existent for existent paths
existence = PathExistence.ExistsAsDirectory;
path = path.GetParent(PathTable);
}
added = true;
return(result);
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
{
return(GetOrAddExistence(path, mode, existence, out var added, updateParents, flags));
}
/// <summary>
/// Computes the existence of a path when not cached. For graph file systems the existence state of all paths
/// is statically known, so non-existent is returned here as not finding a cached path means it does not appear
/// in graph file system.
/// </summary>
private Possible <PathExistence> ComputeAndAddCacheRealFileSystemExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default)
{
Contract.Requires(mode == FileSystemViewMode.Real);
// Optimization. Check if the path can be determined to not exist based on a parent path without
// checking file system
if (m_inferNonExistenceBasedOnParentPathInRealFileSystem &&
TryInferNonExistenceBasedOnParentPathInRealFileSystem(path, out var trackedParentPath, out var intermediateParentPath) &&
TrackRealFileSystemAbsentChildPath(trackedParentPath, descendantPath: path))
{
return(PathExistence.Nonexistent);
}
// TODO: Some kind of strategy to trigger enumerating directories with commonly probed members
// TODO: Perhaps probabilistically enumerate based on hash of path and some counter
var possibleExistence = LocalDiskFileSystem.TryProbeAndTrackPathForExistence(path.Expand(PathTable), isReadOnly);
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemDiskProbes);
if (possibleExistence.Succeeded)
{
GetOrAddExistence(path, mode, possibleExistence.Result);
}
return(possibleExistence);
}
/// <summary>
/// Queries the existence and members of a directory in the specified file system mode
/// </summary>
public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
{
FileSystemEntry entry;
PathExistence existence;
if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
{
if (existence == PathExistence.Nonexistent)
{
return(existence);
}
if (existence == PathExistence.ExistsAsFile)
{
bool isDirectorySymlinkOrJunction = false;
if (entry.HasFlag(FileSystemEntryFlags.CheckedIsDirectorySymlink))
{
isDirectorySymlinkOrJunction = entry.HasFlag(FileSystemEntryFlags.IsDirectorySymlink);
}
else
{
isDirectorySymlinkOrJunction = FileUtilities.IsDirectorySymlinkOrJunction(path.ToString(PathTable));
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(
FileSystemEntryFlags.CheckedIsDirectorySymlink
| (isDirectorySymlinkOrJunction ? FileSystemEntryFlags.IsDirectorySymlink : FileSystemEntryFlags.None)));
}
if (!isDirectorySymlinkOrJunction)
{
return(existence);
}
}
// For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
// For real file system, this same is true if the directory has already been enumerated
if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
{
foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
{
var childPath = new AbsolutePath(childPathValue);
PathExistence childExistence;
if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
{
var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
handleEntry(entryName, childPath, childExistence);
}
}
return(existence);
}
}
if (mode == FileSystemViewMode.Real)
{
var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
{
// Reparse points are always treated as files. Otherwise, honor the directory attribute to determine
// existence
var childExistence = (entryAttributes & FileAttributes.ReparsePoint) != 0 ?
PathExistence.ExistsAsFile :
(entryAttributes & FileAttributes.Directory) != 0 ?
PathExistence.ExistsAsDirectory :
PathExistence.ExistsAsFile;
var childPath = path.Combine(PathTable, entryName);
childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);
// NOTE: Because we are caching file system state in memory, it is possible that the existence state of
// files does not match the state from the file system.
if (childExistence != PathExistence.Nonexistent)
{
handleEntry(entryName, childPath, childExistence);
}
});
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);
if (cachePathExistence)
{
Possible <PathExistence> possibleExistence;
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(
path,
handleDirectoryEntry,
// This method is called during observed input processing. Currently, we simply include all entries.
// TODO: In the future, we may want to restrict it based on pip's untracked scopes/paths.
shouldIncludeEntry: null /* include all entries */);
}
if (possibleExistence.Succeeded)
{
existence = GetOrAddExistence(path, mode, possibleExistence.Result);
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));
return(existence);
}
return(possibleExistence);
}
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
path.Expand(PathTable).ExpandedPath,
handleEntry: handleDirectoryEntry);
return(possibleFingerprintResult.Succeeded
? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
: new Possible <PathExistence>(possibleFingerprintResult.Failure));
}
}
else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
return(PathExistence.ExistsAsFile);
}
return(PathExistence.Nonexistent);
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true)
{
return(GetOrAddExistence(path, mode, existence, out var added, updateParents));
}
/// <summary>
/// Queries the existence and members of a directory in the specified file system mode
/// </summary>
public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
{
FileSystemEntry entry;
PathExistence existence;
if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
{
if (existence != PathExistence.ExistsAsDirectory)
{
return(existence);
}
// For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
// For real file system, this same is true if the directory has already been enumerated
if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
{
foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
{
var childPath = new AbsolutePath(childPathValue);
PathExistence childExistence;
if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
{
var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
handleEntry(entryName, childPath, childExistence);
}
}
return(existence);
}
}
if (mode == FileSystemViewMode.Real)
{
var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
{
var childExistence = (entryAttributes & FileAttributes.Directory) != 0 ? PathExistence.ExistsAsDirectory : PathExistence.ExistsAsFile;
var childPath = path.Combine(PathTable, entryName);
childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);
// NOTE: Because we are caching file system state in memory, it is possible that the existence state of
// files does not match the state from the file system.
if (childExistence != PathExistence.Nonexistent)
{
handleEntry(entryName, childPath, childExistence);
}
});
Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);
if (cachePathExistence)
{
Possible <PathExistence> possibleExistence;
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(path, handleDirectoryEntry);
}
if (possibleExistence.Succeeded)
{
existence = GetOrAddExistence(path, mode, possibleExistence.Result);
PathExistenceCache.AddOrUpdate(path, false,
(key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
(key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));
return(existence);
}
return(possibleExistence);
}
using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
{
var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
path.Expand(PathTable).ExpandedPath,
handleEntry: handleDirectoryEntry);
return(possibleFingerprintResult.Succeeded
? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
: new Possible <PathExistence>(possibleFingerprintResult.Failure));
}
}
else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
return(PathExistence.ExistsAsFile);
}
return(PathExistence.Nonexistent);
}
