/// <summary> /// Gets the existence of a path for the given file system view mode /// </summary> public Possible <PathExistence> GetExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default, bool cachePathExistence = true) { PathExistence existence; if (TryGetKnownPathExistence(path, mode, out existence)) { return(existence); } if (mode == FileSystemViewMode.Real) { // Compute and cache the real file system existence so subsequent calls // do not have to query file system if (cachePathExistence) { return(ComputeAndAddCacheRealFileSystemExistence(path, mode, isReadOnly)); } var possibleExistence = FileUtilities.TryProbePathExistence(path.Expand(PathTable).ExpandedPath, followSymlink: false); return(possibleExistence.Succeeded ? new Possible <PathExistence>(possibleExistence.Result) : new Possible <PathExistence>(possibleExistence.Failure)); } else { // All graph filesystem path existences are statically known so just return NonExistent // if not found in the existence cache or underlying graph filesystem view return(PathExistence.Nonexistent); } }
public void VerifyGraphExistence(string fullPath, FileSystemViewMode mode, PathExistence existence) { Contract.Assert(mode != FileSystemViewMode.Real); // Graph queries should never probe the file system VerifyExistence(fullPath, mode, existence, shouldBeProbed: false); var path = GetPath(fullPath); if (existence == PathExistence.ExistsAsFile) { // File should be queried from underlying pip graph file system Assert.True(TryGetLatestFileArtifactPaths.Contains(path) && m_latestWriteCountByPath.ContainsKey(path)); } else { // Directory and non-existent paths will query underlying pip graph file system but will not be found Assert.True(TryGetLatestFileArtifactPaths.Contains(path)); if (mode == FileSystemViewMode.FullGraph) { Assert.False(m_latestWriteCountByPath.ContainsKey(path)); } else { Assert.True(!m_latestWriteCountByPath.ContainsKey(path) || m_latestWriteCountByPath[path] == 0); } } }
public void VerifyExistence(string fullPath, FileSystemViewMode mode, PathExistence existence, bool shouldBeProbed) { ProbePaths.Clear(); var path = GetPath(fullPath); var actualExistence = FileSystemView.GetExistence(path, mode); Assert.Equal(existence, actualExistence.Result); Assert.Equal(shouldBeProbed, ProbePaths.Contains(path)); }
public void ExpectEnumerationMembers(string path, FileSystemViewMode mode, params FileOrDirectoryArtifact[] expectedMembers) { Dictionary <AbsolutePath, PathExistence> expectations = expectedMembers.ToDictionary(a => a.Path, a => a.IsFile ? PathExistence.ExistsAsFile : PathExistence.ExistsAsDirectory); AssertSuccess( FileSystemView.TryEnumerateDirectory( GetPath(path), mode, (memberName, memberPath, existence) => { Assert.True(expectations.Remove(memberPath), "Path not found in expected member paths"); }) ); Assert.True(expectations.Count == 0, "Expected path not encountered during enumeration"); }
/// <summary> /// Gets the existence of a path for the given file system view mode if set /// </summary> private bool TryGetKnownPathExistence(AbsolutePath path, FileSystemViewMode mode, out PathExistence existence) { if (mode == FileSystemViewMode.FullGraph || mode == FileSystemViewMode.Output) { if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode)) { existence = PathExistence.ExistsAsFile; return(true); } } // For graph file systems, we query the path existence cache for existence of directories. Files are tracked // by the pip graph. For dynamic files, we need to check PathExistenceCache. existence = PathExistence.Nonexistent; FileSystemEntry entry; return(PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence)); }
/// <summary> /// Adds the existence state of the path if not present or returns the current cached existence state of the path /// </summary> /// <remarks> /// If flags are provided, those are added to potentially existing flags /// </remarks> private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, out bool added, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None) { PathExistence result = existence; var originalPath = path; while (path.IsValid) { var getOrAddResult = PathExistenceCache.GetOrAdd(path, FileSystemEntry.Create(mode, existence).SetFlag(flags)); if (getOrAddResult.IsFound) { var currentExistence = getOrAddResult.Item.Value.GetExistence(mode); var currentFlags = getOrAddResult.Item.Value.Flags; if (currentExistence != null && currentFlags.HasFlag(flags)) { if (originalPath != path) { // Parent entry with existence already exists, just return the result added = true; return(result); } else { // Entry with existence already exists, just return the value added = false; return(currentExistence.Value); } } // found entry for the 'path', but it does not contain existence for the specified 'mode' var updateResult = PathExistenceCache.AddOrUpdate(path, (mode, existence), (key, data) => FileSystemEntry.Create(data.mode, data.existence).SetFlag(flags), (key, data, oldValue) => oldValue.TryUpdate(data.mode, data.existence).SetFlag(flags)); // the same entry might be updated concurrently; check whether we lost the race, and if we did, stop further processing currentExistence = updateResult.OldItem.Value.GetExistence(mode); currentFlags = updateResult.OldItem.Value.Flags; if (currentExistence != null && currentFlags.HasFlag(flags)) { if (originalPath != path) { // Parent entry with existence already exists, just return the result added = true; return(result); } else { // Entry with existence already exists, just return the value added = false; return(currentExistence.Value); } } } // Only register parents as directories if path exists AND updateParents=true if (existence == PathExistence.Nonexistent || !updateParents) { break; } // Set ancestor paths existence to directory existent for existent paths existence = PathExistence.ExistsAsDirectory; path = path.GetParent(PathTable); } added = true; return(result); }
/// <summary> /// Adds the existence state of the path if not present or returns the current cached existence state of the path /// </summary> private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None) { return(GetOrAddExistence(path, mode, existence, out var added, updateParents, flags)); }
/// <summary> /// Computes the existence of a path when not cached. For graph file systems the existence state of all paths /// is statically known, so non-existent is returned here as not finding a cached path means it does not appear /// in graph file system. /// </summary> private Possible <PathExistence> ComputeAndAddCacheRealFileSystemExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default) { Contract.Requires(mode == FileSystemViewMode.Real); // Optimization. Check if the path can be determined to not exist based on a parent path without // checking file system if (m_inferNonExistenceBasedOnParentPathInRealFileSystem && TryInferNonExistenceBasedOnParentPathInRealFileSystem(path, out var trackedParentPath, out var intermediateParentPath) && TrackRealFileSystemAbsentChildPath(trackedParentPath, descendantPath: path)) { return(PathExistence.Nonexistent); } // TODO: Some kind of strategy to trigger enumerating directories with commonly probed members // TODO: Perhaps probabilistically enumerate based on hash of path and some counter var possibleExistence = LocalDiskFileSystem.TryProbeAndTrackPathForExistence(path.Expand(PathTable), isReadOnly); Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemDiskProbes); if (possibleExistence.Succeeded) { GetOrAddExistence(path, mode, possibleExistence.Result); } return(possibleExistence); }
/// <summary> /// Queries the existence and members of a directory in the specified file system mode /// </summary> public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true) { FileSystemEntry entry; PathExistence existence; if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence)) { if (existence == PathExistence.Nonexistent) { return(existence); } if (existence == PathExistence.ExistsAsFile) { bool isDirectorySymlinkOrJunction = false; if (entry.HasFlag(FileSystemEntryFlags.CheckedIsDirectorySymlink)) { isDirectorySymlinkOrJunction = entry.HasFlag(FileSystemEntryFlags.IsDirectorySymlink); } else { isDirectorySymlinkOrJunction = FileUtilities.IsDirectorySymlinkOrJunction(path.ToString(PathTable)); PathExistenceCache.AddOrUpdate(path, false, (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); }, (key, data, oldValue) => oldValue.SetFlag( FileSystemEntryFlags.CheckedIsDirectorySymlink | (isDirectorySymlinkOrJunction ? FileSystemEntryFlags.IsDirectorySymlink : FileSystemEntryFlags.None))); } if (!isDirectorySymlinkOrJunction) { return(existence); } } // For graph file systems, directory members can be determined by overlaying path table with existence state in-memory // For real file system, this same is true if the directory has already been enumerated if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0)) { foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value)) { var childPath = new AbsolutePath(childPathValue); PathExistence childExistence; if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent) { var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable); handleEntry(entryName, childPath, childExistence); } } return(existence); } } if (mode == FileSystemViewMode.Real) { var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) => { // Reparse points are always treated as files. Otherwise, honor the directory attribute to determine // existence var childExistence = (entryAttributes & FileAttributes.ReparsePoint) != 0 ? PathExistence.ExistsAsFile : (entryAttributes & FileAttributes.Directory) != 0 ? PathExistence.ExistsAsDirectory : PathExistence.ExistsAsFile; var childPath = path.Combine(PathTable, entryName); childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false); // NOTE: Because we are caching file system state in memory, it is possible that the existence state of // files does not match the state from the file system. if (childExistence != PathExistence.Nonexistent) { handleEntry(entryName, childPath, childExistence); } }); Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations); if (cachePathExistence) { Possible <PathExistence> possibleExistence; using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration)) { possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership( path, handleDirectoryEntry, // This method is called during observed input processing. Currently, we simply include all entries. // TODO: In the future, we may want to restrict it based on pip's untracked scopes/paths. shouldIncludeEntry: null /* include all entries */); } if (possibleExistence.Succeeded) { existence = GetOrAddExistence(path, mode, possibleExistence.Result); PathExistenceCache.AddOrUpdate(path, false, (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); }, (key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated)); return(existence); } return(possibleExistence); } using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration)) { var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint( path.Expand(PathTable).ExpandedPath, handleEntry: handleDirectoryEntry); return(possibleFingerprintResult.Succeeded ? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence) : new Possible <PathExistence>(possibleFingerprintResult.Failure)); } } else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode)) { return(PathExistence.ExistsAsFile); } return(PathExistence.Nonexistent); }
/// <summary> /// Adds the existence state of the path if not present or returns the current cached existence state of the path /// </summary> private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true) { return(GetOrAddExistence(path, mode, existence, out var added, updateParents)); }
/// <summary> /// Queries the existence and members of a directory in the specified file system mode /// </summary> public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true) { FileSystemEntry entry; PathExistence existence; if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence)) { if (existence != PathExistence.ExistsAsDirectory) { return(existence); } // For graph file systems, directory members can be determined by overlaying path table with existence state in-memory // For real file system, this same is true if the directory has already been enumerated if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0)) { foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value)) { var childPath = new AbsolutePath(childPathValue); PathExistence childExistence; if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent) { var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable); handleEntry(entryName, childPath, childExistence); } } return(existence); } } if (mode == FileSystemViewMode.Real) { var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) => { var childExistence = (entryAttributes & FileAttributes.Directory) != 0 ? PathExistence.ExistsAsDirectory : PathExistence.ExistsAsFile; var childPath = path.Combine(PathTable, entryName); childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false); // NOTE: Because we are caching file system state in memory, it is possible that the existence state of // files does not match the state from the file system. if (childExistence != PathExistence.Nonexistent) { handleEntry(entryName, childPath, childExistence); } }); Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations); if (cachePathExistence) { Possible <PathExistence> possibleExistence; using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration)) { possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(path, handleDirectoryEntry); } if (possibleExistence.Succeeded) { existence = GetOrAddExistence(path, mode, possibleExistence.Result); PathExistenceCache.AddOrUpdate(path, false, (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); }, (key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated)); return(existence); } return(possibleExistence); } using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration)) { var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint( path.Expand(PathTable).ExpandedPath, handleEntry: handleDirectoryEntry); return(possibleFingerprintResult.Succeeded ? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence) : new Possible <PathExistence>(possibleFingerprintResult.Failure)); } } else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode)) { return(PathExistence.ExistsAsFile); } return(PathExistence.Nonexistent); }