예제 #1
0
        /// <summary>
        /// Gets the existence of a path for the given file system view mode
        /// </summary>
        public Possible <PathExistence> GetExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default, bool cachePathExistence = true)
        {
            PathExistence existence;

            if (TryGetKnownPathExistence(path, mode, out existence))
            {
                return(existence);
            }

            if (mode == FileSystemViewMode.Real)
            {
                // Compute and cache the real file system existence so subsequent calls
                // do not have to query file system
                if (cachePathExistence)
                {
                    return(ComputeAndAddCacheRealFileSystemExistence(path, mode, isReadOnly));
                }

                var possibleExistence = FileUtilities.TryProbePathExistence(path.Expand(PathTable).ExpandedPath, followSymlink: false);
                return(possibleExistence.Succeeded ? new Possible <PathExistence>(possibleExistence.Result) : new Possible <PathExistence>(possibleExistence.Failure));
            }
            else
            {
                // All graph filesystem path existences are statically known so just return NonExistent
                // if not found in the existence cache or underlying graph filesystem view
                return(PathExistence.Nonexistent);
            }
        }
예제 #2
0
            public void VerifyGraphExistence(string fullPath, FileSystemViewMode mode, PathExistence existence)
            {
                Contract.Assert(mode != FileSystemViewMode.Real);

                // Graph queries should never probe the file system
                VerifyExistence(fullPath, mode, existence, shouldBeProbed: false);

                var path = GetPath(fullPath);

                if (existence == PathExistence.ExistsAsFile)
                {
                    // File should be queried from underlying pip graph file system
                    Assert.True(TryGetLatestFileArtifactPaths.Contains(path) && m_latestWriteCountByPath.ContainsKey(path));
                }
                else
                {
                    // Directory and non-existent paths will query underlying pip graph file system but will not be found
                    Assert.True(TryGetLatestFileArtifactPaths.Contains(path));
                    if (mode == FileSystemViewMode.FullGraph)
                    {
                        Assert.False(m_latestWriteCountByPath.ContainsKey(path));
                    }
                    else
                    {
                        Assert.True(!m_latestWriteCountByPath.ContainsKey(path) || m_latestWriteCountByPath[path] == 0);
                    }
                }
            }
예제 #3
0
            public void VerifyExistence(string fullPath, FileSystemViewMode mode, PathExistence existence, bool shouldBeProbed)
            {
                ProbePaths.Clear();
                var path            = GetPath(fullPath);
                var actualExistence = FileSystemView.GetExistence(path, mode);

                Assert.Equal(existence, actualExistence.Result);
                Assert.Equal(shouldBeProbed, ProbePaths.Contains(path));
            }
예제 #4
0
            public void ExpectEnumerationMembers(string path, FileSystemViewMode mode, params FileOrDirectoryArtifact[] expectedMembers)
            {
                Dictionary <AbsolutePath, PathExistence> expectations = expectedMembers.ToDictionary(a => a.Path, a => a.IsFile ? PathExistence.ExistsAsFile : PathExistence.ExistsAsDirectory);

                AssertSuccess(
                    FileSystemView.TryEnumerateDirectory(
                        GetPath(path),
                        mode,
                        (memberName, memberPath, existence) =>
                {
                    Assert.True(expectations.Remove(memberPath), "Path not found in expected member paths");
                })
                    );

                Assert.True(expectations.Count == 0, "Expected path not encountered during enumeration");
            }
예제 #5
0
        /// <summary>
        /// Gets the existence of a path for the given file system view mode if set
        /// </summary>
        private bool TryGetKnownPathExistence(AbsolutePath path, FileSystemViewMode mode, out PathExistence existence)
        {
            if (mode == FileSystemViewMode.FullGraph || mode == FileSystemViewMode.Output)
            {
                if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
                {
                    existence = PathExistence.ExistsAsFile;
                    return(true);
                }
            }

            // For graph file systems, we query the path existence cache for existence of directories. Files are tracked
            // by the pip graph. For dynamic files, we need to check PathExistenceCache.
            existence = PathExistence.Nonexistent;
            FileSystemEntry entry;

            return(PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence));
        }
예제 #6
0
        /// <summary>
        /// Adds the existence state of the path if not present or returns the current cached existence state of the path
        /// </summary>
        /// <remarks>
        /// If flags are provided, those are added to potentially existing flags
        /// </remarks>
        private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, out bool added, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
        {
            PathExistence result       = existence;
            var           originalPath = path;

            while (path.IsValid)
            {
                var getOrAddResult = PathExistenceCache.GetOrAdd(path, FileSystemEntry.Create(mode, existence).SetFlag(flags));
                if (getOrAddResult.IsFound)
                {
                    var currentExistence = getOrAddResult.Item.Value.GetExistence(mode);
                    var currentFlags     = getOrAddResult.Item.Value.Flags;
                    if (currentExistence != null && currentFlags.HasFlag(flags))
                    {
                        if (originalPath != path)
                        {
                            // Parent entry with existence already exists, just return the result
                            added = true;
                            return(result);
                        }
                        else
                        {
                            // Entry with existence already exists, just return the value
                            added = false;
                            return(currentExistence.Value);
                        }
                    }

                    // found entry for the 'path', but it does not contain existence for the specified 'mode'
                    var updateResult = PathExistenceCache.AddOrUpdate(path, (mode, existence),
                                                                      (key, data) => FileSystemEntry.Create(data.mode, data.existence).SetFlag(flags),
                                                                      (key, data, oldValue) => oldValue.TryUpdate(data.mode, data.existence).SetFlag(flags));

                    // the same entry might be updated concurrently; check whether we lost the race, and if we did, stop further processing
                    currentExistence = updateResult.OldItem.Value.GetExistence(mode);
                    currentFlags     = updateResult.OldItem.Value.Flags;
                    if (currentExistence != null && currentFlags.HasFlag(flags))
                    {
                        if (originalPath != path)
                        {
                            // Parent entry with existence already exists, just return the result
                            added = true;
                            return(result);
                        }
                        else
                        {
                            // Entry with existence already exists, just return the value
                            added = false;
                            return(currentExistence.Value);
                        }
                    }
                }

                // Only register parents as directories if path exists AND updateParents=true
                if (existence == PathExistence.Nonexistent || !updateParents)
                {
                    break;
                }

                // Set ancestor paths existence to directory existent for existent paths
                existence = PathExistence.ExistsAsDirectory;
                path      = path.GetParent(PathTable);
            }

            added = true;
            return(result);
        }
예제 #7
0
 /// <summary>
 /// Adds the existence state of the path if not present or returns the current cached existence state of the path
 /// </summary>
 private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
 {
     return(GetOrAddExistence(path, mode, existence, out var added, updateParents, flags));
 }
예제 #8
0
        /// <summary>
        /// Computes the existence of a path when not cached. For graph file systems the existence state of all paths
        /// is statically known, so non-existent is returned here as not finding a cached path means it does not appear
        /// in graph file system.
        /// </summary>
        private Possible <PathExistence> ComputeAndAddCacheRealFileSystemExistence(AbsolutePath path, FileSystemViewMode mode, bool?isReadOnly = default)
        {
            Contract.Requires(mode == FileSystemViewMode.Real);

            // Optimization. Check if the path can be determined to not exist based on a parent path without
            // checking file system
            if (m_inferNonExistenceBasedOnParentPathInRealFileSystem &&
                TryInferNonExistenceBasedOnParentPathInRealFileSystem(path, out var trackedParentPath, out var intermediateParentPath) &&
                TrackRealFileSystemAbsentChildPath(trackedParentPath, descendantPath: path))
            {
                return(PathExistence.Nonexistent);
            }

            // TODO: Some kind of strategy to trigger enumerating directories with commonly probed members
            // TODO: Perhaps probabilistically enumerate based on hash of path and some counter
            var possibleExistence = LocalDiskFileSystem.TryProbeAndTrackPathForExistence(path.Expand(PathTable), isReadOnly);

            Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemDiskProbes);
            if (possibleExistence.Succeeded)
            {
                GetOrAddExistence(path, mode, possibleExistence.Result);
            }

            return(possibleExistence);
        }
예제 #9
0
        /// <summary>
        /// Queries the existence and members of a directory in the specified file system mode
        /// </summary>
        public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
        {
            FileSystemEntry entry;
            PathExistence   existence;

            if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
            {
                if (existence == PathExistence.Nonexistent)
                {
                    return(existence);
                }

                if (existence == PathExistence.ExistsAsFile)
                {
                    bool isDirectorySymlinkOrJunction = false;

                    if (entry.HasFlag(FileSystemEntryFlags.CheckedIsDirectorySymlink))
                    {
                        isDirectorySymlinkOrJunction = entry.HasFlag(FileSystemEntryFlags.IsDirectorySymlink);
                    }
                    else
                    {
                        isDirectorySymlinkOrJunction = FileUtilities.IsDirectorySymlinkOrJunction(path.ToString(PathTable));
                        PathExistenceCache.AddOrUpdate(path, false,
                                                       (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
                                                       (key, data, oldValue) => oldValue.SetFlag(
                                                           FileSystemEntryFlags.CheckedIsDirectorySymlink
                                                           | (isDirectorySymlinkOrJunction ? FileSystemEntryFlags.IsDirectorySymlink : FileSystemEntryFlags.None)));
                    }

                    if (!isDirectorySymlinkOrJunction)
                    {
                        return(existence);
                    }
                }

                // For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
                // For real file system, this same is true if the directory has already been enumerated
                if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
                {
                    foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
                    {
                        var childPath = new AbsolutePath(childPathValue);

                        PathExistence childExistence;
                        if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
                        {
                            var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
                            handleEntry(entryName, childPath, childExistence);
                        }
                    }

                    return(existence);
                }
            }

            if (mode == FileSystemViewMode.Real)
            {
                var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
                {
                    // Reparse points are always treated as files. Otherwise, honor the directory attribute to determine
                    // existence
                    var childExistence = (entryAttributes & FileAttributes.ReparsePoint) != 0 ?
                                         PathExistence.ExistsAsFile :
                                         (entryAttributes & FileAttributes.Directory) != 0 ?
                                         PathExistence.ExistsAsDirectory :
                                         PathExistence.ExistsAsFile;

                    var childPath = path.Combine(PathTable, entryName);

                    childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);

                    // NOTE: Because we are caching file system state in memory, it is possible that the existence state of
                    // files does not match the state from the file system.
                    if (childExistence != PathExistence.Nonexistent)
                    {
                        handleEntry(entryName, childPath, childExistence);
                    }
                });

                Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);

                if (cachePathExistence)
                {
                    Possible <PathExistence> possibleExistence;
                    using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
                    {
                        possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(
                            path,
                            handleDirectoryEntry,
                            // This method is called during observed input processing. Currently, we simply include all entries.
                            // TODO: In the future, we may want to restrict it based on pip's untracked scopes/paths.
                            shouldIncludeEntry: null /* include all entries */);
                    }

                    if (possibleExistence.Succeeded)
                    {
                        existence = GetOrAddExistence(path, mode, possibleExistence.Result);

                        PathExistenceCache.AddOrUpdate(path, false,
                                                       (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
                                                       (key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));

                        return(existence);
                    }

                    return(possibleExistence);
                }

                using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
                {
                    var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
                        path.Expand(PathTable).ExpandedPath,
                        handleEntry: handleDirectoryEntry);

                    return(possibleFingerprintResult.Succeeded
                        ? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
                        : new Possible <PathExistence>(possibleFingerprintResult.Failure));
                }
            }
            else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
            {
                return(PathExistence.ExistsAsFile);
            }

            return(PathExistence.Nonexistent);
        }
예제 #10
0
 /// <summary>
 /// Adds the existence state of the path if not present or returns the current cached existence state of the path
 /// </summary>
 private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true)
 {
     return(GetOrAddExistence(path, mode, existence, out var added, updateParents));
 }
예제 #11
0
        /// <summary>
        /// Queries the existence and members of a directory in the specified file system mode
        /// </summary>
        public Possible <PathExistence> TryEnumerateDirectory(AbsolutePath path, FileSystemViewMode mode, Action <string, AbsolutePath, PathExistence> handleEntry, bool cachePathExistence = true)
        {
            FileSystemEntry entry;
            PathExistence   existence;

            if (PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence))
            {
                if (existence != PathExistence.ExistsAsDirectory)
                {
                    return(existence);
                }

                // For graph file systems, directory members can be determined by overlaying path table with existence state in-memory
                // For real file system, this same is true if the directory has already been enumerated
                if (mode != FileSystemViewMode.Real || ((entry.Flags & FileSystemEntryFlags.IsRealFileSystemEnumerated) != 0))
                {
                    foreach (var childPathValue in PathTable.EnumerateImmediateChildren(path.Value))
                    {
                        var childPath = new AbsolutePath(childPathValue);

                        PathExistence childExistence;
                        if (TryGetKnownPathExistence(childPath, mode, out childExistence) && childExistence != PathExistence.Nonexistent)
                        {
                            var entryName = childPath.GetName(PathTable).ToString(PathTable.StringTable);
                            handleEntry(entryName, childPath, childExistence);
                        }
                    }

                    return(existence);
                }
            }

            if (mode == FileSystemViewMode.Real)
            {
                var handleDirectoryEntry = new Action <string, FileAttributes>((entryName, entryAttributes) =>
                {
                    var childExistence = (entryAttributes & FileAttributes.Directory) != 0 ? PathExistence.ExistsAsDirectory : PathExistence.ExistsAsFile;
                    var childPath      = path.Combine(PathTable, entryName);

                    childExistence = GetOrAddExistence(childPath, mode, childExistence, updateParents: false);

                    // NOTE: Because we are caching file system state in memory, it is possible that the existence state of
                    // files does not match the state from the file system.
                    if (childExistence != PathExistence.Nonexistent)
                    {
                        handleEntry(entryName, childPath, childExistence);
                    }
                });

                Counters.IncrementCounter(FileSystemViewCounters.RealFileSystemEnumerations);

                if (cachePathExistence)
                {
                    Possible <PathExistence> possibleExistence;
                    using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
                    {
                        possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(path, handleDirectoryEntry);
                    }

                    if (possibleExistence.Succeeded)
                    {
                        existence = GetOrAddExistence(path, mode, possibleExistence.Result);

                        PathExistenceCache.AddOrUpdate(path, false,
                                                       (key, data) => { throw Contract.AssertFailure("Entry should already be added for path"); },
                                                       (key, data, oldValue) => oldValue.SetFlag(FileSystemEntryFlags.IsRealFileSystemEnumerated));

                        return(existence);
                    }

                    return(possibleExistence);
                }

                using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
                {
                    var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
                        path.Expand(PathTable).ExpandedPath,
                        handleEntry: handleDirectoryEntry);

                    return(possibleFingerprintResult.Succeeded
                        ? new Possible <PathExistence>(possibleFingerprintResult.Result.PathExistence)
                        : new Possible <PathExistence>(possibleFingerprintResult.Failure));
                }
            }
            else if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
            {
                return(PathExistence.ExistsAsFile);
            }

            return(PathExistence.Nonexistent);
        }