/// <summary> /// Creates an instance of <see cref="DirectoryScrubber"/>. /// </summary> /// <remarks> /// <paramref name="isPathInBuild"/> is a delegate that returns true if a given path is in the build. /// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or /// an output file, or a sealed directory. Paths that are in the build should not be deleted. /// /// <paramref name="pathsToScrub"/> contains a list of paths, including their child paths, that need to be /// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files /// or directories in those list. /// /// <paramref name="blockedPaths"/> stop the above enumeration performed by directory scrubber. All file/directories /// underneath a blocked path should not be removed. /// /// <paramref name="nonDeletableRootDirectories"/> contains list of directory paths that can never be deleted, however, /// the contents of the directory can be scrubbed. For example, mount roots should not be deleted. /// </remarks> public DirectoryScrubber( LoggingContext loggingContext, ILoggingConfiguration loggingConfiguration, Func <string, bool> isPathInBuild, IEnumerable <string> pathsToScrub, IEnumerable <string> blockedPaths, IEnumerable <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, int maxDegreeParallelism, ITempDirectoryCleaner tempDirectoryCleaner = null) { m_loggingContext = loggingContext; m_loggingConfiguration = loggingConfiguration; m_isPathInBuild = isPathInBuild; m_pathsToScrub = CollapsePaths(pathsToScrub).ToList(); m_blockedPaths = new HashSet <string>(blockedPaths, StringComparer.OrdinalIgnoreCase); m_mountPathExpander = mountPathExpander; m_maxDegreeParallelism = maxDegreeParallelism; m_nonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, StringComparer.OrdinalIgnoreCase); if (mountPathExpander != null) { m_nonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable))); } m_tempDirectoryCleaner = tempDirectoryCleaner; }
/// <summary> /// Class constructor /// </summary> public MountPathExpander(MountPathExpander parent) : this(parent.PathTable) { Contract.Requires(parent != null); m_parent = parent; }
/// <summary> /// Gets or creates the expander for the specified module /// </summary> public MountPathExpander CreateModuleMountExpander(ModuleId moduleId) { MountPathExpander moduleExpander; if (!m_moduleExpanders.TryGetValue(moduleId, out moduleExpander)) { moduleExpander = new MountPathExpander(this); m_moduleExpanders.Add(moduleId, moduleExpander); } return(moduleExpander); }
/// <summary> /// Private constructor. Please use CreateAndRegister factory method. /// </summary> private MountsTable(LoggingContext loggingContext, BuildXLContext context, MountPathExpander mountPathExpander = null) { m_parent = null; m_loggingContext = loggingContext; m_context = context; MountPathExpander = mountPathExpander ?? new MountPathExpander(context.PathTable); m_mountMapBuilder = new ConcurrentDictionary <AbsolutePath, IMount>(); m_mountsByName = new ConcurrentDictionary <string, IMount>(StringComparer.OrdinalIgnoreCase); m_mountPathIdentifiersByMount = new ConcurrentDictionary <IMount, PathAtom>(); m_mountLocationsByMount = new ConcurrentDictionary <IMount, LocationData>(); m_alternativeRoots = new ConcurrentDictionary <AbsolutePath, IMount>(); }
/// <summary> /// Class constructor /// </summary> public MountPathExpander(PathTable pathTable) { Contract.Requires(pathTable != null); m_nameExpander = new Expander(); m_parent = null; PathTable = pathTable; m_expandedRootsByName = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); m_mountsByName = new Dictionary <string, SemanticPathInfo>(StringComparer.OrdinalIgnoreCase); m_semanticPathInfoMap = new FlaggedHierarchicalNameDictionary <SemanticPathInfo>(pathTable, HierarchicalNameTable.NameFlags.Root); m_moduleExpanders = new Dictionary <ModuleId, MountPathExpander>(); m_alternativeRoots = new List <SemanticPathInfo>(); }
/// <summary> /// Class constructor /// </summary> public CachedGraph(PipGraph pipGraph, DirectedGraph dataflowGraph, PipExecutionContext context, MountPathExpander mountPathExpander) { Contract.Requires(pipGraph != null); Contract.Requires(dataflowGraph != null); Contract.Requires(context != null); Contract.Requires(mountPathExpander != null); DataflowGraph = dataflowGraph; PipTable = pipGraph.PipTable; MountPathExpander = mountPathExpander; Context = context; PipGraph = pipGraph; }
/// <summary> /// Class constructor /// </summary> public CachedGraph(PipGraph pipGraph, IReadonlyDirectedGraph directedGraph, PipExecutionContext context, MountPathExpander mountPathExpander, EngineSerializer serializer = null) { Contract.Requires(pipGraph != null); Contract.Requires(directedGraph != null); Contract.Requires(context != null); Contract.Requires(mountPathExpander != null); DirectedGraph = directedGraph; PipTable = pipGraph.PipTable; MountPathExpander = mountPathExpander; Context = context; PipGraph = pipGraph; Serializer = serializer; }
/// <nodoc /> public static Xldb.Proto.MountPathExpander ToMountPathExpander(this MountPathExpander mount, PathTable pathTable, NameExpander nameExpander) { var xldbMountPathExpander = new Xldb.Proto.MountPathExpander(); xldbMountPathExpander.WriteableRoots.AddRange(mount.GetWritableRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander))); xldbMountPathExpander.PathsWithAllowedCreateDirectory.AddRange(mount.GetPathsWithAllowedCreateDirectory().Select(path => path.ToAbsolutePath(pathTable, nameExpander))); xldbMountPathExpander.ScrubbableRoots.AddRange(mount.GetScrubbableRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander))); xldbMountPathExpander.AllRoots.AddRange(mount.GetAllRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander))); foreach (var kvp in mount.GetAllMountsByName()) { xldbMountPathExpander.MountsByName.Add(kvp.Key, kvp.Value.ToSemanticPathInfo(pathTable, nameExpander)); } return(xldbMountPathExpander); }
/// <summary> /// Scrubs extraneous files and directories. /// </summary> /// <param name="isPathInBuild"> /// A function that returns true if a given path is in the build. /// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or /// an output file, or a sealed directory. Paths that are in the build should not be deleted. /// </param> /// <param name="pathsToScrub"> /// Contains a list of paths, including their child paths, that need to be /// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files /// or directories in those list. /// </param> /// <param name="blockedPaths"> /// Stop the above enumeration performed by directory scrubber. All file/directories underneath a blocked path should not be removed. /// </param> /// <param name="nonDeletableRootDirectories"> /// Contains list of directory paths that can never be deleted, however, /// the contents of the directory can be scrubbed. For example, mount roots should not be deleted. /// </param> /// <param name="mountPathExpander"> /// Optional mount path expander. When used, its roots are treated as non-deletable. /// </param> public bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, IEnumerable <string> pathsToScrub, IEnumerable <string> blockedPaths, IEnumerable <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander = null) { var finalPathsToScrub = CollapsePaths(pathsToScrub).ToList(); var finalBlockedPaths = new HashSet <string>(blockedPaths, StringComparer.OrdinalIgnoreCase); var finalNonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, StringComparer.OrdinalIgnoreCase); if (mountPathExpander != null) { finalNonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable))); } return(RemoveExtraneousFilesAndDirectories(isPathInBuild, finalPathsToScrub, finalBlockedPaths, finalNonDeletableRootDirectories, mountPathExpander)); }
/// <summary> /// Scrubs extraneous files and directories. /// </summary> /// <param name="isPathInBuild"> /// A function that returns true if a given path is in the build. /// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or /// an output file, or a sealed directory. Paths that are in the build should not be deleted. /// </param> /// <param name="pathsToScrub"> /// Contains a list of paths, including their child paths, that need to be /// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files /// or directories in those list. /// </param> /// <param name="blockedPaths"> /// Stop the above enumeration performed by directory scrubber. All file/directories underneath a blocked path should not be removed. /// </param> /// <param name="statisticIdentifier"> /// Identifies the purpose of this scrubber invocation in telemetry and logging. Use this to differentiate since there can /// be multiple scrubber invocations in the same build session for different purposes. /// </param> /// <param name="nonDeletableRootDirectories"> /// Contains list of directory paths that can never be deleted, however, /// the contents of the directory can be scrubbed. For example, mount roots should not be deleted. /// </param> /// <param name="mountPathExpander"> /// Optional mount path expander. When used, its roots are treated as non-deletable. /// </param> /// <param name="logRemovedFiles"> /// Optional flag for logging removed files. /// </param> public bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, IEnumerable <string> pathsToScrub, IEnumerable <string> blockedPaths, IEnumerable <string> nonDeletableRootDirectories, string statisticIdentifier = Category, MountPathExpander mountPathExpander = null, bool logRemovedFiles = true) { var finalPathsToScrub = CollapsePaths(pathsToScrub).ToList(); var finalBlockedPaths = new HashSet <string>(blockedPaths, OperatingSystemHelper.PathComparer); var finalNonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, OperatingSystemHelper.PathComparer); if (mountPathExpander != null) { finalNonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable))); } return(RemoveExtraneousFilesAndDirectories(isPathInBuild, finalPathsToScrub, finalBlockedPaths, finalNonDeletableRootDirectories, mountPathExpander, logRemovedFiles, statisticIdentifier)); }
private EngineState( Guid graphId, StringTable stringTable, PathTable pathTable, SymbolTable symbolTable, QualifierTable qualifierTable, PipTable pipTable, PipGraph pipGraph, MountPathExpander mountPathExpander, SchedulerState schedulerState, HistoricTableSizes historicTableSizes, FileContentTable fileContentTable) { Contract.Requires(graphId != default(Guid), "GraphId is not unique enough to be represented in EngineState"); Contract.Requires(stringTable != null); Contract.Requires(pathTable != null); Contract.Requires(symbolTable != null); Contract.Requires(qualifierTable != null); Contract.Requires(stringTable == pathTable.StringTable); Contract.Requires(pathTable.StringTable == symbolTable.StringTable); Contract.Requires(pathTable.StringTable == qualifierTable.StringTable); Contract.Requires(pipTable != null); Contract.Requires(!pipTable.IsDisposed); Contract.Requires(pipGraph != null); Contract.Requires(mountPathExpander != null); Contract.Requires(schedulerState != null); Contract.Requires(historicTableSizes != null); Contract.Requires(fileContentTable != null); m_stringTable = stringTable; m_pathTable = pathTable; m_symbolTable = symbolTable; m_qualifierTable = qualifierTable; m_pipTable = pipTable; m_pipGraph = pipGraph; m_mountPathExpander = mountPathExpander; m_schedulerState = schedulerState; m_graphId = graphId; m_historicTableSizes = historicTableSizes; m_fileContentTable = fileContentTable; }
/// <summary> /// Private constructor used for creating module specific mount tables /// </summary> private MountsTable(MountsTable parent, ModuleId moduleId) : this(parent.m_loggingContext, parent.m_context) { m_parent = parent; MountPathExpander = parent.MountPathExpander.CreateModuleMountExpander(moduleId); }
/// <summary> /// Validates directory paths to scrub. /// </summary> /// <remarks> /// A directory path is valid to scrub if it is under a scrubbable mount. /// </remarks> private bool ValidateDirectory(MountPathExpander mountPathExpander, string directory, out SemanticPathInfo foundSemanticPathInfo) { foundSemanticPathInfo = SemanticPathInfo.Invalid; return(mountPathExpander == null || (foundSemanticPathInfo = mountPathExpander.GetSemanticPathInfo(directory)).IsScrubbable); }
private bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, List <string> pathsToScrub, HashSet <string> blockedPaths, HashSet <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, bool logRemovedFiles, string statisticIdentifier) { int directoriesEncountered = 0; int filesEncountered = 0; int filesRemoved = 0; int directoriesRemovedRecursively = 0; using (var pm = PerformanceMeasurement.Start( m_loggingContext, statisticIdentifier, // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing) // with particular messages (_ => {}), loggingContext => { Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively); Logger.Log.BulkStatistic( loggingContext, new Dictionary <string, long> { [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered, [I($"{Category}.FilesEncountered")] = filesEncountered, [I($"{Category}.FilesRemoved")] = filesRemoved, [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively, }); })) using (var timer = new Timer( o => { // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered); }, null, dueTime: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration), period: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration))) { var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var directoriesToEnumerate = new BlockingCollection <string>(); foreach (var path in pathsToScrub) { SemanticPathInfo foundSemanticPathInfo; if (blockedPaths.Contains(path)) { continue; } if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo)) { if (!isPathInBuild(path)) { directoriesToEnumerate.Add(path); } else { nondeletableDirectories.TryAdd(path, true); } } else { string mountName = "Invalid"; string mountPath = "Invalid"; if (mountPathExpander != null && foundSemanticPathInfo.IsValid) { mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable); mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable); } Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath); } } var cleaningThreads = new Thread[m_maxDegreeParallelism]; int pending = directoriesToEnumerate.Count; if (directoriesToEnumerate.Count == 0) { directoriesToEnumerate.CompleteAdding(); } for (int i = 0; i < m_maxDegreeParallelism; i++) { var t = new Thread(() => { while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested) { string currentDirectory; if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite)) { Interlocked.Increment(ref directoriesEncountered); bool shouldDeleteCurrentDirectory = true; var result = FileUtilities.EnumerateDirectoryEntries( currentDirectory, false, (dir, fileName, attributes) => { string fullPath = Path.Combine(dir, fileName); // Skip specifically blocked paths. if (blockedPaths.Contains(fullPath)) { shouldDeleteCurrentDirectory = false; return; } // important to not follow directory symlinks because that can cause // re-enumerating and scrubbing the same physical folder multiple times if (FileUtilities.IsDirectoryNoFollow(attributes)) { if (nondeletableDirectories.ContainsKey(fullPath)) { shouldDeleteCurrentDirectory = false; } if (!isPathInBuild(fullPath)) { // Current directory is not in the build, then recurse to its members. Interlocked.Increment(ref pending); directoriesToEnumerate.Add(fullPath); if (!nonDeletableRootDirectories.Contains(fullPath)) { // Current directory can be deleted, then it is a candidate to be deleted. deletableDirectoryCandidates.TryAdd(fullPath, true); } else { // Current directory can't be deleted (e.g., the root of a mount), then don't delete it. // However, note that we recurse to its members to find all extraneous directories and files. shouldDeleteCurrentDirectory = false; } } else { // Current directory is in the build, i.e., directory is an output directory. // Stop recursive directory traversal because none of its members should be deleted. shouldDeleteCurrentDirectory = false; } } else { Interlocked.Increment(ref filesEncountered); if (!isPathInBuild(fullPath)) { // File is not in the build, delete it. if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles)) { Interlocked.Increment(ref filesRemoved); } } else { // File is in the build, then don't delete it, but mark the current directory that // it should not be deleted. shouldDeleteCurrentDirectory = false; } } }); if (!result.Succeeded) { // Different trace levels based on result. if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound) { Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory( pm.LoggingContext, currentDirectory, result.Status.ToString()); } } if (!shouldDeleteCurrentDirectory) { // If directory should not be deleted, then all of its parents should not be deleted. int index; string preservedDirectory = currentDirectory; bool added; do { added = nondeletableDirectories.TryAdd(preservedDirectory, true); }while (added && (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 && !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index))); } Interlocked.Decrement(ref pending); } if (Volatile.Read(ref pending) == 0) { directoriesToEnumerate.CompleteAdding(); } } }); t.Start(); cleaningThreads[i] = t; } foreach (var t in cleaningThreads) { t.Join(); } // Collect all directories that need to be deleted. var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase); deleteableDirectories.ExceptWith(nondeletableDirectories.Keys); // Delete directories by considering only the top-most ones. try { Parallel.ForEach( CollapsePaths(deleteableDirectories).ToList(), new ParallelOptions { MaxDegreeOfParallelism = m_maxDegreeParallelism, CancellationToken = m_cancellationToken, }, directory => { try { FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner); Interlocked.Increment(ref directoriesRemovedRecursively); } catch (BuildXLException ex) { Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed( pm.LoggingContext, directory, ex.LogEventMessage); } }); } catch (OperationCanceledException) { } return(true); } }
private CachedGraphLoader(CancellationToken cancellationToken, EngineSerializer serializer) { Serializer = serializer; m_pipGraphIdTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.PipGraph, async reader => { var graphId = await PipGraph.DeserializeGraphIdAsync(reader); return(graphId.Item1); }); var directedGraphTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.DirectedGraph, DeserializedDirectedGraph.DeserializeAsync); m_stringTableTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.StringTable, StringTable.DeserializeAsync); m_pathTableTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.PathTable, reader => reader.ReadPathTableAsync(m_stringTableTask.Value)); m_symbolTableTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.SymbolTable, reader => reader.ReadSymbolTableAsync(m_stringTableTask.Value)); m_qualifierTableTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.QualifierTable, reader => reader.ReadQualifierTableAsync(m_stringTableTask.Value)); m_pipTableTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.PipTable, (reader) => PipTable.DeserializeAsync( reader, m_pathTableTask.Value, m_symbolTableTask.Value, EngineSchedule.PipTableInitialBufferSize, EngineSchedule.PipTableMaxDegreeOfParallelismDuringConstruction, debug: false)); m_mountPathExpanderTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.MountPathExpander, reader => MountPathExpander.DeserializeAsync(reader, m_pathTableTask.Value)); m_pipExecutionContextTask = CreateAsyncLazy(() => Task.Run( async() => { var stringTable = await m_stringTableTask.Value; var pathTable = await m_pathTableTask.Value; var symbolTable = await m_symbolTableTask.Value; var qualifierTable = await m_qualifierTableTask.Value; if (stringTable != null && pathTable != null && symbolTable != null && qualifierTable != null) { return((PipExecutionContext) new SchedulerContext(cancellationToken, stringTable, pathTable, symbolTable, qualifierTable)); } return(null); })); m_historicDataTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.HistoricTableSizes, reader => Task.FromResult(HistoricTableSizes.Deserialize(reader))); m_pipGraphTask = CreateLazyFileDeserialization( serializer, GraphCacheFile.PipGraph, reader => PipGraph.DeserializeAsync( reader, serializer.LoggingContext, m_pipTableTask.Value, directedGraphTask.Value, m_pipExecutionContextTask.Value, ToSemanticPathExpander(m_mountPathExpanderTask.Value))); m_cachedGraphTask = CreateAsyncLazy(() => CreateCachedGraph(m_pipTableTask.Value, m_pipGraphTask.Value, directedGraphTask.Value, m_pipExecutionContextTask.Value, m_mountPathExpanderTask.Value)); }
private bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, List <string> pathsToScrub, HashSet <string> blockedPaths, HashSet <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, bool logRemovedFiles, string statisticIdentifier) { int directoriesEncountered = 0; int filesEncountered = 0; int filesRemoved = 0; int directoriesRemovedRecursively = 0; using (var pm = PerformanceMeasurement.Start( m_loggingContext, statisticIdentifier, // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing) // with particular messages (_ => {}), loggingContext => { Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively); Logger.Log.BulkStatistic( loggingContext, new Dictionary <string, long> { [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered, [I($"{Category}.FilesEncountered")] = filesEncountered, [I($"{Category}.FilesRemoved")] = filesRemoved, [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively, }); })) using (var timer = new Timer( o => { // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered); }, null, dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(), period: m_loggingConfiguration.GetTimerUpdatePeriodInMs())) { var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var directoriesToEnumerate = new BlockingCollection <string>(); var allEnumeratedDirectories = new ConcurrentBigSet <string>(); foreach (var path in pathsToScrub) { SemanticPathInfo foundSemanticPathInfo; if (blockedPaths.Contains(path)) { continue; } if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo)) { if (!isPathInBuild(path)) { directoriesToEnumerate.Add(path); allEnumeratedDirectories.Add(path); } else { nondeletableDirectories.TryAdd(path, true); } } else { string mountName = "Invalid"; string mountPath = "Invalid"; if (mountPathExpander != null && foundSemanticPathInfo.IsValid) { mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable); mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable); } Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath); } } var cleaningThreads = new Thread[m_maxDegreeParallelism]; int pending = directoriesToEnumerate.Count; if (directoriesToEnumerate.Count == 0) { directoriesToEnumerate.CompleteAdding(); } for (int i = 0; i < m_maxDegreeParallelism; i++) { var t = new Thread(() => { while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested) { string currentDirectory; if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite)) { Interlocked.Increment(ref directoriesEncountered); bool shouldDeleteCurrentDirectory = true; var result = FileUtilities.EnumerateDirectoryEntries( currentDirectory, false, (dir, fileName, attributes) => { string fullPath = Path.Combine(dir, fileName); // Skip specifically blocked paths. if (blockedPaths.Contains(fullPath)) { shouldDeleteCurrentDirectory = false; return; } string realPath = fullPath; // If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath); if (isDirectorySymlink && FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath && maybeRealPath.Succeeded) { realPath = maybeRealPath.Result; } // If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks) var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound; if (shouldEnumerateDirectory) { if (nondeletableDirectories.ContainsKey(fullPath)) { shouldDeleteCurrentDirectory = false; } if (!isPathInBuild(fullPath)) { // Current directory is not in the build, then recurse to its members. Interlocked.Increment(ref pending); directoriesToEnumerate.Add(fullPath); if (!nonDeletableRootDirectories.Contains(fullPath)) { // Current directory can be deleted, then it is a candidate to be deleted. deletableDirectoryCandidates.TryAdd(fullPath, true); } else { // Current directory can't be deleted (e.g., the root of a mount), then don't delete it. // However, note that we recurse to its members to find all extraneous directories and files. shouldDeleteCurrentDirectory = false; } } else { // Current directory is in the build, i.e., directory is an output directory. // Stop recursive directory traversal because none of its members should be deleted. shouldDeleteCurrentDirectory = false; } } // On Mac directory symlinks are treated like any files, and so we must delete them if // when they happen to be marked as shared opaque directory output. // // When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to // 'deletableDirectoryCandidates'; there is code that deletes all directories added to this // list but that code expects a real directory and so might fail to delete a directory symlink. if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS)) { Interlocked.Increment(ref filesEncountered); if (!isPathInBuild(fullPath)) { // File is not in the build, delete it. if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles)) { Interlocked.Increment(ref filesRemoved); } } else { // File is in the build, then don't delete it, but mark the current directory that // it should not be deleted. shouldDeleteCurrentDirectory = false; } } }); if (!result.Succeeded) { // Different trace levels based on result. if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound) { Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory( pm.LoggingContext, currentDirectory, result.Status.ToString()); } } if (!shouldDeleteCurrentDirectory) { // If directory should not be deleted, then all of its parents should not be deleted. int index; string preservedDirectory = currentDirectory; bool added; do { added = nondeletableDirectories.TryAdd(preservedDirectory, true); }while (added && (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 && !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index))); } Interlocked.Decrement(ref pending); } if (Volatile.Read(ref pending) == 0) { directoriesToEnumerate.CompleteAdding(); } } });