private bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, List <string> pathsToScrub, HashSet <string> blockedPaths, HashSet <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, bool logRemovedFiles, string statisticIdentifier) { int directoriesEncountered = 0; int filesEncountered = 0; int filesRemoved = 0; int directoriesRemovedRecursively = 0; using (var pm = PerformanceMeasurement.Start( m_loggingContext, statisticIdentifier, // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing) // with particular messages (_ => {}), loggingContext => { Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively); Logger.Log.BulkStatistic( loggingContext, new Dictionary <string, long> { [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered, [I($"{Category}.FilesEncountered")] = filesEncountered, [I($"{Category}.FilesRemoved")] = filesRemoved, [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively, }); })) using (var timer = new Timer( o => { // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered); }, null, dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(), period: m_loggingConfiguration.GetTimerUpdatePeriodInMs())) { var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var directoriesToEnumerate = new BlockingCollection <string>(); foreach (var path in pathsToScrub) { SemanticPathInfo foundSemanticPathInfo; if (blockedPaths.Contains(path)) { continue; } if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo)) { if (!isPathInBuild(path)) { directoriesToEnumerate.Add(path); } else { nondeletableDirectories.TryAdd(path, true); } } else { string mountName = "Invalid"; string mountPath = "Invalid"; if (mountPathExpander != null && foundSemanticPathInfo.IsValid) { mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable); mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable); } Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath); } } var cleaningThreads = new Thread[m_maxDegreeParallelism]; int pending = directoriesToEnumerate.Count; if (directoriesToEnumerate.Count == 0) { directoriesToEnumerate.CompleteAdding(); } for (int i = 0; i < m_maxDegreeParallelism; i++) { var t = new Thread(() => { while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested) { string currentDirectory; if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite)) { Interlocked.Increment(ref directoriesEncountered); bool shouldDeleteCurrentDirectory = true; var result = FileUtilities.EnumerateDirectoryEntries( currentDirectory, false, (dir, fileName, attributes) => { string fullPath = Path.Combine(dir, fileName); // Skip specifically blocked paths. if (blockedPaths.Contains(fullPath)) { shouldDeleteCurrentDirectory = false; return; } // important to not follow directory symlinks because that can cause // re-enumerating and scrubbing the same physical folder multiple times if (FileUtilities.IsDirectoryNoFollow(attributes)) { if (nondeletableDirectories.ContainsKey(fullPath)) { shouldDeleteCurrentDirectory = false; } if (!isPathInBuild(fullPath)) { // Current directory is not in the build, then recurse to its members. Interlocked.Increment(ref pending); directoriesToEnumerate.Add(fullPath); if (!nonDeletableRootDirectories.Contains(fullPath)) { // Current directory can be deleted, then it is a candidate to be deleted. deletableDirectoryCandidates.TryAdd(fullPath, true); } else { // Current directory can't be deleted (e.g., the root of a mount), then don't delete it. // However, note that we recurse to its members to find all extraneous directories and files. shouldDeleteCurrentDirectory = false; } } else { // Current directory is in the build, i.e., directory is an output directory. // Stop recursive directory traversal because none of its members should be deleted. shouldDeleteCurrentDirectory = false; } } else { Interlocked.Increment(ref filesEncountered); if (!isPathInBuild(fullPath)) { // File is not in the build, delete it. if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles)) { Interlocked.Increment(ref filesRemoved); } } else { // File is in the build, then don't delete it, but mark the current directory that // it should not be deleted. shouldDeleteCurrentDirectory = false; } } }); if (!result.Succeeded) { // Different trace levels based on result. if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound) { Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory( pm.LoggingContext, currentDirectory, result.Status.ToString()); } } if (!shouldDeleteCurrentDirectory) { // If directory should not be deleted, then all of its parents should not be deleted. int index; string preservedDirectory = currentDirectory; bool added; do { added = nondeletableDirectories.TryAdd(preservedDirectory, true); }while (added && (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 && !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index))); } Interlocked.Decrement(ref pending); } if (Volatile.Read(ref pending) == 0) { directoriesToEnumerate.CompleteAdding(); } } }); t.Start(); cleaningThreads[i] = t; } foreach (var t in cleaningThreads) { t.Join(); } // Collect all directories that need to be deleted. var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase); deleteableDirectories.ExceptWith(nondeletableDirectories.Keys); // Delete directories by considering only the top-most ones. try { Parallel.ForEach( CollapsePaths(deleteableDirectories).ToList(), new ParallelOptions { MaxDegreeOfParallelism = m_maxDegreeParallelism, CancellationToken = m_cancellationToken, }, directory => { try { FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner); Interlocked.Increment(ref directoriesRemovedRecursively); } catch (BuildXLException ex) { Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed( pm.LoggingContext, directory, ex.LogEventMessage); } }); } catch (OperationCanceledException) { } return(true); } }
private bool RemoveExtraneousFilesAndDirectories( Func <string, bool> isPathInBuild, List <string> pathsToScrub, HashSet <string> blockedPaths, HashSet <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, bool logRemovedFiles, string statisticIdentifier) { int directoriesEncountered = 0; int filesEncountered = 0; int filesRemoved = 0; int directoriesRemovedRecursively = 0; using (var pm = PerformanceMeasurement.Start( m_loggingContext, statisticIdentifier, // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing) // with particular messages (_ => {}), loggingContext => { Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively); Logger.Log.BulkStatistic( loggingContext, new Dictionary <string, long> { [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered, [I($"{Category}.FilesEncountered")] = filesEncountered, [I($"{Category}.FilesRemoved")] = filesRemoved, [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively, }); })) using (var timer = new Timer( o => { // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered); }, null, dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(), period: m_loggingConfiguration.GetTimerUpdatePeriodInMs())) { var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase); var directoriesToEnumerate = new BlockingCollection <string>(); var allEnumeratedDirectories = new ConcurrentBigSet <string>(); foreach (var path in pathsToScrub) { SemanticPathInfo foundSemanticPathInfo; if (blockedPaths.Contains(path)) { continue; } if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo)) { if (!isPathInBuild(path)) { directoriesToEnumerate.Add(path); allEnumeratedDirectories.Add(path); } else { nondeletableDirectories.TryAdd(path, true); } } else { string mountName = "Invalid"; string mountPath = "Invalid"; if (mountPathExpander != null && foundSemanticPathInfo.IsValid) { mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable); mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable); } Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath); } } var cleaningThreads = new Thread[m_maxDegreeParallelism]; int pending = directoriesToEnumerate.Count; if (directoriesToEnumerate.Count == 0) { directoriesToEnumerate.CompleteAdding(); } for (int i = 0; i < m_maxDegreeParallelism; i++) { var t = new Thread(() => { while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested) { string currentDirectory; if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite)) { Interlocked.Increment(ref directoriesEncountered); bool shouldDeleteCurrentDirectory = true; var result = FileUtilities.EnumerateDirectoryEntries( currentDirectory, false, (dir, fileName, attributes) => { string fullPath = Path.Combine(dir, fileName); // Skip specifically blocked paths. if (blockedPaths.Contains(fullPath)) { shouldDeleteCurrentDirectory = false; return; } string realPath = fullPath; // If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath); if (isDirectorySymlink && FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath && maybeRealPath.Succeeded) { realPath = maybeRealPath.Result; } // If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks) var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound; if (shouldEnumerateDirectory) { if (nondeletableDirectories.ContainsKey(fullPath)) { shouldDeleteCurrentDirectory = false; } if (!isPathInBuild(fullPath)) { // Current directory is not in the build, then recurse to its members. Interlocked.Increment(ref pending); directoriesToEnumerate.Add(fullPath); if (!nonDeletableRootDirectories.Contains(fullPath)) { // Current directory can be deleted, then it is a candidate to be deleted. deletableDirectoryCandidates.TryAdd(fullPath, true); } else { // Current directory can't be deleted (e.g., the root of a mount), then don't delete it. // However, note that we recurse to its members to find all extraneous directories and files. shouldDeleteCurrentDirectory = false; } } else { // Current directory is in the build, i.e., directory is an output directory. // Stop recursive directory traversal because none of its members should be deleted. shouldDeleteCurrentDirectory = false; } } // On Mac directory symlinks are treated like any files, and so we must delete them if // when they happen to be marked as shared opaque directory output. // // When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to // 'deletableDirectoryCandidates'; there is code that deletes all directories added to this // list but that code expects a real directory and so might fail to delete a directory symlink. if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS)) { Interlocked.Increment(ref filesEncountered); if (!isPathInBuild(fullPath)) { // File is not in the build, delete it. if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles)) { Interlocked.Increment(ref filesRemoved); } } else { // File is in the build, then don't delete it, but mark the current directory that // it should not be deleted. shouldDeleteCurrentDirectory = false; } } }); if (!result.Succeeded) { // Different trace levels based on result. if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound) { Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory( pm.LoggingContext, currentDirectory, result.Status.ToString()); } } if (!shouldDeleteCurrentDirectory) { // If directory should not be deleted, then all of its parents should not be deleted. int index; string preservedDirectory = currentDirectory; bool added; do { added = nondeletableDirectories.TryAdd(preservedDirectory, true); }while (added && (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 && !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index))); } Interlocked.Decrement(ref pending); } if (Volatile.Read(ref pending) == 0) { directoriesToEnumerate.CompleteAdding(); } } });