Esempio n. 1
0
        private bool RemoveExtraneousFilesAndDirectories(
            Func <string, bool> isPathInBuild,
            List <string> pathsToScrub,
            HashSet <string> blockedPaths,
            HashSet <string> nonDeletableRootDirectories,
            MountPathExpander mountPathExpander,
            bool logRemovedFiles,
            string statisticIdentifier)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       statisticIdentifier,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
                           period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var directoriesToEnumerate       = new BlockingCollection <string>();

                    foreach (var path in pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
                        {
                            if (!isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        // important to not follow directory symlinks because that can cause
                                        // re-enumerating and scrubbing the same physical folder multiple times
                                        if (FileUtilities.IsDirectoryNoFollow(attributes))
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                        else
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
                                                {
                                                    Interlocked.Increment(ref filesRemoved);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });
                        t.Start();
                        cleaningThreads[i] = t;
                    }

                    foreach (var t in cleaningThreads)
                    {
                        t.Join();
                    }

                    // Collect all directories that need to be deleted.
                    var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase);
                    deleteableDirectories.ExceptWith(nondeletableDirectories.Keys);

                    // Delete directories by considering only the top-most ones.
                    try
                    {
                        Parallel.ForEach(
                            CollapsePaths(deleteableDirectories).ToList(),
                            new ParallelOptions
                        {
                            MaxDegreeOfParallelism = m_maxDegreeParallelism,
                            CancellationToken      = m_cancellationToken,
                        },
                            directory =>
                        {
                            try
                            {
                                FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner);
                                Interlocked.Increment(ref directoriesRemovedRecursively);
                            }
                            catch (BuildXLException ex)
                            {
                                Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
                                    pm.LoggingContext,
                                    directory,
                                    ex.LogEventMessage);
                            }
                        });
                    }
                    catch (OperationCanceledException) { }
                    return(true);
                }
        }
Esempio n. 2
0
        private bool RemoveExtraneousFilesAndDirectories(
            Func <string, bool> isPathInBuild,
            List <string> pathsToScrub,
            HashSet <string> blockedPaths,
            HashSet <string> nonDeletableRootDirectories,
            MountPathExpander mountPathExpander,
            bool logRemovedFiles,
            string statisticIdentifier)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       statisticIdentifier,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
                           period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var directoriesToEnumerate       = new BlockingCollection <string>();
                    var allEnumeratedDirectories     = new ConcurrentBigSet <string>();

                    foreach (var path in pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
                        {
                            if (!isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                                allEnumeratedDirectories.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        string realPath = fullPath;

                                        // If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly
                                        var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath);
                                        if (isDirectorySymlink &&
                                            FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath &&
                                            maybeRealPath.Succeeded)
                                        {
                                            realPath = maybeRealPath.Result;
                                        }

                                        // If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks)
                                        var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound;
                                        if (shouldEnumerateDirectory)
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }

                                        // On Mac directory symlinks are treated like any files, and so we must delete them if
                                        // when they happen to be marked as shared opaque directory output.
                                        //
                                        // When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to
                                        // 'deletableDirectoryCandidates'; there is code that deletes all directories added to this
                                        // list but that code expects a real directory and so might fail to delete a directory symlink.
                                        if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS))
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
                                                {
                                                    Interlocked.Increment(ref filesRemoved);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });