/// <inheritdoc />
        /// <remarks>
        /// This function should ideally queue up the work
        /// </remarks>
        public override async Task <Possible <PackageDownloadResult> > DownloadPackage(
            string weakPackageFingerprint,
            PackageIdentity package,
            AbsolutePath packageTargetFolder,
            Func <Task <Possible <IReadOnlyList <RelativePath> > > > producePackage)
        {
            // Check if we can reuse a package that is layed out on disk already.
            // If the hash file is present packageHash variable will contain the content of it.
            if (TryGetPackageFromFromDisk(package, packageTargetFolder, weakPackageFingerprint, out var possiblePackageFromDisk, out var packageHash))
            {
                return(possiblePackageFromDisk);
            }

            // Slower path: checking the cache first
            var targetLocation = packageTargetFolder.ToString(PathTable);
            var friendlyName   = package.GetFriendlyName();

            using (var pm = PerformanceMeasurement.StartWithoutStatistic(
                       LoggingContext,
                       nestedLoggingContext => m_logger.StartRetrievingPackage(nestedLoggingContext, friendlyName, targetLocation),
                       nestedLoggingContext => m_logger.EndRetrievingPackage(nestedLoggingContext, friendlyName)))
            {
                var loggingContext = pm.LoggingContext;

                // Getting the package content from the cache
                var packageFromCache = await TryGetPackageFromCache(loggingContext, weakPackageFingerprint, package, packageTargetFolder, packageHash);

                if (!packageFromCache.Succeeded || packageFromCache.Result.IsValid)
                {
                    // Cache failure or package was successfully obtained from the cache.
                    return(packageFromCache);
                }

                // Cache miss
                m_logger.PackageNotFoundInCacheAndStartedDownloading(loggingContext, package.Id, package.Version);

                // Step: We couldn't retrieve the package from the cache, or some blobs were missing, generate the package with the given function
                var possiblePackageContents = await producePackage();

                if (!possiblePackageContents.Succeeded)
                {
                    return(possiblePackageContents.Failure);
                }

                // Now we can save the content to the cache
                Analysis.IgnoreResult(
                    await TryAddPackageToCache(
                        loggingContext,
                        weakPackageFingerprint,
                        package,
                        packageTargetFolder,
                        possiblePackageContents.Result),
                    justification: "Okay to ignore putting in cache failure, will happen next time"
                    );

                // Step: Return descriptor indicating it wasn't restored from the cache
                return(PackageDownloadResult.FromRemote(package, packageTargetFolder, possiblePackageContents.Result));
            }
        }
        private void Start()
        {
            p1 = new PerformanceMeasurement();
            p2 = new PerformanceMeasurement();

            p1.Iterate(0, int.MaxValue, AsyncFlags.LowPriority);
            // p2.Iterate(int.MinValue, 0, AsyncFlags.HighPriority);
        }
Exemple #3
0
        /// <summary>
        /// Scrubs extraneous files and directories.
        /// </summary>
        public bool RemoveExtraneousFilesAndDirectories(CancellationToken cancellationToken)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       Category,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration),
                           period: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration)))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var directoriesToEnumerate       = new BlockingCollection <string>();

                    foreach (var path in m_pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (m_blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(path, out foundSemanticPathInfo))
                        {
                            if (!m_isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (m_mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(m_mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(m_mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (m_blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        // important to not follow directory symlinks because that can cause
                                        // re-enumerating and scrubbing the same physical folder multiple times
                                        if (FileUtilities.IsDirectoryNoFollow(attributes))
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!m_isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!m_nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                        else
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!m_isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                try
                                                {
                                                    FileUtilities.DeleteFile(fullPath, tempDirectoryCleaner: m_tempDirectoryCleaner);
                                                    Interlocked.Increment(ref filesRemoved);

                                                    Tracing.Logger.Log.ScrubbingFile(pm.LoggingContext, fullPath);
                                                }
                                                catch (BuildXLException ex)
                                                {
                                                    Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
                                                        pm.LoggingContext,
                                                        fullPath,
                                                        ex.LogEventMessage);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });
                        t.Start();
                        cleaningThreads[i] = t;
                    }

                    foreach (var t in cleaningThreads)
                    {
                        t.Join();
                    }

                    // Collect all directories that need to be deleted.
                    var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase);
                    deleteableDirectories.ExceptWith(nondeletableDirectories.Keys);

                    // Delete directories by considering only the top-most ones.
                    try
                    {
                        Parallel.ForEach(
                            CollapsePaths(deleteableDirectories).ToList(),
                            new ParallelOptions
                        {
                            MaxDegreeOfParallelism = m_maxDegreeParallelism,
                            CancellationToken      = cancellationToken,
                        },
                            directory =>
                        {
                            try
                            {
                                FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner);
                                Interlocked.Increment(ref directoriesRemovedRecursively);
                            }
                            catch (BuildXLException ex)
                            {
                                Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
                                    pm.LoggingContext,
                                    directory,
                                    ex.LogEventMessage);
                            }
                        });
                    }
                    catch (OperationCanceledException) { }
                    return(true);
                }
        }
        public static CacheInitializationTask GetCacheInitializationTask(
            LoggingContext loggingContext,
            PathTable pathTable,
            string cacheDirectory,
            ICacheConfiguration config,
            RootTranslator rootTranslator,
            bool?recoveryStatus,
            CancellationToken cancellationToken,

            // Only used for testing purposes to inject cache.
            Func <EngineCache> testHookCacheFactory = null)
        {
            Contract.Requires(recoveryStatus.HasValue, "Recovery attempt should have been done before initializing the cache");
            DateTime startTime = DateTime.UtcNow;

            var task = Task.Run(
                async() =>
            {
                using (PerformanceMeasurement.Start(
                           loggingContext,
                           "CacheInitialization",
                           Tracing.Logger.Log.StartInitializingCache,
                           Tracing.Logger.Log.EndInitializingCache))
                {
                    if (testHookCacheFactory != null)
                    {
                        return(new MemoryCacheInitializer(
                                   testHookCacheFactory,
                                   loggingContext,
                                   new List <IDisposable>(),
                                   enableFingerprintLookup: config.Incremental));
                    }

                    Possible <CacheCoreCacheInitializer> maybeCacheCoreEngineCache =
                        await CacheCoreCacheInitializer.TryInitializeCacheInternalAsync(
                            loggingContext,
                            pathTable,
                            cacheDirectory,
                            config,
                            enableFingerprintLookup: config.Incremental,
                            rootTranslator: rootTranslator);

                    if (!maybeCacheCoreEngineCache.Succeeded)
                    {
                        string errorMessage = maybeCacheCoreEngineCache.Failure.Describe();
                        if (errorMessage.Contains(LockAcquisitionFailureMessagePrefix))
                        {
                            Tracing.Logger.Log.FailedToAcquireDirectoryLock(
                                loggingContext,
                                maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures());
                        }
                        else
                        {
                            Tracing.Logger.Log.StorageCacheStartupError(
                                loggingContext,
                                maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures());
                        }
                    }

                    return(maybeCacheCoreEngineCache.Then <CacheInitializer>(c => c));
                }
            }, cancellationToken);

            return(new CacheInitializationTask(
                       loggingContext,
                       startTime,
                       task,
                       cancellationToken));
        }
 public bool PredicateDelegate(PerformanceMeasurement memberObject)
 {
     return memberObject.Description.ToUpper() == _description.ToUpper();
 }
Exemple #6
0
        private bool RemoveExtraneousFilesAndDirectories(
            Func <string, bool> isPathInBuild,
            List <string> pathsToScrub,
            HashSet <string> blockedPaths,
            HashSet <string> nonDeletableRootDirectories,
            MountPathExpander mountPathExpander,
            bool logRemovedFiles,
            string statisticIdentifier)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       statisticIdentifier,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
                           period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var directoriesToEnumerate       = new BlockingCollection <string>();
                    var allEnumeratedDirectories     = new ConcurrentBigSet <string>();

                    foreach (var path in pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
                        {
                            if (!isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                                allEnumeratedDirectories.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        string realPath = fullPath;

                                        // If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly
                                        var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath);
                                        if (isDirectorySymlink &&
                                            FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath &&
                                            maybeRealPath.Succeeded)
                                        {
                                            realPath = maybeRealPath.Result;
                                        }

                                        // If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks)
                                        var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound;
                                        if (shouldEnumerateDirectory)
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }

                                        // On Mac directory symlinks are treated like any files, and so we must delete them if
                                        // when they happen to be marked as shared opaque directory output.
                                        //
                                        // When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to
                                        // 'deletableDirectoryCandidates'; there is code that deletes all directories added to this
                                        // list but that code expects a real directory and so might fail to delete a directory symlink.
                                        if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS))
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
                                                {
                                                    Interlocked.Increment(ref filesRemoved);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });
Exemple #7
0
        /// <summary>
        /// Cleans output files and directories.
        /// </summary>
        public static bool DeleteOutputs(
            LoggingContext loggingContext,
            Func <DirectoryArtifact, bool> isOutputDir,
            IList <FileOrDirectoryArtifact> filesOrDirectoriesToDelete,
            PathTable pathTable,
            ITempCleaner tempDirectoryCleaner = null)
        {
            int fileFailCount         = 0;
            int fileSuccessCount      = 0;
            int directoryFailCount    = 0;
            int directorySuccessCount = 0;

            using (PerformanceMeasurement.Start(
                       loggingContext,
                       Category,
                       Tracing.Logger.Log.CleaningStarted,
                       localLoggingContext =>
            {
                Tracing.Logger.Log.CleaningFinished(loggingContext, fileSuccessCount, fileFailCount);
                LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "FilesDeleted", fileSuccessCount);
                LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "FilesFailed", fileFailCount);
                LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "DirectoriesDeleted", directorySuccessCount);
                LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "DirectoriesFailed", directoryFailCount);
            }))
            {
                // Note: filesOrDirectoriesToDelete better be an IList<...> in order to get good Parallel.ForEach performance
                Parallel.ForEach(
                    filesOrDirectoriesToDelete,
                    fileOrDirectory =>
                {
                    string path = fileOrDirectory.Path.ToString(pathTable);
                    Tracing.Logger.Log.CleaningOutputFile(loggingContext, path);

                    try
                    {
                        if (fileOrDirectory.IsFile)
                        {
                            Contract.Assume(fileOrDirectory.FileArtifact.IsOutputFile, "Encountered non-output file");
                            if (FileUtilities.FileExistsNoFollow(path))
                            {
                                FileUtilities.DeleteFile(path, waitUntilDeletionFinished: true, tempDirectoryCleaner: tempDirectoryCleaner);
                                Interlocked.Increment(ref fileSuccessCount);
                            }
                        }
                        else
                        {
                            if (FileUtilities.DirectoryExistsNoFollow(path))
                            {
                                // TODO:1011977 this is a hacky fix for a bug where we delete SourceSealDirectories in /cleanonly mode
                                // The bug stems from the fact that FilterOutputs() returns SourceSealDirectories, which aren't inputs.
                                // Once properly addressed, this check should remain in here as a safety precaution and turn into
                                // a Contract.Assume() like the check above
                                if (isOutputDir(fileOrDirectory.DirectoryArtifact))
                                {
                                    FileUtilities.DeleteDirectoryContents(path, deleteRootDirectory: false, tempDirectoryCleaner: tempDirectoryCleaner);
                                    Interlocked.Increment(ref directorySuccessCount);
                                }
                            }
                        }
                    }
                    catch (BuildXLException ex)
                    {
                        if (fileOrDirectory.IsFile)
                        {
                            Interlocked.Increment(ref fileFailCount);
                            Tracing.Logger.Log.CleaningFileFailed(loggingContext, path, ex.LogEventMessage);
                        }
                        else
                        {
                            Interlocked.Increment(ref directoryFailCount);
                            Tracing.Logger.Log.CleaningDirectoryFailed(loggingContext, path, ex.LogEventMessage);
                        }
                    }
                });
            }

            return(fileFailCount + directoryFailCount == 0);
        }
        private bool RemoveExtraneousFilesAndDirectories(
            Func <string, bool> isPathInBuild,
            List <string> pathsToScrub,
            HashSet <string> blockedPaths,
            HashSet <string> nonDeletableRootDirectories,
            MountPathExpander mountPathExpander,
            bool logRemovedFiles,
            string statisticIdentifier)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       statisticIdentifier,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
                           period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(OperatingSystemHelper.PathComparer);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(OperatingSystemHelper.PathComparer);
                    var directoriesToEnumerate       = new BlockingCollection <string>();

                    foreach (var path in pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
                        {
                            if (!isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        // Only enumerate real directories. We don't follow junctions/symlinks since if there are outputs to scrub under those
                                        // they should also be reachable through real directories from roots BuildXL knows about. This is because we are fully
                                        // resolving dir junctions on detours, and therefore the real paths will also be reported, and proper declarations on
                                        // those will be required
                                        if (FileUtilities.IsDirectoryNoFollow(attributes))
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                        // On Mac directory symlinks are treated like any files, and so we must delete them if
                                        // when they happen to be marked as shared opaque directory output.
                                        else if (OperatingSystemHelper.IsMacOS || !FileUtilities.IsDirectorySymlinkOrJunction(attributes))
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
                                                {
                                                    Interlocked.Increment(ref filesRemoved);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                        // Finally, this is the case of Windows and the file being a directory symlink
                                        else
                                        {
                                            // Since on Windows we don't track dir symlinks for outputs properly, we don't
                                            // want to delete them. This may happen if dir symlinks is the only content of the
                                            // directory, so we flag it as non deletable
                                            shouldDeleteCurrentDirectory = false;
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });
                        t.Start();
                        cleaningThreads[i] = t;
                    }

                    foreach (var t in cleaningThreads)
                    {
                        t.Join();
                    }

                    // Collect all directories that need to be deleted.
                    var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, OperatingSystemHelper.PathComparer);
                    deleteableDirectories.ExceptWith(nondeletableDirectories.Keys);

                    // Delete directories by considering only the top-most ones.
                    try
                    {
                        Parallel.ForEach(
                            CollapsePaths(deleteableDirectories).ToList(),
                            new ParallelOptions
                        {
                            MaxDegreeOfParallelism = m_maxDegreeParallelism,
                            CancellationToken      = m_cancellationToken,
                        },
                            directory =>
                        {
                            try
                            {
                                FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner);
                                Interlocked.Increment(ref directoriesRemovedRecursively);
                            }
                            catch (BuildXLException ex)
                            {
                                Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
                                    pm.LoggingContext,
                                    directory,
                                    ex.LogEventMessage);
                            }
                        });
                    }
                    catch (OperationCanceledException) { }
                    return(true);
                }
        }