/// <summary>
/// Creates an instance of <see cref="DirectoryScrubber"/>.
/// </summary>
/// <remarks>
/// <paramref name="isPathInBuild"/> is a delegate that returns true if a given path is in the build.
/// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or
/// an output file, or a sealed directory. Paths that are in the build should not be deleted.
///
/// <paramref name="pathsToScrub"/> contains a list of paths, including their child paths, that need to be
/// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files
/// or directories in those list.
///
/// <paramref name="blockedPaths"/> stop the above enumeration performed by directory scrubber. All file/directories
/// underneath a blocked path should not be removed.
///
/// <paramref name="nonDeletableRootDirectories"/> contains list of directory paths that can never be deleted, however,
/// the contents of the directory can be scrubbed. For example, mount roots should not be deleted.
/// </remarks>
public DirectoryScrubber(
LoggingContext loggingContext,
ILoggingConfiguration loggingConfiguration,
Func <string, bool> isPathInBuild,
IEnumerable <string> pathsToScrub,
IEnumerable <string> blockedPaths,
IEnumerable <string> nonDeletableRootDirectories,
MountPathExpander mountPathExpander,
int maxDegreeParallelism,
ITempDirectoryCleaner tempDirectoryCleaner = null)
{
m_loggingContext = loggingContext;
m_loggingConfiguration = loggingConfiguration;
m_isPathInBuild = isPathInBuild;
m_pathsToScrub = CollapsePaths(pathsToScrub).ToList();
m_blockedPaths = new HashSet <string>(blockedPaths, StringComparer.OrdinalIgnoreCase);
m_mountPathExpander = mountPathExpander;
m_maxDegreeParallelism = maxDegreeParallelism;
m_nonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, StringComparer.OrdinalIgnoreCase);
if (mountPathExpander != null)
{
m_nonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable)));
}
m_tempDirectoryCleaner = tempDirectoryCleaner;
}
/// <summary>
/// Class constructor
/// </summary>
public MountPathExpander(MountPathExpander parent)
: this(parent.PathTable)
{
Contract.Requires(parent != null);
m_parent = parent;
}
/// <summary>
/// Gets or creates the expander for the specified module
/// </summary>
public MountPathExpander CreateModuleMountExpander(ModuleId moduleId)
{
MountPathExpander moduleExpander;
if (!m_moduleExpanders.TryGetValue(moduleId, out moduleExpander))
{
moduleExpander = new MountPathExpander(this);
m_moduleExpanders.Add(moduleId, moduleExpander);
}
return(moduleExpander);
}
/// <summary>
/// Private constructor. Please use CreateAndRegister factory method.
/// </summary>
private MountsTable(LoggingContext loggingContext, BuildXLContext context, MountPathExpander mountPathExpander = null)
{
m_parent = null;
m_loggingContext = loggingContext;
m_context = context;
MountPathExpander = mountPathExpander ?? new MountPathExpander(context.PathTable);
m_mountMapBuilder = new ConcurrentDictionary <AbsolutePath, IMount>();
m_mountsByName = new ConcurrentDictionary <string, IMount>(StringComparer.OrdinalIgnoreCase);
m_mountPathIdentifiersByMount = new ConcurrentDictionary <IMount, PathAtom>();
m_mountLocationsByMount = new ConcurrentDictionary <IMount, LocationData>();
m_alternativeRoots = new ConcurrentDictionary <AbsolutePath, IMount>();
}
/// <summary>
/// Class constructor
/// </summary>
public MountPathExpander(PathTable pathTable)
{
Contract.Requires(pathTable != null);
m_nameExpander = new Expander();
m_parent = null;
PathTable = pathTable;
m_expandedRootsByName = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
m_mountsByName = new Dictionary <string, SemanticPathInfo>(StringComparer.OrdinalIgnoreCase);
m_semanticPathInfoMap = new FlaggedHierarchicalNameDictionary <SemanticPathInfo>(pathTable, HierarchicalNameTable.NameFlags.Root);
m_moduleExpanders = new Dictionary <ModuleId, MountPathExpander>();
m_alternativeRoots = new List <SemanticPathInfo>();
}
/// <summary>
/// Class constructor
/// </summary>
public CachedGraph(PipGraph pipGraph, DirectedGraph dataflowGraph, PipExecutionContext context, MountPathExpander mountPathExpander)
{
Contract.Requires(pipGraph != null);
Contract.Requires(dataflowGraph != null);
Contract.Requires(context != null);
Contract.Requires(mountPathExpander != null);
DataflowGraph = dataflowGraph;
PipTable = pipGraph.PipTable;
MountPathExpander = mountPathExpander;
Context = context;
PipGraph = pipGraph;
}
/// <summary>
/// Class constructor
/// </summary>
public CachedGraph(PipGraph pipGraph, IReadonlyDirectedGraph directedGraph, PipExecutionContext context, MountPathExpander mountPathExpander, EngineSerializer serializer = null)
{
Contract.Requires(pipGraph != null);
Contract.Requires(directedGraph != null);
Contract.Requires(context != null);
Contract.Requires(mountPathExpander != null);
DirectedGraph = directedGraph;
PipTable = pipGraph.PipTable;
MountPathExpander = mountPathExpander;
Context = context;
PipGraph = pipGraph;
Serializer = serializer;
}
/// <nodoc />
public static Xldb.Proto.MountPathExpander ToMountPathExpander(this MountPathExpander mount, PathTable pathTable, NameExpander nameExpander)
{
var xldbMountPathExpander = new Xldb.Proto.MountPathExpander();
xldbMountPathExpander.WriteableRoots.AddRange(mount.GetWritableRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander)));
xldbMountPathExpander.PathsWithAllowedCreateDirectory.AddRange(mount.GetPathsWithAllowedCreateDirectory().Select(path => path.ToAbsolutePath(pathTable, nameExpander)));
xldbMountPathExpander.ScrubbableRoots.AddRange(mount.GetScrubbableRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander)));
xldbMountPathExpander.AllRoots.AddRange(mount.GetAllRoots().Select(path => path.ToAbsolutePath(pathTable, nameExpander)));
foreach (var kvp in mount.GetAllMountsByName())
{
xldbMountPathExpander.MountsByName.Add(kvp.Key, kvp.Value.ToSemanticPathInfo(pathTable, nameExpander));
}
return(xldbMountPathExpander);
}
/// <summary>
/// Scrubs extraneous files and directories.
/// </summary>
/// <param name="isPathInBuild">
/// A function that returns true if a given path is in the build.
/// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or
/// an output file, or a sealed directory. Paths that are in the build should not be deleted.
/// </param>
/// <param name="pathsToScrub">
/// Contains a list of paths, including their child paths, that need to be
/// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files
/// or directories in those list.
/// </param>
/// <param name="blockedPaths">
/// Stop the above enumeration performed by directory scrubber. All file/directories underneath a blocked path should not be removed.
/// </param>
/// <param name="nonDeletableRootDirectories">
/// Contains list of directory paths that can never be deleted, however,
/// the contents of the directory can be scrubbed. For example, mount roots should not be deleted.
/// </param>
/// <param name="mountPathExpander">
/// Optional mount path expander. When used, its roots are treated as non-deletable.
/// </param>
public bool RemoveExtraneousFilesAndDirectories(
Func <string, bool> isPathInBuild,
IEnumerable <string> pathsToScrub,
IEnumerable <string> blockedPaths,
IEnumerable <string> nonDeletableRootDirectories,
MountPathExpander mountPathExpander = null)
{
var finalPathsToScrub = CollapsePaths(pathsToScrub).ToList();
var finalBlockedPaths = new HashSet <string>(blockedPaths, StringComparer.OrdinalIgnoreCase);
var finalNonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, StringComparer.OrdinalIgnoreCase);
if (mountPathExpander != null)
{
finalNonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable)));
}
return(RemoveExtraneousFilesAndDirectories(isPathInBuild, finalPathsToScrub, finalBlockedPaths, finalNonDeletableRootDirectories, mountPathExpander));
}
/// <summary>
/// Scrubs extraneous files and directories.
/// </summary>
/// <param name="isPathInBuild">
/// A function that returns true if a given path is in the build.
/// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or
/// an output file, or a sealed directory. Paths that are in the build should not be deleted.
/// </param>
/// <param name="pathsToScrub">
/// Contains a list of paths, including their child paths, that need to be
/// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files
/// or directories in those list.
/// </param>
/// <param name="blockedPaths">
/// Stop the above enumeration performed by directory scrubber. All file/directories underneath a blocked path should not be removed.
/// </param>
/// <param name="statisticIdentifier">
/// Identifies the purpose of this scrubber invocation in telemetry and logging. Use this to differentiate since there can
/// be multiple scrubber invocations in the same build session for different purposes.
/// </param>
/// <param name="nonDeletableRootDirectories">
/// Contains list of directory paths that can never be deleted, however,
/// the contents of the directory can be scrubbed. For example, mount roots should not be deleted.
/// </param>
/// <param name="mountPathExpander">
/// Optional mount path expander. When used, its roots are treated as non-deletable.
/// </param>
/// <param name="logRemovedFiles">
/// Optional flag for logging removed files.
/// </param>
public bool RemoveExtraneousFilesAndDirectories(
Func <string, bool> isPathInBuild,
IEnumerable <string> pathsToScrub,
IEnumerable <string> blockedPaths,
IEnumerable <string> nonDeletableRootDirectories,
string statisticIdentifier = Category,
MountPathExpander mountPathExpander = null,
bool logRemovedFiles = true)
{
var finalPathsToScrub = CollapsePaths(pathsToScrub).ToList();
var finalBlockedPaths = new HashSet <string>(blockedPaths, OperatingSystemHelper.PathComparer);
var finalNonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, OperatingSystemHelper.PathComparer);
if (mountPathExpander != null)
{
finalNonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable)));
}
return(RemoveExtraneousFilesAndDirectories(isPathInBuild, finalPathsToScrub, finalBlockedPaths, finalNonDeletableRootDirectories, mountPathExpander, logRemovedFiles, statisticIdentifier));
}
private EngineState(
Guid graphId,
StringTable stringTable,
PathTable pathTable,
SymbolTable symbolTable,
QualifierTable qualifierTable,
PipTable pipTable,
PipGraph pipGraph,
MountPathExpander mountPathExpander,
SchedulerState schedulerState,
HistoricTableSizes historicTableSizes,
FileContentTable fileContentTable)
{
Contract.Requires(graphId != default(Guid), "GraphId is not unique enough to be represented in EngineState");
Contract.Requires(stringTable != null);
Contract.Requires(pathTable != null);
Contract.Requires(symbolTable != null);
Contract.Requires(qualifierTable != null);
Contract.Requires(stringTable == pathTable.StringTable);
Contract.Requires(pathTable.StringTable == symbolTable.StringTable);
Contract.Requires(pathTable.StringTable == qualifierTable.StringTable);
Contract.Requires(pipTable != null);
Contract.Requires(!pipTable.IsDisposed);
Contract.Requires(pipGraph != null);
Contract.Requires(mountPathExpander != null);
Contract.Requires(schedulerState != null);
Contract.Requires(historicTableSizes != null);
Contract.Requires(fileContentTable != null);
m_stringTable = stringTable;
m_pathTable = pathTable;
m_symbolTable = symbolTable;
m_qualifierTable = qualifierTable;
m_pipTable = pipTable;
m_pipGraph = pipGraph;
m_mountPathExpander = mountPathExpander;
m_schedulerState = schedulerState;
m_graphId = graphId;
m_historicTableSizes = historicTableSizes;
m_fileContentTable = fileContentTable;
}
/// <summary>
/// Private constructor used for creating module specific mount tables
/// </summary>
private MountsTable(MountsTable parent, ModuleId moduleId)
: this(parent.m_loggingContext, parent.m_context)
{
m_parent = parent;
MountPathExpander = parent.MountPathExpander.CreateModuleMountExpander(moduleId);
}
/// <summary>
/// Validates directory paths to scrub.
/// </summary>
/// <remarks>
/// A directory path is valid to scrub if it is under a scrubbable mount.
/// </remarks>
private bool ValidateDirectory(MountPathExpander mountPathExpander, string directory, out SemanticPathInfo foundSemanticPathInfo)
{
foundSemanticPathInfo = SemanticPathInfo.Invalid;
return(mountPathExpander == null ||
(foundSemanticPathInfo = mountPathExpander.GetSemanticPathInfo(directory)).IsScrubbable);
}
private bool RemoveExtraneousFilesAndDirectories(
Func <string, bool> isPathInBuild,
List <string> pathsToScrub,
HashSet <string> blockedPaths,
HashSet <string> nonDeletableRootDirectories,
MountPathExpander mountPathExpander,
bool logRemovedFiles,
string statisticIdentifier)
{
int directoriesEncountered = 0;
int filesEncountered = 0;
int filesRemoved = 0;
int directoriesRemovedRecursively = 0;
using (var pm = PerformanceMeasurement.Start(
m_loggingContext,
statisticIdentifier,
// The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
// with particular messages
(_ => {}),
loggingContext =>
{
Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
Logger.Log.BulkStatistic(
loggingContext,
new Dictionary <string, long>
{
[I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
[I($"{Category}.FilesEncountered")] = filesEncountered,
[I($"{Category}.FilesRemoved")] = filesRemoved,
[I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
});
}))
using (var timer = new Timer(
o =>
{
// We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
},
null,
dueTime: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration),
period: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration)))
{
var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var directoriesToEnumerate = new BlockingCollection <string>();
foreach (var path in pathsToScrub)
{
SemanticPathInfo foundSemanticPathInfo;
if (blockedPaths.Contains(path))
{
continue;
}
if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
{
if (!isPathInBuild(path))
{
directoriesToEnumerate.Add(path);
}
else
{
nondeletableDirectories.TryAdd(path, true);
}
}
else
{
string mountName = "Invalid";
string mountPath = "Invalid";
if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
{
mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
}
Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
}
}
var cleaningThreads = new Thread[m_maxDegreeParallelism];
int pending = directoriesToEnumerate.Count;
if (directoriesToEnumerate.Count == 0)
{
directoriesToEnumerate.CompleteAdding();
}
for (int i = 0; i < m_maxDegreeParallelism; i++)
{
var t = new Thread(() =>
{
while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
{
string currentDirectory;
if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
{
Interlocked.Increment(ref directoriesEncountered);
bool shouldDeleteCurrentDirectory = true;
var result = FileUtilities.EnumerateDirectoryEntries(
currentDirectory,
false,
(dir, fileName, attributes) =>
{
string fullPath = Path.Combine(dir, fileName);
// Skip specifically blocked paths.
if (blockedPaths.Contains(fullPath))
{
shouldDeleteCurrentDirectory = false;
return;
}
// important to not follow directory symlinks because that can cause
// re-enumerating and scrubbing the same physical folder multiple times
if (FileUtilities.IsDirectoryNoFollow(attributes))
{
if (nondeletableDirectories.ContainsKey(fullPath))
{
shouldDeleteCurrentDirectory = false;
}
if (!isPathInBuild(fullPath))
{
// Current directory is not in the build, then recurse to its members.
Interlocked.Increment(ref pending);
directoriesToEnumerate.Add(fullPath);
if (!nonDeletableRootDirectories.Contains(fullPath))
{
// Current directory can be deleted, then it is a candidate to be deleted.
deletableDirectoryCandidates.TryAdd(fullPath, true);
}
else
{
// Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
// However, note that we recurse to its members to find all extraneous directories and files.
shouldDeleteCurrentDirectory = false;
}
}
else
{
// Current directory is in the build, i.e., directory is an output directory.
// Stop recursive directory traversal because none of its members should be deleted.
shouldDeleteCurrentDirectory = false;
}
}
else
{
Interlocked.Increment(ref filesEncountered);
if (!isPathInBuild(fullPath))
{
// File is not in the build, delete it.
if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
{
Interlocked.Increment(ref filesRemoved);
}
}
else
{
// File is in the build, then don't delete it, but mark the current directory that
// it should not be deleted.
shouldDeleteCurrentDirectory = false;
}
}
});
if (!result.Succeeded)
{
// Different trace levels based on result.
if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
{
Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
pm.LoggingContext,
currentDirectory,
result.Status.ToString());
}
}
if (!shouldDeleteCurrentDirectory)
{
// If directory should not be deleted, then all of its parents should not be deleted.
int index;
string preservedDirectory = currentDirectory;
bool added;
do
{
added = nondeletableDirectories.TryAdd(preservedDirectory, true);
}while (added &&
(index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
!string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
}
Interlocked.Decrement(ref pending);
}
if (Volatile.Read(ref pending) == 0)
{
directoriesToEnumerate.CompleteAdding();
}
}
});
t.Start();
cleaningThreads[i] = t;
}
foreach (var t in cleaningThreads)
{
t.Join();
}
// Collect all directories that need to be deleted.
var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase);
deleteableDirectories.ExceptWith(nondeletableDirectories.Keys);
// Delete directories by considering only the top-most ones.
try
{
Parallel.ForEach(
CollapsePaths(deleteableDirectories).ToList(),
new ParallelOptions
{
MaxDegreeOfParallelism = m_maxDegreeParallelism,
CancellationToken = m_cancellationToken,
},
directory =>
{
try
{
FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner);
Interlocked.Increment(ref directoriesRemovedRecursively);
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
pm.LoggingContext,
directory,
ex.LogEventMessage);
}
});
}
catch (OperationCanceledException) { }
return(true);
}
}
private CachedGraphLoader(CancellationToken cancellationToken, EngineSerializer serializer)
{
Serializer = serializer;
m_pipGraphIdTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.PipGraph,
async reader =>
{
var graphId = await PipGraph.DeserializeGraphIdAsync(reader);
return(graphId.Item1);
});
var directedGraphTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.DirectedGraph,
DeserializedDirectedGraph.DeserializeAsync);
m_stringTableTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.StringTable,
StringTable.DeserializeAsync);
m_pathTableTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.PathTable,
reader => reader.ReadPathTableAsync(m_stringTableTask.Value));
m_symbolTableTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.SymbolTable,
reader => reader.ReadSymbolTableAsync(m_stringTableTask.Value));
m_qualifierTableTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.QualifierTable,
reader => reader.ReadQualifierTableAsync(m_stringTableTask.Value));
m_pipTableTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.PipTable,
(reader) => PipTable.DeserializeAsync(
reader,
m_pathTableTask.Value,
m_symbolTableTask.Value,
EngineSchedule.PipTableInitialBufferSize,
EngineSchedule.PipTableMaxDegreeOfParallelismDuringConstruction,
debug: false));
m_mountPathExpanderTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.MountPathExpander,
reader => MountPathExpander.DeserializeAsync(reader, m_pathTableTask.Value));
m_pipExecutionContextTask = CreateAsyncLazy(() => Task.Run(
async() =>
{
var stringTable = await m_stringTableTask.Value;
var pathTable = await m_pathTableTask.Value;
var symbolTable = await m_symbolTableTask.Value;
var qualifierTable = await m_qualifierTableTask.Value;
if (stringTable != null && pathTable != null && symbolTable != null && qualifierTable != null)
{
return((PipExecutionContext) new SchedulerContext(cancellationToken, stringTable, pathTable, symbolTable, qualifierTable));
}
return(null);
}));
m_historicDataTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.HistoricTableSizes,
reader => Task.FromResult(HistoricTableSizes.Deserialize(reader)));
m_pipGraphTask = CreateLazyFileDeserialization(
serializer,
GraphCacheFile.PipGraph,
reader => PipGraph.DeserializeAsync(
reader,
serializer.LoggingContext,
m_pipTableTask.Value,
directedGraphTask.Value,
m_pipExecutionContextTask.Value,
ToSemanticPathExpander(m_mountPathExpanderTask.Value)));
m_cachedGraphTask = CreateAsyncLazy(() => CreateCachedGraph(m_pipTableTask.Value, m_pipGraphTask.Value, directedGraphTask.Value, m_pipExecutionContextTask.Value, m_mountPathExpanderTask.Value));
}
private bool RemoveExtraneousFilesAndDirectories(
Func <string, bool> isPathInBuild,
List <string> pathsToScrub,
HashSet <string> blockedPaths,
HashSet <string> nonDeletableRootDirectories,
MountPathExpander mountPathExpander,
bool logRemovedFiles,
string statisticIdentifier)
{
int directoriesEncountered = 0;
int filesEncountered = 0;
int filesRemoved = 0;
int directoriesRemovedRecursively = 0;
using (var pm = PerformanceMeasurement.Start(
m_loggingContext,
statisticIdentifier,
// The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
// with particular messages
(_ => {}),
loggingContext =>
{
Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
Logger.Log.BulkStatistic(
loggingContext,
new Dictionary <string, long>
{
[I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
[I($"{Category}.FilesEncountered")] = filesEncountered,
[I($"{Category}.FilesRemoved")] = filesRemoved,
[I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
});
}))
using (var timer = new Timer(
o =>
{
// We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
},
null,
dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
{
var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var directoriesToEnumerate = new BlockingCollection <string>();
var allEnumeratedDirectories = new ConcurrentBigSet <string>();
foreach (var path in pathsToScrub)
{
SemanticPathInfo foundSemanticPathInfo;
if (blockedPaths.Contains(path))
{
continue;
}
if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
{
if (!isPathInBuild(path))
{
directoriesToEnumerate.Add(path);
allEnumeratedDirectories.Add(path);
}
else
{
nondeletableDirectories.TryAdd(path, true);
}
}
else
{
string mountName = "Invalid";
string mountPath = "Invalid";
if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
{
mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
}
Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
}
}
var cleaningThreads = new Thread[m_maxDegreeParallelism];
int pending = directoriesToEnumerate.Count;
if (directoriesToEnumerate.Count == 0)
{
directoriesToEnumerate.CompleteAdding();
}
for (int i = 0; i < m_maxDegreeParallelism; i++)
{
var t = new Thread(() =>
{
while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
{
string currentDirectory;
if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
{
Interlocked.Increment(ref directoriesEncountered);
bool shouldDeleteCurrentDirectory = true;
var result = FileUtilities.EnumerateDirectoryEntries(
currentDirectory,
false,
(dir, fileName, attributes) =>
{
string fullPath = Path.Combine(dir, fileName);
// Skip specifically blocked paths.
if (blockedPaths.Contains(fullPath))
{
shouldDeleteCurrentDirectory = false;
return;
}
string realPath = fullPath;
// If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly
var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath);
if (isDirectorySymlink &&
FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath &&
maybeRealPath.Succeeded)
{
realPath = maybeRealPath.Result;
}
// If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks)
var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound;
if (shouldEnumerateDirectory)
{
if (nondeletableDirectories.ContainsKey(fullPath))
{
shouldDeleteCurrentDirectory = false;
}
if (!isPathInBuild(fullPath))
{
// Current directory is not in the build, then recurse to its members.
Interlocked.Increment(ref pending);
directoriesToEnumerate.Add(fullPath);
if (!nonDeletableRootDirectories.Contains(fullPath))
{
// Current directory can be deleted, then it is a candidate to be deleted.
deletableDirectoryCandidates.TryAdd(fullPath, true);
}
else
{
// Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
// However, note that we recurse to its members to find all extraneous directories and files.
shouldDeleteCurrentDirectory = false;
}
}
else
{
// Current directory is in the build, i.e., directory is an output directory.
// Stop recursive directory traversal because none of its members should be deleted.
shouldDeleteCurrentDirectory = false;
}
}
// On Mac directory symlinks are treated like any files, and so we must delete them if
// when they happen to be marked as shared opaque directory output.
//
// When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to
// 'deletableDirectoryCandidates'; there is code that deletes all directories added to this
// list but that code expects a real directory and so might fail to delete a directory symlink.
if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS))
{
Interlocked.Increment(ref filesEncountered);
if (!isPathInBuild(fullPath))
{
// File is not in the build, delete it.
if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
{
Interlocked.Increment(ref filesRemoved);
}
}
else
{
// File is in the build, then don't delete it, but mark the current directory that
// it should not be deleted.
shouldDeleteCurrentDirectory = false;
}
}
});
if (!result.Succeeded)
{
// Different trace levels based on result.
if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
{
Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
pm.LoggingContext,
currentDirectory,
result.Status.ToString());
}
}
if (!shouldDeleteCurrentDirectory)
{
// If directory should not be deleted, then all of its parents should not be deleted.
int index;
string preservedDirectory = currentDirectory;
bool added;
do
{
added = nondeletableDirectories.TryAdd(preservedDirectory, true);
}while (added &&
(index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
!string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
}
Interlocked.Decrement(ref pending);
}
if (Volatile.Read(ref pending) == 0)
{
directoriesToEnumerate.CompleteAdding();
}
}
});