/// <summary> /// Creates an instance of <see cref="DirectoryScrubber"/>. /// </summary> /// <remarks> /// <paramref name="isPathInBuild"/> is a delegate that returns true if a given path is in the build. /// Basically, a path is in a build if it points to an artifact in the pip graph, i.e., path to a source file, or /// an output file, or a sealed directory. Paths that are in the build should not be deleted. /// /// <paramref name="pathsToScrub"/> contains a list of paths, including their child paths, that need to be /// scrubbed. Basically, the directory scrubber enumerates those paths recursively for removing extraneous files /// or directories in those list. /// /// <paramref name="blockedPaths"/> stop the above enumeration performed by directory scrubber. All file/directories /// underneath a blocked path should not be removed. /// /// <paramref name="nonDeletableRootDirectories"/> contains list of directory paths that can never be deleted, however, /// the contents of the directory can be scrubbed. For example, mount roots should not be deleted. /// </remarks> public DirectoryScrubber( LoggingContext loggingContext, ILoggingConfiguration loggingConfiguration, Func <string, bool> isPathInBuild, IEnumerable <string> pathsToScrub, IEnumerable <string> blockedPaths, IEnumerable <string> nonDeletableRootDirectories, MountPathExpander mountPathExpander, int maxDegreeParallelism, ITempCleaner tempDirectoryCleaner = null) { m_loggingContext = loggingContext; m_loggingConfiguration = loggingConfiguration; m_isPathInBuild = isPathInBuild; m_pathsToScrub = CollapsePaths(pathsToScrub).ToList(); m_blockedPaths = new HashSet <string>(blockedPaths, StringComparer.OrdinalIgnoreCase); m_mountPathExpander = mountPathExpander; m_maxDegreeParallelism = maxDegreeParallelism; m_nonDeletableRootDirectories = new HashSet <string>(nonDeletableRootDirectories, StringComparer.OrdinalIgnoreCase); if (mountPathExpander != null) { m_nonDeletableRootDirectories.UnionWith(mountPathExpander.GetAllRoots().Select(p => p.ToString(mountPathExpander.PathTable))); } m_tempDirectoryCleaner = tempDirectoryCleaner; }
/// <inheritdoc /> public void DeleteDirectoryContents( string path, bool deleteRootDirectory = false, Func <string, bool> shouldDelete = null, ITempCleaner tempDirectoryCleaner = null, CancellationToken?cancellationToken = default) { DeleteDirectoryContentsInternal(path, deleteRootDirectory, shouldDelete, tempDirectoryCleaner, cancellationToken); }
/// <inheritdoc /> public void DeleteFile( string path, bool waitUntilDeletionFinished = true, ITempCleaner tempDirectoryCleaner = null) { Contract.Requires(!string.IsNullOrEmpty(path)); bool successfullyDeletedFile = false; if (!Exists(path)) { // Skip deletion all together if nothing exists at the specified path return; } Action <string> delete = (string pathToDelete) => { var isDirectory = FileUtilities.DirectoryExistsNoFollow(pathToDelete); if (isDirectory) { DeleteDirectoryContents(pathToDelete, deleteRootDirectory: true); } else { File.Delete(pathToDelete); } }; if (waitUntilDeletionFinished) { successfullyDeletedFile = Helpers.RetryOnFailure( attempt => { delete(path); return(true); }); } else { try { delete(path); successfullyDeletedFile = true; } #pragma warning disable ERP022 // Unobserved exception in generic exception handler catch { } #pragma warning restore ERP022 // Unobserved exception in generic exception handler } if (!successfullyDeletedFile) { throw new BuildXLException("Deleting file '" + path + "' failed!"); } }
/// <summary> /// Creates an instance of <see cref="IncrementalSchedulingStateFactory"/>. /// </summary> public IncrementalSchedulingStateFactory( LoggingContext loggingContext, bool analysisMode = false, ITempCleaner tempDirectoryCleaner = null) { Contract.Requires(loggingContext != null); m_loggingContext = loggingContext; m_analysisMode = analysisMode; m_tempDirectoryCleaner = tempDirectoryCleaner; }
public TestScheduler( PipGraph graph, TestPipQueue pipQueue, PipExecutionContext context, FileContentTable fileContentTable, EngineCache cache, IConfiguration configuration, FileAccessWhitelist fileAccessWhitelist, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null, ITempCleaner tempCleaner = null, PipRuntimeTimeTable runningTimeTable = null, JournalState journalState = null, PerformanceCollector performanceCollector = null, string fingerprintSalt = null, PreserveOutputsInfo?previousInputsSalt = null, IEnumerable <Pip> successfulPips = null, IEnumerable <Pip> failedPips = null, LoggingContext loggingContext = null, IIpcProvider ipcProvider = null, DirectoryTranslator directoryTranslator = null, VmInitializer vmInitializer = null, SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache, configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules, tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt, ipcProvider: ipcProvider, directoryTranslator: directoryTranslator, journalState: journalState, vmInitializer: vmInitializer, testHooks: testHooks) { m_testPipQueue = pipQueue; if (successfulPips != null) { foreach (var pip in successfulPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded); } } if (failedPips != null) { foreach (var pip in failedPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed); } } m_loggingContext = loggingContext; }
/// <summary> /// Creates an instance of <see cref="DirectoryScrubber"/>. /// </summary> public DirectoryScrubber( CancellationToken cancellationToken, LoggingContext loggingContext, ILoggingConfiguration loggingConfiguration, int maxDegreeParallelism, ITempCleaner tempDirectoryCleaner = null) { m_cancellationToken = cancellationToken; m_loggingContext = loggingContext; m_loggingConfiguration = loggingConfiguration; m_maxDegreeParallelism = maxDegreeParallelism; m_tempDirectoryCleaner = tempDirectoryCleaner; }
/// <inheritdoc /> public Possible <string, DeletionFailure> TryDeleteFile( string path, bool waitUntilDeletionFinished = true, ITempCleaner tempDirectoryCleaner = null) { try { DeleteFile(path, waitUntilDeletionFinished, tempDirectoryCleaner); return(path); } catch (BuildXLException ex) { return(new DeletionFailure(path, ex)); } }
/// <inheritdoc /> public Possible <Unit, RecoverableExceptionFailure> TryDeleteFile( string path, bool waitUntilDeletionFinished = true, ITempCleaner tempDirectoryCleaner = null) { try { DeleteFile(path, waitUntilDeletionFinished, tempDirectoryCleaner); return(Unit.Void); } catch (BuildXLException ex) { return(new RecoverableExceptionFailure(ex)); } }
/// <summary> /// Constructor /// </summary> public EngineSerializer( LoggingContext loggingContext, string engineCacheLocation, FileEnvelopeId?correlationId = null, bool useCompression = false, bool debug = false, bool readOnly = false, FileSystemStreamProvider readStreamProvider = null, ITempCleaner tempDirectoryCleaner = null) { Contract.Requires(loggingContext != null); Contract.Requires(engineCacheLocation != null); Contract.Requires(Path.IsPathRooted(engineCacheLocation)); Contract.Requires(!string.IsNullOrWhiteSpace(engineCacheLocation)); LoggingContext = loggingContext; m_engineCacheLocation = engineCacheLocation; m_debug = debug; m_correlationId = correlationId; m_useCompression = useCompression; m_readStreamProvider = readStreamProvider ?? FileSystemStreamProvider.Default; m_tempDirectoryCleaner = tempDirectoryCleaner; if (!readOnly) { try { FileUtilities.CreateDirectoryWithRetry(engineCacheLocation); } catch (Exception ex) { ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); throw new BuildXLException("Unable to create engine serializer cache directory: ", ex); } } }
/// <summary> /// Load symlink definitions serialized using <see cref="PathMapSerializer"/> /// </summary> public static async Task <Possible <SymlinkDefinitions> > TryLoadAsync( LoggingContext loggingContext, PathTable pathTable, string filePath, string symlinksDebugPath, ITempCleaner tempDirectoryCleaner = null) { try { var pathMap = await PathMapSerializer.LoadAsync(filePath, pathTable); var definitions = new SymlinkDefinitions(pathTable, pathMap); Logger.Log.SymlinkFileTraceMessage( loggingContext, I($"Loaded symlink definitions with {definitions.m_symlinkDefinitionMap.Count} entries and {definitions.m_directorySymlinkContents.Count} directories.")); if (EngineEnvironmentSettings.DebugSymlinkDefinitions && symlinksDebugPath != null) { FileUtilities.DeleteFile(symlinksDebugPath, tempDirectoryCleaner: tempDirectoryCleaner); using (var writer = new StreamWriter(symlinksDebugPath)) { foreach (var entry in pathMap) { writer.WriteLine("Source: {0}", entry.Key.ToString(pathTable)); writer.WriteLine("Target: {0}", entry.Value.ToString(pathTable)); } } } return(definitions); } catch (Exception ex) { Logger.Log.FailedLoadSymlinkFile(loggingContext, ex.GetLogEventMessage()); return(new Failure <string>("Failed loading symlink definition file")); } }
private int DeleteDirectoryContentsInternal( string path, bool deleteRootDirectory, Func <string, bool> shouldDelete, ITempCleaner tempDirectoryCleaner, CancellationToken?cancellationToken) { int remainingChildCount = 0; if (!Directory.Exists(path)) { return(remainingChildCount); } shouldDelete = shouldDelete ?? (p => true); EnumerateDirectoryResult result = s_fileSystem.EnumerateDirectoryEntries( path, (name, attributes) => { cancellationToken?.ThrowIfCancellationRequested(); var isDirectory = FileUtilities.IsDirectoryNoFollow(attributes); string childPath = Path.Combine(path, name); if (isDirectory) { int subDirectoryCount = DeleteDirectoryContentsInternal( childPath, deleteRootDirectory: true, shouldDelete: shouldDelete, tempDirectoryCleaner: tempDirectoryCleaner, cancellationToken: cancellationToken); if (subDirectoryCount > 0) { ++remainingChildCount; } } else { if (shouldDelete(childPath)) { // This method already has retry logic, so no need to do retry in DeleteFile DeleteFile(childPath, waitUntilDeletionFinished: true, tempDirectoryCleaner: tempDirectoryCleaner); } else { ++remainingChildCount; } } }, isEnumerationForDirectoryDeletion: true); if (deleteRootDirectory && remainingChildCount == 0) { bool success = false; success = Helpers.RetryOnFailure( finalRound => { // An exception will be thrown on failure, which will trigger a retry, this deletes the path itself // and any file or dir still in recursively through the 'true' flag Directory.Delete(path, true); // Only reached if there are no exceptions return(true); }); if (!success) { throw new BuildXLException(path); } } return(remainingChildCount); }
/// <summary> /// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over /// any state from any previous run such as the cache /// </summary> public ScheduleRunResult RunScheduler( SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, ITempCleaner tempCleaner = null, IEnumerable <(Pip before, Pip after)> constraintExecutionOrder = null,
private int DeleteDirectoryContentsInternal( string path, bool deleteRootDirectory, Func <string, bool> shouldDelete, ITempCleaner tempDirectoryCleaner, bool bestEffort, CancellationToken?cancellationToken) { int remainingChildCount = 0; if (!Directory.Exists(path)) { return(remainingChildCount); } shouldDelete = shouldDelete ?? (p => true); EnumerateDirectoryResult result = m_fileSystem.EnumerateDirectoryEntries( path, (name, attributes) => { cancellationToken?.ThrowIfCancellationRequested(); var isDirectory = FileUtilities.IsDirectoryNoFollow(attributes); string childPath = Path.Combine(path, name); if (isDirectory) { int subDirectoryCount = DeleteDirectoryContentsInternal( childPath, deleteRootDirectory: true, shouldDelete: shouldDelete, tempDirectoryCleaner: tempDirectoryCleaner, bestEffort: bestEffort, cancellationToken: cancellationToken); if (subDirectoryCount > 0) { ++remainingChildCount; } } else { if (shouldDelete(childPath)) { // This method already has retry logic, so no need to do retry in DeleteFile DeleteFile(childPath, retryOnFailure: !bestEffort, tempDirectoryCleaner: tempDirectoryCleaner); } else { ++remainingChildCount; } } }, isEnumerationForDirectoryDeletion: true); if (deleteRootDirectory && remainingChildCount == 0) { bool success = Helpers.RetryOnFailure( finalRound => { // An exception will be thrown on failure, which will trigger a retry, this deletes the path itself // and any file or dir still in recursively through the 'true' flag Directory.Delete(path, true); // Only reached if there are no exceptions return(true); }, numberOfAttempts: bestEffort ? 1 : Helpers.DefaultNumberOfAttempts); if (!success && Directory.Exists(path)) { var code = (int)Tracing.LogEventId.RetryOnFailureException; throw new BuildXLException($"Failed to delete directory: {path}. Search for DX{code:0000} log messages to see why."); } } return(remainingChildCount); }
/// <summary> /// Loads configured symlink definitions (if not already loaded) /// Stores to cache for use by workers in distributed build /// Eagerly creates symlinks if lazy symlink creation is disabled /// </summary> public static async Task <Possible <SymlinkDefinitions> > TryPrepareSymlinkDefinitionsAsync( LoggingContext loggingContext, GraphReuseResult reuseResult, IConfiguration configuration, MasterService masterService, CacheInitializationTask cacheInitializerTask, PipExecutionContext context, ITempCleaner tempDirectoryCleaner = null) { var pathTable = context.PathTable; bool isDistributedMaster = configuration.Distribution.BuildRole == DistributedBuildRoles.Master; Possible <SymlinkDefinitions> maybeSymlinkDefinitions = new Possible <SymlinkDefinitions>((SymlinkDefinitions)null); if (reuseResult?.IsFullReuse == true) { maybeSymlinkDefinitions = reuseResult.EngineSchedule.Scheduler.SymlinkDefinitions; } else if (configuration.Layout.SymlinkDefinitionFile.IsValid) { var symlinkFilePath = configuration.Layout.SymlinkDefinitionFile.ToString(pathTable); Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Loading symlink file from location '{symlinkFilePath}'.")); maybeSymlinkDefinitions = await SymlinkDefinitions.TryLoadAsync( loggingContext, pathTable, symlinkFilePath, symlinksDebugPath : configuration.Logging.LogsDirectory.Combine(pathTable, "DebugSymlinksDefinitions.log").ToString(pathTable), tempDirectoryCleaner : tempDirectoryCleaner); } if (!maybeSymlinkDefinitions.Succeeded || maybeSymlinkDefinitions.Result == null) { return(maybeSymlinkDefinitions); } // Need to store symlinks to cache for workers if (configuration.Distribution.BuildRole == DistributedBuildRoles.Master) { var possibleCacheInitializer = await cacheInitializerTask; if (!possibleCacheInitializer.Succeeded) { return(possibleCacheInitializer.Failure); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Storing symlink file for use by workers.")); var symlinkFile = configuration.Layout.SymlinkDefinitionFile.Expand(pathTable); var possibleStore = await TryStoreToCacheAsync( loggingContext, cache : possibleCacheInitializer.Result.CreateCacheForContext().ArtifactContentCache, symlinkFile : symlinkFile); if (!possibleStore.Succeeded) { return(possibleStore.Failure); } masterService.SymlinkFileContentHash = possibleStore.Result; Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file for use by workers.")); } if (!configuration.Schedule.UnsafeLazySymlinkCreation || configuration.Engine.PopulateSymlinkDirectories.Count != 0) { // Symlink definition file is defined, and BuildXL intends to create it eagerly. // At this point master and worker should have had its symlink definition file, if specified. if (!FileContentManager.CreateSymlinkEagerly(loggingContext, configuration, pathTable, maybeSymlinkDefinitions.Result, context.CancellationToken)) { return(new Failure <string>("Failed eagerly creating symlinks")); } } return(maybeSymlinkDefinitions); }
/// <summary> /// Cleans output files and directories. /// </summary> public static bool DeleteOutputs( LoggingContext loggingContext, Func <DirectoryArtifact, bool> isOutputDir, IList <FileOrDirectoryArtifact> filesOrDirectoriesToDelete, PathTable pathTable, ITempCleaner tempDirectoryCleaner = null) { int fileFailCount = 0; int fileSuccessCount = 0; int directoryFailCount = 0; int directorySuccessCount = 0; using (PerformanceMeasurement.Start( loggingContext, Category, Tracing.Logger.Log.CleaningStarted, localLoggingContext => { Tracing.Logger.Log.CleaningFinished(loggingContext, fileSuccessCount, fileFailCount); LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "FilesDeleted", fileSuccessCount); LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "FilesFailed", fileFailCount); LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "DirectoriesDeleted", directorySuccessCount); LoggingHelpers.LogCategorizedStatistic(loggingContext, Category, "DirectoriesFailed", directoryFailCount); })) { // Note: filesOrDirectoriesToDelete better be an IList<...> in order to get good Parallel.ForEach performance Parallel.ForEach( filesOrDirectoriesToDelete, fileOrDirectory => { string path = fileOrDirectory.Path.ToString(pathTable); Tracing.Logger.Log.CleaningOutputFile(loggingContext, path); try { if (fileOrDirectory.IsFile) { Contract.Assume(fileOrDirectory.FileArtifact.IsOutputFile, "Encountered non-output file"); if (FileUtilities.FileExistsNoFollow(path)) { FileUtilities.DeleteFile(path, waitUntilDeletionFinished: true, tempDirectoryCleaner: tempDirectoryCleaner); Interlocked.Increment(ref fileSuccessCount); } } else { if (FileUtilities.DirectoryExistsNoFollow(path)) { // TODO:1011977 this is a hacky fix for a bug where we delete SourceSealDirectories in /cleanonly mode // The bug stems from the fact that FilterOutputs() returns SourceSealDirectories, which aren't inputs. // Once properly addressed, this check should remain in here as a safety precaution and turn into // a Contract.Assume() like the check above if (isOutputDir(fileOrDirectory.DirectoryArtifact)) { FileUtilities.DeleteDirectoryContents(path, deleteRootDirectory: false, tempDirectoryCleaner: tempDirectoryCleaner); Interlocked.Increment(ref directorySuccessCount); } } } } catch (BuildXLException ex) { if (fileOrDirectory.IsFile) { Interlocked.Increment(ref fileFailCount); Tracing.Logger.Log.CleaningFileFailed(loggingContext, path, ex.LogEventMessage); } else { Interlocked.Increment(ref directoryFailCount); Tracing.Logger.Log.CleaningDirectoryFailed(loggingContext, path, ex.LogEventMessage); } } }); } return(fileFailCount + directoryFailCount == 0); }