public void TempCleanerCleanRootDirectory(bool deleteRoot)
{
string moveDeletionTemp = Path.Combine(TemporaryDirectory, "MoveDeletionTemp");
Directory.CreateDirectory(moveDeletionTemp);
string directoryToBeDeleted = Path.Combine(TemporaryDirectory, Guid.NewGuid().ToString());
Directory.CreateDirectory(directoryToBeDeleted);
string deletedFile = Path.Combine(directoryToBeDeleted, "file");
File.WriteAllText(deletedFile, "asdf");
string movedFile = Path.Combine(TemporaryDirectory, "movedFile");
File.WriteAllText(movedFile, "asdf");
// Create a temp cleaner with a temp directory
using (TempCleaner cleaner = new TempCleaner(moveDeletionTemp))
{
// Move-delete a file into the TempCleaner temp directory
FileUtilities.TryMoveDelete(movedFile, cleaner.TempDirectory);
cleaner.RegisterDirectoryToDelete(directoryToBeDeleted, deleteRoot);
cleaner.WaitPendingTasksForCompletion();
}
XAssert.AreEqual(!deleteRoot, Directory.Exists(directoryToBeDeleted));
XAssert.IsTrue(Directory.Exists(moveDeletionTemp));
}
public void CleanTempCleanerTempDirectory()
{
// Make a temp directory for TempCleaner
string deletionTemp = Path.Combine(TemporaryDirectory, "DeletionTemp");
Directory.CreateDirectory(deletionTemp);
// Make a file outside of TempCleaner temp directory
string deletedFile = Path.Combine(TemporaryDirectory, Guid.NewGuid().ToString());
File.WriteAllText(deletedFile, "asdf");
// Create a temp cleaner with a temp directory
using (TempCleaner cleaner = new TempCleaner(deletionTemp))
{
// Move-delete a file into the TempCleaner temp directory
FileUtilities.TryMoveDelete(deletedFile, cleaner.TempDirectory);
cleaner.WaitPendingTasksForCompletion();
XAssert.AreEqual(1, cleaner.SucceededDirectories);
}
// TempCleaner should clean out \deletionTemp before or during Dispose
XAssert.IsFalse(File.Exists(deletedFile));
}
public TestScheduler(
PipGraph graph,
TestPipQueue pipQueue,
PipExecutionContext context,
FileContentTable fileContentTable,
EngineCache cache,
IConfiguration configuration,
FileAccessWhitelist fileAccessWhitelist,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
TempCleaner tempCleaner = null,
PipRuntimeTimeTable runningTimeTable = null,
JournalState journalState = null,
PerformanceCollector performanceCollector = null,
string fingerprintSalt = null,
ContentHash?previousInputsSalt = null,
IEnumerable <Pip> successfulPips = null,
IEnumerable <Pip> failedPips = null,
LoggingContext loggingContext = null,
IIpcProvider ipcProvider = null,
DirectoryTranslator directoryTranslator = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache,
configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules,
tempCleaner, Task.FromResult <PipRuntimeTimeTable>(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
ipcProvider: ipcProvider,
directoryTranslator: directoryTranslator,
journalState: journalState,
vmInitializer: vmInitializer,
testHooks: testHooks)
{
m_testPipQueue = pipQueue;
if (successfulPips != null)
{
foreach (var pip in successfulPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
}
}
if (failedPips != null)
{
foreach (var pip in failedPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
}
}
m_loggingContext = loggingContext;
}
public static void Register(string dir)
{
try
{
if (!_WatchedDirectories.ContainsKey(dir))
{
lock (_WatchedDirectories)
if (!_WatchedDirectories.ContainsKey(dir))
{
TempCleaner c = new TempCleaner(dir);
_CleanerPool.BeginAsync(c);
_WatchedDirectories[dir] = dir;
}
}
}
catch { }
}
public void CleanNonexisting()
{
RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
string baseDir = TemporaryDirectory;
string dir = Path.Combine(baseDir, "DirDoesntExist");
string file = Path.Combine(baseDir, "FileDoesntExist.txt");
using (TempCleaner cleaner = new TempCleaner())
{
cleaner.RegisterDirectoryToDelete(dir, deleteRootDirectory: false);
cleaner.RegisterFileToDelete(file);
cleaner.WaitPendingTasksForCompletion();
}
// No warnings for missing file/folder
m_expectedWarningCount = 0;
}
public void CleanTempDirsAndFilesWithOpenedFile()
{
// Arrange
Tuple <string, string> dirs = CreateDirs();
string file = Path.Combine(dirs.Item1, "openedFile.txt");
// Warning for a failed deleted file
IgnoreWarnings();
using (StreamWriter writer = new StreamWriter(file))
{
writer.Write("asdf");
writer.Flush();
XAssert.IsTrue(File.Exists(file));
// Act
using (TempCleaner cleaner = new TempCleaner())
{
// This should fail
cleaner.RegisterDirectoryToDelete(dirs.Item1, deleteRootDirectory: false);
cleaner.RegisterDirectoryToDelete(dirs.Item2, deleteRootDirectory: false);
// This should fail
cleaner.RegisterFileToDelete(file);
// Waiting for pending tasks to complete
cleaner.WaitPendingTasksForCompletion();
// Assert
XAssert.AreEqual(1, cleaner.SucceededDirectories);
XAssert.AreEqual(0, cleaner.PendingDirectories);
XAssert.AreEqual(1, cleaner.FailedDirectories);
XAssert.AreEqual(0, cleaner.SucceededFiles);
XAssert.AreEqual(0, cleaner.PendingFiles);
XAssert.AreEqual(1, cleaner.FailedFiles);
}
XAssert.IsTrue(File.Exists(file));
}
AssertDirectoryEmpty(dirs.Item2);
}
public void CheckCleanTempDirDependingOnPipResult(bool shouldPipFail)
{
Configuration.Engine.CleanTempDirectories = true;
var tempdir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory());
var tempdirStr = ArtifactToString(tempdir);
var file = CreateOutputFileArtifact(tempdirStr);
var fileStr = ArtifactToString(file);
var operations = new List <Operation>()
{
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.WriteFile(file, doNotInfer: true),
};
if (shouldPipFail)
{
operations.Add(Operation.Fail());
}
var builder = CreatePipBuilder(operations);
builder.SetTempDirectory(tempdir);
SchedulePipBuilder(builder);
using (var tempCleaner = new TempCleaner())
{
if (shouldPipFail)
{
RunScheduler(tempCleaner: tempCleaner).AssertFailure();
AssertErrorEventLogged(EventId.PipProcessError);
tempCleaner.WaitPendingTasksForCompletion();
XAssert.IsTrue(Directory.Exists(tempdirStr), $"TEMP directory deleted but wasn't supposed to: {tempdirStr}");
XAssert.IsTrue(File.Exists(fileStr), $"Temp file deleted but wasn't supposed to: {fileStr}");
}
else
{
RunScheduler(tempCleaner: tempCleaner).AssertSuccess();
tempCleaner.WaitPendingTasksForCompletion();
XAssert.IsFalse(File.Exists(fileStr), $"Temp file not deleted: {fileStr}");
XAssert.IsFalse(Directory.Exists(tempdirStr), $"TEMP directory not deleted: {tempdirStr}");
}
}
}
public void CleanTempDirsAndFiles()
{
// Arrange
Tuple <string, string> dirs = CreateDirs();
string tempFile = Path.Combine(dirs.Item1, "SomeTemporaryFile.txt");
PrepareFileWithConent(tempFile, "sampleContent");
// Act
using (TempCleaner cleaner = new TempCleaner())
{
cleaner.RegisterDirectoryToDelete(dirs.Item1, deleteRootDirectory: false);
cleaner.RegisterDirectoryToDelete(dirs.Item2, deleteRootDirectory: false);
cleaner.RegisterFileToDelete(tempFile);
// Registering file that is not exists
string notExistedPath = Path.Combine(dirs.Item1, "UnknownFile.txt");
XAssert.IsFalse(File.Exists(notExistedPath));
cleaner.RegisterFileToDelete(notExistedPath);
// Waiting for pending tasks to complete
cleaner.WaitPendingTasksForCompletion();
// Assert
XAssert.AreEqual(2, cleaner.SucceededDirectories);
XAssert.AreEqual(0, cleaner.PendingDirectories);
XAssert.AreEqual(0, cleaner.FailedDirectories);
XAssert.AreEqual(2, cleaner.SucceededFiles);
XAssert.AreEqual(0, cleaner.PendingFiles);
XAssert.AreEqual(0, cleaner.FailedFiles);
}
AssertDirectoryEmpty(dirs.Item1);
AssertDirectoryEmpty(dirs.Item2);
}
/// <summary>
/// Runs the scheduler allowing various options to be specifically set
/// </summary>
public ScheduleRunResult RunSchedulerSpecific(
PipGraph graph,
SchedulerTestHooks testHooks = null,
SchedulerState schedulerState = null,
RootFilter filter = null,
TempCleaner tempCleaner = null)
{
var config = new CommandLineConfiguration(Configuration);
// Populating the configuration may modify the configuration, so it should occur first.
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext);
FileAccessWhitelist whitelist = new FileAccessWhitelist(Context);
whitelist.Initialize(config);
IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList();
var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots);
var optionalAccessor = TryGetJournalAccessor(map);
// Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one.
m_journalState = map == null || !optionalAccessor.IsValid
? JournalState.DisabledJournal
: JournalState.CreateEnabledJournal(map, optionalAccessor.Value);
if (config.Schedule.IncrementalScheduling)
{
// Ensure that we can scan the journal when incremental scheduling is enabled.
XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled");
}
// Seal the translator if not sealed
DirectoryTranslator.Seal();
// .....................................................................................
// some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt
// .....................................................................................
string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache");
Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here
ContentHash?previousOutputsSalt =
EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config);
Contract.Assert(previousOutputsSalt.HasValue);
// .....................................................................................
testHooks = testHooks ?? new SchedulerTestHooks();
Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null));
using (var queue = new PipQueue(config.Schedule))
using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false))
using (var testScheduler = new TestScheduler(
graph: graph,
pipQueue: testQueue,
context: Context,
fileContentTable: FileContentTable,
loggingContext: LoggingContext,
cache: Cache,
configuration: config,
journalState: m_journalState,
fileAccessWhitelist: whitelist,
fingerprintSalt: Configuration.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable),
tempCleaner: tempCleaner,
previousInputsSalt: previousOutputsSalt.Value,
successfulPips: null,
failedPips: null,
ipcProvider: null,
directoryTranslator: DirectoryTranslator,
testHooks: testHooks))
{
if (filter == null)
{
EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter);
}
XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler");
testScheduler.Start(LoggingContext);
bool success = testScheduler.WhenDone().GetAwaiter().GetResult();
testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait();
return(new ScheduleRunResult
{
Graph = graph,
Config = config,
Success = success,
PipResults = testScheduler.PipResults,
PipExecutorCounters = testScheduler.PipExecutionCounters,
PathSets = testScheduler.PathSets,
ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter,
ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag,
SchedulerState = new SchedulerState(testScheduler)
});
}
}
/// <summary>
/// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over
/// any state from any previous run such as the cache
/// </summary>
public ScheduleRunResult RunScheduler(SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null)
{
if (m_graphWasModified || m_lastGraph == null)
{
m_lastGraph = PipGraphBuilder.Build();
XAssert.IsNotNull(m_lastGraph, "Failed to build pip graph");
}
m_graphWasModified = false;
return(RunSchedulerSpecific(m_lastGraph, testHooks, schedulerState, filter, tempCleaner));
}
/// <summary> /// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over /// any state from any previous run such as the cache /// </summary> public ScheduleRunResult RunScheduler(SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null, IEnumerable <(Pip before, Pip after)> constraintExecutionOrder = null)
static void _Act(string source, string destination, FileExistsAction ifExists, out string destinationFilename, int retryCount, int retryDelaySeconds, bool useTempHop, bool isMove)
{
if (String.IsNullOrEmpty(source))
{
throw new ArgumentNullException(nameof(source));
}
if (String.IsNullOrEmpty(destination))
{
throw new ArgumentNullException(nameof(destination));
}
source = STEM.Sys.IO.Path.ChangeIpToMachineName(STEM.Sys.IO.Path.AdjustPath(source));
destination = STEM.Sys.IO.Path.ChangeIpToMachineName(STEM.Sys.IO.Path.AdjustPath(destination));
if (source.Equals(destination, StringComparison.InvariantCultureIgnoreCase))
{
destinationFilename = destination;
return;
}
destinationFilename = "";
if (isMove)
{
string sourceAddress = STEM.Sys.IO.Net.MachineAddress(STEM.Sys.IO.Path.FirstTokenOfPath(source));
string destinationAddress = STEM.Sys.IO.Net.MachineAddress(STEM.Sys.IO.Path.FirstTokenOfPath(destination));
if (sourceAddress == destinationAddress)
{
string sp = STEM.Sys.IO.Path.AdjustPath(source).TrimStart('\\').TrimStart('/');
if (sp.IndexOf(System.IO.Path.DirectorySeparatorChar) != -1)
{
sp = sp.Substring(sp.IndexOf(System.IO.Path.DirectorySeparatorChar));
}
sp = STEM.Sys.IO.Path.FirstTokenOfPath(sp);
string dp = STEM.Sys.IO.Path.AdjustPath(destination).TrimStart('\\').TrimStart('/');
if (dp.IndexOf(System.IO.Path.DirectorySeparatorChar) != -1)
{
dp = dp.Substring(dp.IndexOf(System.IO.Path.DirectorySeparatorChar));
}
dp = STEM.Sys.IO.Path.FirstTokenOfPath(dp);
if (sp.Equals(dp, StringComparison.InvariantCultureIgnoreCase))
{
useTempHop = false;
}
}
}
while (retryCount >= 0)
{
if (!System.IO.File.Exists(source))
{
throw new System.IO.FileNotFoundException("Could not find file " + source);
}
string d = destination;
if (useTempHop)
{
string dTemp = System.IO.Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(d), "Temp");
if (!System.IO.Directory.Exists(dTemp))
{
System.IO.Directory.CreateDirectory(dTemp);
}
TempCleaner.Register(dTemp);
}
else
{
if (!System.IO.Directory.Exists(STEM.Sys.IO.Path.GetDirectoryName(d)))
{
System.IO.Directory.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(d));
}
}
try
{
if (useTempHop)
{
string dTemp = System.IO.Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(d), "Temp");
if (!System.IO.Directory.Exists(dTemp))
{
System.IO.Directory.CreateDirectory(dTemp);
}
dTemp = System.IO.Path.Combine(dTemp, Guid.NewGuid() + ".tmp");
try
{
if (System.IO.File.Exists(d))
{
switch (ifExists)
{
case FileExistsAction.Throw:
throw new System.IO.IOException("Destination file already exists: " + destination);
case FileExistsAction.Skip:
return;
case FileExistsAction.Overwrite:
System.IO.File.Copy(source, dTemp, true);
try
{
System.IO.File.SetAttributes(d, System.IO.FileAttributes.Normal);
System.IO.File.Copy(dTemp, d, true);
}
catch (Exception ex)
{
throw new System.IO.IOException("Destination file could not be overwritten: " + d, ex);
}
break;
case FileExistsAction.OverwriteIfNewer:
if (System.IO.File.GetLastWriteTimeUtc(d) >= System.IO.File.GetLastWriteTimeUtc(source))
{
return;
}
System.IO.File.Copy(source, dTemp, true);
try
{
System.IO.File.SetAttributes(d, System.IO.FileAttributes.Normal);
System.IO.File.Copy(dTemp, d, true);
}
catch (Exception ex)
{
throw new System.IO.IOException("Destination file could not be overwritten: " + d, ex);
}
break;
case FileExistsAction.MakeUnique:
d = File.UniqueFilename(destination);
System.IO.File.Copy(source, dTemp, true);
System.IO.File.Move(dTemp, d);
break;
}
}
else
{
System.IO.File.Copy(source, dTemp, true);
System.IO.File.Move(dTemp, d);
}
destinationFilename = d;
if (isMove)
{
while (retryCount >= 0)
{
try
{
System.IO.File.Delete(source);
break;
}
catch
{
if (retryCount <= 0)
{
throw;
}
}
if (retryCount > 0)
{
System.Threading.Thread.Sleep(retryDelaySeconds * 1000);
}
retryCount--;
}
}
}
finally
{
int retry = 3;
while (retry-- > 0)
{
try
{
if (System.IO.File.Exists(dTemp))
{
System.IO.File.Delete(dTemp);
}
break;
}
catch
{
if (retry > 0)
{
System.Threading.Thread.Sleep(1000);
}
}
}
}
}
else
{
if (System.IO.File.Exists(d))
{
switch (ifExists)
{
case FileExistsAction.Throw:
throw new System.IO.IOException("Destination file already exists: " + destination);
case FileExistsAction.Skip:
return;
case FileExistsAction.Overwrite:
System.IO.File.SetAttributes(d, System.IO.FileAttributes.Normal);
System.IO.File.Copy(source, d, true);
if (isMove)
{
while (retryCount >= 0)
{
try
{
System.IO.File.Delete(source);
break;
}
catch
{
if (retryCount <= 0)
{
throw;
}
}
if (retryCount > 0)
{
System.Threading.Thread.Sleep(retryDelaySeconds * 1000);
}
retryCount--;
}
}
break;
case FileExistsAction.OverwriteIfNewer:
if (System.IO.File.GetLastWriteTimeUtc(d) >= System.IO.File.GetLastWriteTimeUtc(source))
{
return;
}
System.IO.File.SetAttributes(d, System.IO.FileAttributes.Normal);
System.IO.File.Copy(source, d, true);
if (isMove)
{
while (retryCount >= 0)
{
try
{
System.IO.File.Delete(source);
break;
}
catch
{
if (retryCount <= 0)
{
throw;
}
}
if (retryCount > 0)
{
System.Threading.Thread.Sleep(retryDelaySeconds * 1000);
}
retryCount--;
}
}
break;
case FileExistsAction.MakeUnique:
d = File.UniqueFilename(destination);
if (isMove)
{
System.IO.File.Move(source, d);
}
else
{
System.IO.File.Copy(source, d);
}
break;
}
}
else
{
if (isMove)
{
System.IO.File.Move(source, d);
}
else
{
System.IO.File.Copy(source, d);
}
}
destinationFilename = d;
}
if (destinationFilename == d)
{
break;
}
}
catch
{
if (retryCount <= 0)
{
throw;
}
}
if (retryCount > 0)
{
System.Threading.Thread.Sleep(retryDelaySeconds * 1000);
}
retryCount--;
}
if (String.IsNullOrEmpty(destinationFilename))
{
if (isMove)
{
throw new Exception("File move failed: " + source);
}
else
{
throw new Exception("File copy failed: " + source);
}
}
}
