public void DirectorySymlinksAreTraversed()
{
string rootDir = Path.Combine(TemporaryDirectory, nameof(DirectorySymlinksAreTraversed));
string fullTargetDirPath = Path.Combine(rootDir, "target-dir");
Directory.CreateDirectory(fullTargetDirPath);
var fileUnderTarget = Path.Combine(fullTargetDirPath, "file.txt");
File.WriteAllText(fileUnderTarget, "content");
string rootToScrub = Path.Combine(rootDir, "root-to-scrub");
Directory.CreateDirectory(rootToScrub);
string fullSymlinkPath = WriteSymlink(Path.Combine(rootToScrub, "directory symlink"), fullTargetDirPath, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(fullSymlinkPath));
// Scrub starting on a root dir that contains a symlink directory. We should be able to find the file underneath and delete it
// We consider all artifacts to be not part of the build, so symlink directories are followed and all files are deleted
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => false,
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.IsFalse(File.Exists(fileUnderTarget));
XAssert.Equals(OperatingSystemHelper.IsMacOS, !Directory.Exists(fullSymlinkPath));
}
public void DeleteFilesCancellationDoesNotCrash()
{
const int FileDeletionsAllowed = 2;
var testHook = new DirectoryScrubber.TestHooks
{
OnDeletion = new Action <string, int>((path, numDeletedSoFar) =>
{
if (numDeletedSoFar > FileDeletionsAllowed)
{
m_cancellationTokenSource.Cancel();
}
})
};
string rootDir = Path.Combine(TemporaryDirectory, nameof(DeleteFilesCanDeleteFile));
List <string> files = new List <string>();
for (var i = 0; i < FileDeletionsAllowed * 3; i++)
{
string fullFilePath = WriteFile(Path.Combine(rootDir, $"out{i}.txt"));
XAssert.FileExists(fullFilePath);
files.Add(fullFilePath);
}
XAssert.IsFalse(m_cancellationTokenSource.IsCancellationRequested);
var numDeleted = Scrubber.DeleteFiles(files.ToArray(), testHook: testHook);
XAssert.IsTrue(m_cancellationTokenSource.IsCancellationRequested);
XAssert.Equals(FileDeletionsAllowed, numDeleted);
}
public void ABTestingOptionRelatedActivityIdSeed()
{
PathTable pt = new PathTable();
var argsParser = new Args();
string relatedActivityArg = "/relatedActivityId:cd0adef6-abef-4990-8cde-32441a54f747";
string abTestingArg1 = "/abTesting:Id1=\"/maxProc:5\"";
string abTestingArg2 = "/abTesting:Id2=\"/maxProc:10\"";
string[] args = new[] { @"/c:" + m_specFilePath, relatedActivityArg, abTestingArg1, abTestingArg2 };
XAssert.IsTrue(argsParser.TryParse(args, pt, out var config1));
var chosen = config1.Startup.ChosenABTestingKey;
XAssert.IsNotNull(chosen);
XAssert.IsTrue(argsParser.TryParse(args, pt, out var config2));
XAssert.Equals(chosen, config2.Startup.ChosenABTestingKey);
XAssert.IsTrue(argsParser.TryParse(args, pt, out var config3));
XAssert.Equals(chosen, config3.Startup.ChosenABTestingKey);
XAssert.IsTrue(argsParser.TryParse(args, pt, out var config4));
XAssert.Equals(chosen, config4.Startup.ChosenABTestingKey);
}
public void BlockExclusionFlagIsHonored(bool blockWritesUnderNodeModules)
{
var project = CreateRushProject();
var exclusions = Start(new RushResolverSettings {
BlockWritesUnderNodeModules = blockWritesUnderNodeModules
})
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.OutputDirectoryExclusions;
XAssert.Equals(blockWritesUnderNodeModules? 1 : 0, exclusions.Length);
}
public void SymlinkDirectoriesDoNotIntroduceDuplicateWork()
{
// We are creating this layout
//
// root
// -- real-dir
// -- symlink-dir -> real-dir
// --file.txt
//
// so there is in fact a cycle, where root/real-dir/symlink-dir/.../symlink-dir/... is a valid path
// The fact that scrubbing finishes proves we are deduping work
string rootDir = Path.Combine(TemporaryDirectory, nameof(SymlinkDirectoriesDoNotIntroduceDuplicateWork));
string realDirectory = Path.Combine(rootDir, "real-dir");
Directory.CreateDirectory(realDirectory);
string symlinkDirectory = WriteSymlink(Path.Combine(realDirectory, "symlink-dir"), rootDir, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(symlinkDirectory));
var fileUnderSymlinkDir = Path.Combine(symlinkDirectory, "file.txt");
File.WriteAllText(fileUnderSymlinkDir, "content");
try
{
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => false,
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.IsFalse(File.Exists(fileUnderSymlinkDir));
XAssert.Equals(!OperatingSystemHelper.IsMacOS, Directory.Exists(realDirectory));
XAssert.Equals(!OperatingSystemHelper.IsMacOS, Directory.Exists(symlinkDirectory));
}
finally
{
// On Windows, the temp directory cleaner has problems with cycles. So let's remove the symlink dir explicitly here
if (!OperatingSystemHelper.IsMacOS)
{
Directory.Delete(symlinkDirectory);
}
}
}
public void PredictedInputsUnderUntrackedDirectoriesAreSkipped()
{
var project = CreateProjectWithPredictions(inputs: CreatePath(@"untracked\input.txt", "input2.txt"));
var processInputs = Start(new MsBuildResolverSettings
{
UntrackedDirectories = CreatePath("untracked").Select(path => DirectoryArtifact.CreateWithZeroPartialSealId(path)).ToList()
})
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.Dependencies;
// The only source file (besides MSBuild.exe itself) should be input2
var input = processInputs.Single(i => (i.IsSourceFile && i.Path.GetName(PathTable) != PathAtom.Create(StringTable, "MSBuild.exe")));
XAssert.Equals("input2.txt", input.Path.GetName(PathTable).ToString(PathTable.StringTable));
}
public void PredictedInputsInKnownOutputDirectoriesAreSkipped()
{
var dependency = CreateProjectWithPredictions(outputs: CreatePath("OutDir"));
// We create 4 predicted inputs. 3 of them under predicted output directories. So only the last one should be added as a true input, the rest are assumed to be intermediates
var dependent = CreateProjectWithPredictions(
outputs: CreatePath("AnotherOutput"),
inputs: CreatePath(@"AnotherOutput\input.txt", @"OutDir\input1.txt", @"OutDir\nested\input2.txt", "input3.txt"),
references: new[] { dependency });
var processInputs = Start()
.Add(dependency)
.Add(dependent)
.ScheduleAll()
.RetrieveSuccessfulProcess(dependent)
.Dependencies;
// The only source file (besides MSBuild.exe itself) should be input3
var input = processInputs.Single(i => (i.IsSourceFile && i.Path.GetName(PathTable) != PathAtom.Create(StringTable, "MSBuild.exe")));
XAssert.Equals("input3.txt", input.Path.GetName(PathTable).ToString(PathTable.StringTable));
}
public void AllowedDoubleWriteCachesTheRightContent()
{
string sharedOpaqueDir = Path.Combine(ObjectRoot, "sharedopaquedir");
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
FileArtifact doubleWriteArtifact = CreateOutputFileArtifact(sharedOpaqueDir);
var firstProducerBuilder = CreateFileInSharedOpaqueBuilder(ContainerIsolationLevel.IsolateAllOutputs, RewritePolicy.UnsafeFirstDoubleWriteWins, doubleWriteArtifact, "first", sharedOpaqueDirPath);
var secondProducerBuilder = CreateFileInSharedOpaqueBuilder(ContainerIsolationLevel.IsolateAllOutputs, RewritePolicy.UnsafeFirstDoubleWriteWins, doubleWriteArtifact, "second", sharedOpaqueDirPath);
var firstProducer = SchedulePipBuilder(firstProducerBuilder);
var secondProducer = SchedulePipBuilder(secondProducerBuilder);
// Given the policy and isolation level, both producers should get cached
RunScheduler().AssertSuccess();
AssertWarningEventLogged(LogEventId.FileMonitoringWarning);
string doubleWritePath = doubleWriteArtifact.Path.ToString(Context.PathTable);
// Run the first producer alone. It should be a cache hit, and the content of the produced
// file should correspond to the first producer
RootFilter filter = new RootFilter(new PipIdFilter(firstProducer.Process.SemiStableHash));
var result = RunScheduler(filter: filter).AssertSuccess();
result.AssertCacheHit(firstProducer.Process.PipId);
XAssert.IsTrue(File.Exists(doubleWritePath));
XAssert.Equals("first", File.ReadAllText(doubleWritePath));
// Same procedure with the second producer
filter = new RootFilter(new PipIdFilter(secondProducer.Process.SemiStableHash));
result = RunScheduler(filter: filter).AssertSuccess();
result.AssertCacheHit(secondProducer.Process.PipId);
XAssert.IsTrue(File.Exists(doubleWritePath));
XAssert.Equals("second", File.ReadAllText(doubleWritePath));
}