public void DirectorySymlinksAreTraversed()
{
string rootDir = Path.Combine(TemporaryDirectory, nameof(DirectorySymlinksAreTraversed));
string fullTargetDirPath = Path.Combine(rootDir, "target-dir");
Directory.CreateDirectory(fullTargetDirPath);
var fileUnderTarget = Path.Combine(fullTargetDirPath, "file.txt");
File.WriteAllText(fileUnderTarget, "content");
string rootToScrub = Path.Combine(rootDir, "root-to-scrub");
Directory.CreateDirectory(rootToScrub);
string fullSymlinkPath = WriteSymlink(Path.Combine(rootToScrub, "directory symlink"), fullTargetDirPath, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(fullSymlinkPath));
// Scrub starting on a root dir that contains a symlink directory. We should be able to find the file underneath and delete it
// We consider all artifacts to be not part of the build, so symlink directories are followed and all files are deleted
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => false,
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.IsFalse(File.Exists(fileUnderTarget));
XAssert.Equals(OperatingSystemHelper.IsMacOS, !Directory.Exists(fullSymlinkPath));
}
public void ScrubbingDirectoriesWithMounts()
{
string rootDirectory = Path.Combine(TemporaryDirectory, nameof(ScrubbingDirectoriesWithMounts));
const string NonScrubbableMountName = "NonScrubbable";
string nonScrubbableMountPath = Path.Combine(rootDirectory, NonScrubbableMountName);
const string ScrubbableMountName = "Scrubbable";
string scrubbableMountPath = Path.Combine(rootDirectory, ScrubbableMountName);
const string NestedScrubbableMountName = "NestedScrubbable";
string nestedScrubbableMountPath = Path.Combine(scrubbableMountPath, NestedScrubbableMountName);
MountPathExpander mountPathExpander =
CreateMountPathExpander(
new TestMount(NonScrubbableMountName, nonScrubbableMountPath, MountFeatures.Writable | MountFeatures.Readable),
new TestMount(ScrubbableMountName, scrubbableMountPath, MountFeatures.Scrubbable),
new TestMount(NestedScrubbableMountName, nestedScrubbableMountPath, MountFeatures.Scrubbable));
string f = WriteFile(Path.Combine(nonScrubbableMountPath, "D", "f"));
string g1 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g1"));
string g2 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g2"));
string h = WriteFile(Path.Combine(scrubbableMountPath, "D", "E", "h"));
string i = WriteFile(Path.Combine(scrubbableMountPath, "D", "F", "i"));
string j = WriteFile(Path.Combine(nestedScrubbableMountPath, "D", "j"));
var inBuild = new HashSet <string>(OperatingSystemHelper.PathComparer)
{
Path.Combine(scrubbableMountPath, "D", "E"), g1
};
Scrubber.RemoveExtraneousFilesAndDirectories(
path => inBuild.Contains(path),
pathsToScrub: new[] { Path.Combine(nonScrubbableMountPath, "D"), scrubbableMountPath },
blockedPaths: new string[0],
nonDeletableRootDirectories: new string[0],
mountPathExpander: mountPathExpander);
// NonScrubbable\D produces a warning.
AssertWarningEventLogged(global::BuildXL.Engine.Tracing.LogEventId.ScrubbingFailedBecauseDirectoryIsNotScrubbable);
// f is in NonScrubbable.
XAssert.IsTrue(File.Exists(f));
// g1 & h (via Scrubbable\D\E) is in build.
XAssert.IsTrue(File.Exists(g1));
XAssert.IsTrue(File.Exists(h));
// NestedScrubbable, although not in build, but is a mount root.
XAssert.IsTrue(Directory.Exists(nestedScrubbableMountPath));
// Scrubbed files/directories.
XAssert.IsFalse(File.Exists(g2));
XAssert.IsFalse(File.Exists(i));
XAssert.IsFalse(Directory.Exists(Path.Combine(scrubbableMountPath, "D", "F")));
XAssert.IsFalse(File.Exists(j));
XAssert.IsFalse(Directory.Exists(Path.Combine(nestedScrubbableMountPath, "D")));
}
public void DoNotScrubDeclaredOutputDirectories()
{
string rootDirectory = Path.Combine(TemporaryDirectory, nameof(ScrubFilesAndDirectories));
string a = WriteFile(Path.Combine(rootDirectory, "1", "1", "out.txt"));
var inBuild = new HashSet <string>(OperatingSystemHelper.PathComparer)
{
Path.GetDirectoryName(a)
};
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => inBuild.Contains(path),
pathsToScrub: new[] { Path.GetDirectoryName(a) },
blockedPaths: new string[] { },
nonDeletableRootDirectories: new string[0]);
XAssert.IsTrue(File.Exists(a));
}
public void SymlinkDirectoriesDoNotIntroduceDuplicateWork()
{
// We are creating this layout
//
// root
// -- real-dir
// -- symlink-dir -> real-dir
// --file.txt
//
// so there is in fact a cycle, where root/real-dir/symlink-dir/.../symlink-dir/... is a valid path
// The fact that scrubbing finishes proves we are deduping work
string rootDir = Path.Combine(TemporaryDirectory, nameof(SymlinkDirectoriesDoNotIntroduceDuplicateWork));
string realDirectory = Path.Combine(rootDir, "real-dir");
Directory.CreateDirectory(realDirectory);
string symlinkDirectory = WriteSymlink(Path.Combine(realDirectory, "symlink-dir"), rootDir, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(symlinkDirectory));
var fileUnderSymlinkDir = Path.Combine(symlinkDirectory, "file.txt");
File.WriteAllText(fileUnderSymlinkDir, "content");
try
{
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => false,
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.IsFalse(File.Exists(fileUnderSymlinkDir));
XAssert.Equals(!OperatingSystemHelper.IsMacOS, Directory.Exists(realDirectory));
XAssert.Equals(!OperatingSystemHelper.IsMacOS, Directory.Exists(symlinkDirectory));
}
finally
{
// On Windows, the temp directory cleaner has problems with cycles. So let's remove the symlink dir explicitly here
if (!OperatingSystemHelper.IsMacOS)
{
Directory.Delete(symlinkDirectory);
}
}
}
public void DontScrubBlockedPathsEvenIfAskedTo()
{
string rootDirectory = Path.Combine(TemporaryDirectory, nameof(ScrubFilesAndDirectories));
string a = WriteFile(Path.Combine(rootDirectory, "a", "b", "c", "out.txt"));
var inBuild = new HashSet <string>(OperatingSystemHelper.PathComparer)
{
};
Scrubber.RemoveExtraneousFilesAndDirectories(
path => inBuild.Contains(path),
pathsToScrub: new[] { rootDirectory },
blockedPaths: new[] { rootDirectory },
nonDeletableRootDirectories: new string[0]);
// The file should still exist since it is under a blocked path
XAssert.IsTrue(File.Exists(a));
}
public void ScrubFilesAndDirectories()
{
// Create a layout with various paths. We will clean at the root dir.
// Some files will be in the build, some out, and some paths excluded
string rootDirectory = Path.Combine(TemporaryDirectory, nameof(ScrubFilesAndDirectories));
string a = WriteFile(Path.Combine(rootDirectory, "1", "1", "out.txt"));
string b = WriteFile(Path.Combine(rootDirectory, "1", "out.txt"));
string c = WriteFile(Path.Combine(rootDirectory, "2", "1", "in.txt"));
string d = WriteFile(Path.Combine(rootDirectory, "2", "2", "in.txt"));
string e = WriteFile(Path.Combine(rootDirectory, "2", "2", "out.txt"));
string f = WriteFile(Path.Combine(rootDirectory, "3", "out.txt"));
string g = WriteFile(Path.Combine(rootDirectory, "4", "out.txt"));
string h = WriteFile(Path.Combine(rootDirectory, "5", "6", "out.txt"));
string i = WriteFile(Path.Combine(rootDirectory, "5", "out.txt"));
var inBuild = new HashSet <string>(OperatingSystemHelper.PathComparer)
{
c, d, Path.GetDirectoryName(f)
};
Scrubber.RemoveExtraneousFilesAndDirectories(
pathsToScrub: new[] { rootDirectory },
isPathInBuild: path => inBuild.Contains(path),
blockedPaths: new[] { Path.Combine(rootDirectory, "1"), Path.Combine(rootDirectory, "5", "6") },
nonDeletableRootDirectories: new string[0]);
// Files in the "1" directory should still exist even though they were not in the build, since that directory was excluded.
XAssert.IsTrue(File.Exists(a));
XAssert.IsTrue(File.Exists(b));
// The files that were in the build should still exist.
XAssert.IsTrue(File.Exists(c));
XAssert.IsTrue(File.Exists(d));
XAssert.IsTrue(File.Exists(f));
// File in the blocked path still exists.
XAssert.IsTrue(File.Exists(h));
// The file outside of the build should not exist.
XAssert.IsFalse(File.Exists(e));
XAssert.IsFalse(Directory.Exists(Path.GetDirectoryName(g)));
XAssert.IsFalse(File.Exists(i));
}
public void DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty()
{
string rootDir = Path.Combine(TemporaryDirectory, nameof(DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty));
string fullTargetDirPath = Path.Combine(rootDir, "target-dir");
Directory.CreateDirectory(fullTargetDirPath);
XAssert.IsTrue(Directory.Exists(fullTargetDirPath));
var fileUnderTarget = Path.Combine(fullTargetDirPath, "file.txt");
File.WriteAllText(fileUnderTarget, "content");
string fullSymlinkPath = WriteSymlink(Path.Combine(rootDir, "directory symlink"), fullTargetDirPath, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(fullSymlinkPath));
if (OperatingSystemHelper.IsMacOS)
{
SharedOpaqueOutputHelper.EnforceFileIsSharedOpaqueOutput(fullSymlinkPath);
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(fullSymlinkPath));
}
// This is somewhat subtle. On Windows, IsSharedOpaqueOutput will say yes for any directory, including symlink directories. This means they won't be
// considered part of the build and therefore should be traversed.
// So if the symlink is traversed, fileUnderTarget will be found, which is not a shared opaque output. So the file won't be deleted. And
// so nor the symlink directory. If the symlink directory wasn't traversed, then it would be deleted.
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => !SharedOpaqueOutputHelper.IsSharedOpaqueOutput(path),
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.FileExists(fileUnderTarget);
// On Mac:
// - any symlink is a file, any file under shared opaque dir gets scrubber ==> fullSymlinkPath should be scrubbed
//
// On Windows:
// - directories under shared opaques are always removed unless they have files underneath that shouldn't be
// removed. This test verifies this behavior also applies to symlink directories
XAssert.AreEqual(!OperatingSystemHelper.IsMacOS, Directory.Exists(fullSymlinkPath));
}