public void OutputsUnderSharedOpaqueAreProperlyMarkedEvenOnCacheReplay()
{
var file = X("out/SharedOpaqueOutput.txt");
var spec0 = ProduceFileUnderSharedOpaque(file);
AddModule("Module0", ("spec0.dsc", spec0), placeInRoot: true);
RunEngine(rememberAllChangedTrackedInputs: true);
var objDir = Configuration.Layout.ObjectDirectory.ToString(Context.PathTable);
var producedFile = Path.Combine(objDir, file);
// Make sure the file was produced
Assert.True(File.Exists(producedFile));
// And that it has been marked as shared opaque output
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile));
File.Delete(producedFile);
// Replay from cache this time
RunEngine(rememberAllChangedTrackedInputs: true);
IgnoreWarnings();
// Make sure this is a cache replay
AssertVerboseEventLogged(global::BuildXL.Scheduler.Tracing.LogEventId.ProcessPipCacheHit);
// And check again that the file is still properly marked
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile));
}
public void StaleSharedOpaqueOutputsAreNotPartOfTheFingerprint()
{
AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, Path.Combine(SourceRoot, "dir"));
DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath);
var alienFile = CreateSourceFile(root: dirPath);
string alienFilePath = alienFile.Path.ToString(Context.PathTable);
// Create a fake stale shared opaque output under the directory we will enumerate
File.WriteAllText(alienFilePath, "some text");
SharedOpaqueOutputHelper.EnforceFileIsSharedOpaqueOutput(alienFilePath);
var builder = CreatePipBuilder(new Operation[]
{
Operation.EnumerateDir(dirToEnumerate, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
});
// This makes sure we use the right file system, which is aware of alien files
builder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads;
var pip = SchedulePipBuilder(builder);
// Run once
RunScheduler().AssertSuccess();
// Delete the sharedOpaque under the enumerated directory. We should get a cache hit on re-run since
// old outputs are ignored
File.Delete(alienFile.Path.ToString(Context.PathTable));
RunScheduler().AssertCacheHit(pip.Process.PipId);
}
public void AllowedRewrittenSourcesAreNotFlaggedAsSharedOpaques()
{
var objDir = Configuration.Layout.ObjectDirectory.ToString(Context.PathTable);
var file = X("out/SharedOpaqueOutput.txt");
var producedFile = Path.Combine(objDir, file);
// Create the file beforehand so it introduces an allowed rewrite
Directory.CreateDirectory(Directory.GetParent(producedFile).FullName);
string originalContent = "content";
string rewrittenContent = "rewritten";
File.WriteAllText(producedFile, originalContent);
var spec0 = ProduceFileUnderSharedOpaque(file, allowSourceRewrites: true, allowUndeclaredReads: true, content: rewrittenContent);
AddModule("Module0", ("spec0.dsc", spec0), placeInRoot: true);
RunEngine(rememberAllChangedTrackedInputs: true);
// Make sure the file was produced with rewritten content
Assert.True(File.Exists(producedFile));
Assert.Equal(rewrittenContent, File.ReadAllText(producedFile).Trim(' ', '\r', '\n'));
// And that it has not been marked as shared opaque output
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile));
// We should place the file as a copy, not hardlinked to the cache, and therefore the file should be modifiable
// This operation will throw otherwise
File.AppendAllText(producedFile, " we should be able to modify the file");
// Restore the file to its initial shape
File.Delete(producedFile);
File.WriteAllText(producedFile, originalContent);
// Replay from cache this time
RunEngine(rememberAllChangedTrackedInputs: true);
// Make sure the file was produced with rewritten content
Assert.True(File.Exists(producedFile));
Assert.Equal(rewrittenContent, File.ReadAllText(producedFile).Trim(' ', '\r', '\n'));
IgnoreWarnings();
// Make sure this is a cache replay
AssertVerboseEventLogged(global::BuildXL.Scheduler.Tracing.LogEventId.ProcessPipCacheHit);
// And check again that the file is still not marked
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile));
// We should place the file as a copy, not hardlinked to the cache, and therefore the file should be modifiable
// This operation will throw otherwise
File.AppendAllText(producedFile, " we should be able to modify the file");
}
public void StaticOutputBecomingASharedOpaqueOutputIsProperlyMarkedAsSharedOpaqueOutput()
{
var file = X($"out/MyFile.txt");
XAssert.PossiblySucceeded(FileUtilities.TryDeleteFile(file));
var message = Guid.NewGuid().ToString();
var spec0 = ProduceFileStatically(file, content: message);
AddModule("Module0", ("spec0.dsc", spec0), placeInRoot: true);
RunEngine(rememberAllChangedTrackedInputs: true);
var objDir = Configuration.Layout.ObjectDirectory.ToString(Context.PathTable);
var producedFile = Path.Combine(objDir, file);
// Make sure the file was produced
Assert.True(File.Exists(producedFile));
// Since this is a statically declared file, it shouldn't be marked as a shared opaque output
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile), "Statically declared file marked as shared opaque output: " + producedFile);
// Delete the created file (since scrubbing is not on for this test, we have to simulate it)
File.Delete(producedFile);
// Overrite the spec so now the same file is generated as a shared opaque output
spec0 = ProduceFileUnderSharedOpaque(file, content: message);
File.WriteAllText(Path.Combine(Configuration.Layout.SourceDirectory.ToString(Context.PathTable), "spec0.dsc"), spec0);
// Run the pip
RunEngine(rememberAllChangedTrackedInputs: true);
// Check the timestamp is the right one
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile), "SOD file not marked on cache miss");
// Delete the file
File.Delete(producedFile);
// Replay from cache. Since the file content is unchanged, the cache should have a blob
// corresponding to the first pip, where the file was a statically declared output
RunEngine(rememberAllChangedTrackedInputs: true);
IgnoreWarnings();
// Make sure this is a cache replay
AssertVerboseEventLogged(global::BuildXL.Scheduler.Tracing.LogEventId.ProcessPipCacheHit);
// Check the timestamp is the right one now
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile), "SOD file not marked on cache replay");
}
private void AssertSharedOpaqueOputputsInNestedDestDirectory(AbsolutePath rootDirPath)
{
string root = ToString(rootDirPath);
var expectedOutputs = new[]
{
X($"{root}/{NestedDirDest}/{FileNameInSrc}"),
X($"{root}/{NestedDirDest}/{FileNameInDest}"),
};
foreach (var path in expectedOutputs)
{
XAssert.IsTrue(
SharedOpaqueOutputHelper.IsSharedOpaqueOutput(path),
$"Path '{path}' does not have magic shared opaque output timestamp");
}
}
public void SkipFlaggingSharedOpaquesIsHonored()
{
// Set up PipA => sharedOpaqueDirectory => PipB
string sharedOpaqueDir = Path.Combine(ObjectRoot, "partialDir");
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
FileArtifact outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir);
FileArtifact source = CreateSourceFile();
var pipA = CreateAndScheduleSharedOpaqueProducer(sharedOpaqueDir, fileToProduceStatically: FileArtifact.Invalid, sourceFileToRead: source, new KeyValuePair <FileArtifact, string>(outputInSharedOpaque, null));
var builderB = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(outputInSharedOpaque, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builderB.AddInputDirectory(pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath));
var pipB = SchedulePipBuilder(builderB);
// B should be able to consume the file in the opaque directory. Second build should have both cached
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId);
// The output shouldn't be flagged as a shared opaque
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(outputInSharedOpaque.Path.ToString(Context.PathTable)));
RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId);
// The output shouldn't be flagged as a shared opaque
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(outputInSharedOpaque.Path.ToString(Context.PathTable)));
// Make sure we can replay the file in the opaque directory
File.Delete(ArtifactToString(outputInSharedOpaque));
RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId);
XAssert.IsTrue(File.Exists(ArtifactToString(outputInSharedOpaque)));
// The output shouldn't be flagged as a shared opaque
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(outputInSharedOpaque.Path.ToString(Context.PathTable)));
// Modify the input and make sure both are rerun
File.WriteAllText(ArtifactToString(source), "New content");
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId);
RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId);
// The output shouldn't be flagged as a shared opaque
XAssert.IsFalse(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(outputInSharedOpaque.Path.ToString(Context.PathTable)));
}
public void SharedOpaqueOutputsOnFailingPipMustBeProperlyMarked()
{
var file = X("out/MyFile.txt");
var objDir = Configuration.Layout.ObjectDirectory.ToString(Context.PathTable);
var producedFile = Path.Combine(objDir, file);
var spec0 = ProduceFileUnderSharedOpaque(file, failOnExit: true);
AddModule("Module0", ("spec0.dsc", spec0), placeInRoot: true);
// Run the pip
RunEngine(rememberAllChangedTrackedInputs: true, expectSuccess: false);
AssertErrorEventLogged(LogEventId.PipProcessError);
// Check the timestamp is the right one
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(producedFile), "SOD file not marked on pip failure");
}
public void DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty()
{
string rootDir = Path.Combine(TemporaryDirectory, nameof(DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty));
string fullTargetDirPath = Path.Combine(rootDir, "target-dir");
Directory.CreateDirectory(fullTargetDirPath);
XAssert.IsTrue(Directory.Exists(fullTargetDirPath));
var fileUnderTarget = Path.Combine(fullTargetDirPath, "file.txt");
File.WriteAllText(fileUnderTarget, "content");
string fullSymlinkPath = WriteSymlink(Path.Combine(rootDir, "directory symlink"), fullTargetDirPath, isTargetFile: false);
XAssert.IsTrue(FileUtilities.FileExistsNoFollow(fullSymlinkPath));
if (OperatingSystemHelper.IsMacOS)
{
SharedOpaqueOutputHelper.EnforceFileIsSharedOpaqueOutput(fullSymlinkPath);
XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(fullSymlinkPath));
}
// This is somewhat subtle. On Windows, IsSharedOpaqueOutput will say yes for any directory, including symlink directories. This means they won't be
// considered part of the build and therefore should be traversed.
// So if the symlink is traversed, fileUnderTarget will be found, which is not a shared opaque output. So the file won't be deleted. And
// so nor the symlink directory. If the symlink directory wasn't traversed, then it would be deleted.
Scrubber.RemoveExtraneousFilesAndDirectories(
isPathInBuild: path => !SharedOpaqueOutputHelper.IsSharedOpaqueOutput(path),
pathsToScrub: new[] { rootDir },
blockedPaths: CollectionUtilities.EmptyArray <string>(),
nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>());
XAssert.FileExists(fileUnderTarget);
// On Mac:
// - any symlink is a file, any file under shared opaque dir gets scrubber ==> fullSymlinkPath should be scrubbed
//
// On Windows:
// - directories under shared opaques are always removed unless they have files underneath that shouldn't be
// removed. This test verifies this behavior also applies to symlink directories
XAssert.AreEqual(!OperatingSystemHelper.IsMacOS, Directory.Exists(fullSymlinkPath));
}
