public void AbsentProbeOnOutputFileIsBlocked()
{
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, SharedOpaqueDirectoryRoot);
FileArtifact source = CreateSourceFile(sharedOpaqueDirPath);
// Create a pip that creates and deletes the source file under a shared opaque
var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var deletePipBuilder = CreatePipBuilder(new[]
{
Operation.WriteFile(source.CreateNextWrittenVersion(), doNotInfer: true),
Operation.DeleteFile(source.CreateNextWrittenVersion(), doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
});
deletePipBuilder.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque);
var deletePip = SchedulePipBuilder(deletePipBuilder);
// Creates a pip that probes the (deleted) source file (undeclared).
// Make the reader depend on the dummy output of the writer to ensure read happens after write
ScheduleProcessWithUndeclaredReads(source, fileDependencies: deletePip.ProcessOutputs.GetOutputFiles(), probeInsteadOfRead: true);
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.DependencyViolationWriteInUndeclaredSourceRead);
}
/// <inheritdoc />
public virtual DirectoryArtifact AddSealDirectory([NotNull] SealDirectory sealDirectory, PipId valuePip)
{
AddPip(sealDirectory);
DirectoryArtifact artifactForNewSeal;
if (sealDirectory.Kind == SealDirectoryKind.SharedOpaque)
{
Contract.Assume(sealDirectory.Directory.IsSharedOpaque);
artifactForNewSeal = sealDirectory.Directory;
}
else
{
// For the regular dynamic case, the directory artifact is always
// created with sealId 0. For other cases, we reserve it
artifactForNewSeal = sealDirectory.Kind == SealDirectoryKind.Opaque
? DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot)
: SealDirectoryTable.ReserveDirectoryArtifact(sealDirectory);
sealDirectory.SetDirectoryArtifact(artifactForNewSeal);
}
SealDirectoryTable.AddSeal(sealDirectory);
ComputeStaticFingerprint(sealDirectory);
return(artifactForNewSeal);
}
public void ValidateCachingTreatDirectoryAsAbsentFileOnHashingInputContent()
{
Configuration.Schedule.TreatDirectoryAsAbsentFileOnHashingInputContent = true;
// start with absent /dir
DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory());
FileArtifact fileVersionOfDir = new FileArtifact(dir.Path);
Process pip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.Probe(fileVersionOfDir),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
// /dir will be hashed as an absent file on every run, so the pip fingerprint remains the same
// Create /dir
Directory.CreateDirectory(ArtifactToString(dir));
RunScheduler().AssertCacheHit(pip.PipId);
// Create nested /dir/file
CreateSourceFile(ArtifactToString(dir));
RunScheduler().AssertCacheHit(pip.PipId);
}
/// <inheritdoc/>
protected override void ProcessOutputs(JavaScriptProject project, ProcessBuilder processBuilder)
{
base.ProcessOutputs(project, processBuilder);
// This makes sure the folder the user profile is pointing to gets actually created
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(UserProfile(project, PathTable)), SealDirectoryKind.SharedOpaque);
}
public void RunSingleProcessWithSharedOpaqueOutputLogging()
{
Configuration.Schedule.UnsafeLazySODeletion = true;
var sharedOpaqueDir = Path.Combine(ObjectRoot, "partialDir");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir);
var source = CreateSourceFile();
var builder = CreatePipBuilder(new[]
{
Operation.WriteFile(outputInSharedOpaque, content: "sod-out", doNotInfer: true)
});
builder.AddOutputDirectory(sharedOpaqueDirectoryArtifact, SealDirectoryKind.SharedOpaque);
builder.Options |= Process.Options.RequiresAdmin;
var pip = SchedulePipBuilder(builder);
// run once and assert success
var result = RunScheduler().AssertSuccess();
// check that shared opaque outputs have been logged in the sideband file
AssertWritesJournaled(result, pip, outputInSharedOpaque);
// run again, assert cache hit, assert sideband files were used to postpone scrubbing
RunScheduler().AssertCacheHit(pip.Process.PipId);
AssertWritesJournaled(result, pip, outputInSharedOpaque);
AssertInformationalEventLogged(EventId.PostponingDeletionOfSharedOpaqueOutputs, count: 1);
}
/// <summary>
/// Adds all known output directories
/// </summary>
/// <remarks>
/// The project root (excluding node_modules) is always considered an output directory
/// </remarks>
protected virtual void ProcessOutputs(JavaScriptProject project, ProcessBuilder processBuilder)
{
// Each project is automatically allowed to write anything under its project root
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder), SealDirectoryKind.SharedOpaque);
if (m_resolverSettings.BlockWritesUnderNodeModules == true)
{
// There shouldn't be any writes under node_modules. So exclude it explicitly, since that also avoids a usually expensive enumeration
// under node_modules when scrubbing.
processBuilder.AddOutputDirectoryExclusion(project.NodeModulesFolder(m_context.PathTable));
}
// Some projects share their temp folder across their build scripts (e.g. build and test)
// So we cannot make them share the temp folder with the infrastructure we have today
// (even though not impossible to fix, we could allow temp directories among pips that are part
// of the same depedency chain and eagerly delete the folder every time a pip finishes)
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.TempFolder), SealDirectoryKind.SharedOpaque);
// Add all the additional output directories that the graph knows about
foreach (var outputDirectory in project.OutputDirectories)
{
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(outputDirectory), SealDirectoryKind.SharedOpaque);
}
// Add additional output directories configured in the main config file
AddAdditionalOutputDirectories(processBuilder, project.ProjectFolder);
}
private static EvaluationResult GetBuildEngineDirectoryToBeDeprecated(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
var pathTable = context.FrontEndContext.PathTable;
var executingEnginePath = AbsolutePath.Create(pathTable, AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(executingEnginePath.GetParent(pathTable))));
}
protected ProcessWithOutputs CreateAndScheduleSharedOpaqueProducer(string sharedOpaqueDir, FileArtifact fileToProduceStatically, FileArtifact sourceFileToRead, params KeyValuePair <FileArtifact, string>[] filesAndContentToProduceDynamically)
{
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var operations = new List <Operation>();
if (fileToProduceStatically.IsValid)
{
operations.Add(Operation.WriteFile(fileToProduceStatically));
}
if (sourceFileToRead.IsValid)
{
operations.Add(Operation.ReadFile(sourceFileToRead));
}
foreach (var fac in filesAndContentToProduceDynamically)
{
operations.Add(Operation.WriteFile(fac.Key, content: fac.Value, doNotInfer: true));
}
var builder = CreatePipBuilder(operations);
builder.AddOutputDirectory(sharedOpaqueDirectoryArtifact, SealDirectoryKind.SharedOpaque);
return(SchedulePipBuilder(builder));
}
public void ToolEnumeratingFile()
{
var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot);
var path = sealDirectoryPath.ToString(Context.PathTable);
var nestedFile1 = CreateSourceFile(path);
var nestedFile2 = CreateUniquePath("nonExistentFile", path);
DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.SourceAllDirectories);
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.UnexpectedFileAccessesAreErrors = false;
var ops = new Operation[]
{
Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(nestedFile2), doNotInfer: true, enumeratePattern: "*"),
Operation.ReadFile(nestedFile1, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()),
// This write will be allowed because UnexpectedFileAccessesAreErrors = false.
// But this will make the above reporting of enumerating non-existing file as enumerating existing file.
Operation.WriteFile(FileArtifact.CreateOutputFile(nestedFile2), doNotInfer: true)
};
var builder = CreatePipBuilder(ops);
builder.AddInputDirectory(dir);
Process pip = SchedulePipBuilder(builder).Process;
var result = RunScheduler().AssertSuccess();
AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 2);
AssertWarningEventLogged(EventId.FileMonitoringWarning, count: 1);
}
public void ValidateBasicCacheMissCounters()
{
AbsolutePath.TryCreate(Context.PathTable, ReadonlyRoot, out var readonlyRootPath);
var readonlyRootDir = DirectoryArtifact.CreateWithZeroPartialSealId(readonlyRootPath);
var childDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ReadonlyRoot));
// Enumerate /readonlyroot and /readonlyroot/childDir
Process enumeratorPip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.EnumerateDir(childDir),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
Process writerPip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
ScheduleRunResult result1 = RunScheduler();
result1.AssertCacheMiss(enumeratorPip.PipId, writerPip.PipId);
result1.AssertNumWeakFingerprintMisses(2);
// Create /childDir/nestedFile, causing strong fingerprint miss
FileArtifact nestedFile = CreateSourceFile(ArtifactToString(childDir));
ScheduleRunResult result2 = RunScheduler();
result2.AssertCacheHit(writerPip.PipId);
// Weak fingerprint hit on both pips
result2.AssertNumWeakFingerprintMisses(0);
// Strong fingerprint miss on enumeratorPip
result2.AssertCacheMiss(enumeratorPip.PipId);
result2.AssertNumStrongFingerprintMisses(1);
}
protected ProcessBuilder CreateTempDirProcessBuilder(Operation[] ops, int tempArtifactType, AbsolutePath tempRootPath, AbsolutePath tempOut)
{
var pipBuilder = CreatePipBuilder(ops);
switch (tempArtifactType)
{
case TempArtifactType.AdditionalTempDirectory:
var temp = new AbsolutePath[pipBuilder.AdditionalTempDirectories.Length + 1];
for (int i = 0; i < pipBuilder.AdditionalTempDirectories.Length; i++)
{
temp[i] = pipBuilder.AdditionalTempDirectories[i];
}
temp[pipBuilder.AdditionalTempDirectories.Length] = tempRootPath;
pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(temp);
break;
case TempArtifactType.TempDirectory:
pipBuilder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempRootPath));
break;
case TempArtifactType.TempFile:
pipBuilder.AddOutputFile(tempOut, FileExistence.Temporary);
break;
default:
XAssert.Fail("A test using a temporary file or directory didn't specify the temp artifact correctly.");
return(null);
}
return(pipBuilder);
}
public void InconsistentTempDirectoriesCreation(bool ensureTempDirectoriesCreation)
{
Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = ensureTempDirectoriesCreation;
AbsolutePath additionalTempDirectory = TempRootPath.Combine(
Context.PathTable,
nameof(InconsistentTempDirectoriesCreation) + "_" + ensureTempDirectoriesCreation);
var pipBuilder = CreatePipBuilder(new[]
{
Operation.ReadFile(CreateSourceFile()),
Operation.CreateDir(DirectoryArtifact.CreateWithZeroPartialSealId(additionalTempDirectory), additionalArgs: "--failIfExists"),
Operation.WriteFile(CreateOutputFileArtifact())
});
pipBuilder.EnableTempDirectory();
pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(new AbsolutePath[] { additionalTempDirectory });
var processWithOutputs = SchedulePipBuilder(pipBuilder);
var result = RunScheduler();
if (ensureTempDirectoriesCreation)
{
result.AssertFailure();
AssertErrorEventLogged(EventId.PipProcessError, 1);
}
else
{
result.AssertSuccess();
}
}
public void ValidateCachingUntrackedPathInput(bool declareDependency, bool inputUnderUntrackedScope)
{
DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot));
FileArtifact nestedFile = CreateSourceFile(ArtifactToString(dir));
// Take input from an untracked path /dir/nestedFile
var ops = new Operation[]
{
Operation.ReadFile(nestedFile, doNotInfer: !declareDependency),
Operation.WriteFile(CreateOutputFileArtifact())
};
var builder = CreatePipBuilder(ops);
if (inputUnderUntrackedScope)
{
builder.AddUntrackedDirectoryScope(dir);
}
else
{
builder.AddUntrackedFile(nestedFile);
}
Process pip = SchedulePipBuilder(builder).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
// Modify untracked input file /dir/nestedFile
File.WriteAllText(ArtifactToString(nestedFile), "nestedFile");
var runSchedulerResult = RunScheduler();
if (!declareDependency || !inputUnderUntrackedScope)
{
runSchedulerResult.AssertCacheHit(pip.PipId);
}
else
{
runSchedulerResult.AssertCacheMiss(pip.PipId);
}
ResetPipGraphBuilder();
// Validate cache miss if the process removes untracked path declarations
pip = CreateAndSchedulePipBuilder(ops).Process;
runSchedulerResult = RunScheduler().AssertCacheMissWithoutAssertingSuccess(pip.PipId);
if (!declareDependency)
{
runSchedulerResult.AssertPipResultStatus((pip.PipId, PipResultStatus.Failed));
AssertWarningEventLogged(SchedulerLogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations);
AssertErrorEventLogged(SchedulerLogEventId.FileMonitoringError);
}
else
{
runSchedulerResult.AssertSuccess();
}
}
public void PreserveOutputsOpaqueTestWithWhitelist()
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
var input = CreateSourceFile();
var opaquePreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaquePreservedDir"));
var outputUnderPreservedOpaque = CreateOutputFileArtifact(opaquePreservedPath);
var createdDirectoryUnderPreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaquePreservedPath.Combine(Context.PathTable, "CreatedDir"));
var opaqueUnpreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaqueUnpreservedDir"));
var outputUnderUnpreservedOpaque = CreateOutputFileArtifact(opaqueUnpreservedPath);
var createdDirectoryUnderUnpreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaqueUnpreservedPath.Combine(Context.PathTable, "CreatedDir"));
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.WriteFile(outputUnderPreservedOpaque, CONTENT, doNotInfer: true),
Operation.CreateDir(createdDirectoryUnderPreservedOpaque, doNotInfer: true),
Operation.WriteFile(outputUnderUnpreservedOpaque, CONTENT, doNotInfer: true),
Operation.CreateDir(createdDirectoryUnderUnpreservedOpaque, doNotInfer: true),
});
builder.AddOutputDirectory(opaquePreservedPath);
builder.AddOutputDirectory(opaqueUnpreservedPath);
builder.Options |= Process.Options.AllowPreserveOutputs;
builder.PreserveOutputWhitelist = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(opaquePreservedPath);
var processAndOutputs = SchedulePipBuilder(builder);
// No cache hit
string outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Change input
ModifyFile(input);
// No cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
// As the opaque output is preserved, the pip appended the existing file.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
// For the file under unpreserved opaque directory, the file was created, so we did not append.
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: true, id: processAndOutputs.Process.PipId);
XAssert.IsTrue(Directory.Exists(createdDirectoryUnderPreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under preserved opaque should have existed.");
// Incremental scheduling doesn't replay the pip from cache and just leaves the filesystem as-is
if (!Configuration.Schedule.GraphAgnosticIncrementalScheduling && !Configuration.Schedule.GraphAgnosticIncrementalScheduling)
{
XAssert.IsFalse(Directory.Exists(createdDirectoryUnderUnpreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under non-preserved opaque should not exist.");
}
// The appended file (CONTENT_TWICE) should remain the same.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
}
public void CompositeContentIsProperlyMaterialized()
{
// This test verifies that BXL properly materializes composite shared opaques
// (i.e., the files are materialized because we need to materialize the composite opaque
// and not because we somehow materialized the original shared opaque directory).
var root = Path.Combine(ObjectRoot, "root");
var rootPath = AbsolutePath.Create(Context.PathTable, root);
var sharedOpaqueDir = Path.Combine(ObjectRoot, "root", "CreateOutputFileArtifact");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var inputB = CreateSourceFile();
var outputA = CreateOutputFileArtifact(sharedOpaqueDir);
// PipA - produces a dynamic file under /root/CreateOutputFileArtifact
var builderA = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(outputA, doNotInfer: true)
});
builderA.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath), SealDirectoryKind.SharedOpaque);
builderA.AddTags(Context.StringTable, "pipA");
var pipA = SchedulePipBuilder(builderA);
// composite shared opaque - consists of a single shared opaque produced by PipA
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(rootPath, new[] { pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath) }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(success);
// PipB - consumes composite shared opaque directory
// note: there is no direct dependency on PipA
var builderB = CreatePipBuilder(new Operation[]
{
Operation.ReadRequiredFile(outputA, doNotInfer: true), // dynamic output of PipA
Operation.ReadFile(inputB), // dummy input, so we can force pipB to re-run
Operation.WriteFile(CreateOutputFileArtifact())
});
builderB.AddInputDirectory(composedOpaque);
builderB.AddTags(Context.StringTable, "pipB");
var pipB = SchedulePipBuilder(builderB);
// bring content into the cache
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId);
// Lazy materialization and tags are used here to prevent materialization of opaque directory produced by pipA,
// so the only place where 'outputA' might come from is materialization of 'composedOpaque'.
Configuration.Schedule.EnableLazyOutputMaterialization = true;
Configuration.Filter = "tag='pipB'";
// make sure that we start with no files materialized
Directory.Delete(root, recursive: true);
// force pipB to re-run
File.AppendAllText(ArtifactToString(inputB), "foo");
RunScheduler().AssertSuccess().AssertCacheHit(pipA.Process.PipId).AssertCacheMiss(pipB.Process.PipId);
}
/// <summary>
/// Pip consumes a filtered SOD (fileA, fileB) and reads both files
/// Filter is changed -> SOD (fileA)
/// Pip should be a miss and should emit DFA
/// </summary>
public void FilterChangesShouldAffectCachingIfPipReadsFilteredFile()
{
var includeKind = global::BuildXL.Pips.Operations.SealDirectoryContentFilter.ContentFilterKind.Include;
var sharedOpaqueDir = Path.Combine(ObjectRoot, "sod");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var fileA = CreateOutputFileArtifact(sharedOpaqueDir);
var fileB = CreateOutputFileArtifact(sharedOpaqueDir);
var builderA = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(fileA, doNotInfer: true),
Operation.WriteFile(fileB, doNotInfer: true)
});
builderA.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque);
var pipA = SchedulePipBuilder(builderA);
var producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath);
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, ".*"), description: null, tags: new string[] { }, out var filteredOpaque);
XAssert.IsTrue(success);
var builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB);
var pipB = SchedulePipBuilder(builderB);
// first run - populate cache
RunScheduler().AssertSuccess();
// reset the graph and this time specify a filter that would exclude fileB
ResetPipGraphBuilder();
pipA = SchedulePipBuilder(builderA);
producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath);
var filter = $".*{fileA.Path.GetName(Context.PathTable).ToString(Context.StringTable)}$";
success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, filter), description: null, tags: new string[] { }, out filteredOpaque);
XAssert.IsTrue(success);
builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB);
pipB = SchedulePipBuilder(builderB);
// since filteredOpaque does not contain fileB, this run should result in a DFA
IgnoreWarnings();
var result = RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.FileMonitoringError);
result
// pipA should be a hit
.AssertCacheHitWithoutAssertingSuccess(pipA.Process.PipId)
// pipB should be a miss (access to fileB is no longer valid)
.AssertCacheMissWithoutAssertingSuccess(pipB.Process.PipId)
// pipB should have a weak FP hit (file accesses are checked during strong FP computation)
.AssertNumWeakFingerprintMisses(0);
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
private EvaluationResult UnsafeExOutputDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
ThrowIfNotAllowed(context, ExOutputDirectory);
var path = Args.AsPath(args, 0);
return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path)));
}
private void ProcessOutputs(ProjectWithPredictions project, ProcessBuilder processBuilder)
{
// Add all shared opaque directories.
foreach (var sharedOutputDirectory in GetOutputDirectories(project))
{
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(sharedOutputDirectory), SealDirectoryKind.SharedOpaque);
}
}
public void ValidateCachingNonhashableMount()
{
// Absent file and directory in nonhashable mount
FileArtifact nonhashableFile = new FileArtifact(CreateUniqueSourcePath(SourceRootPrefix, NonHashableRoot));
DirectoryArtifact nonhashableDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueSourcePath(SourceRootPrefix, NonHashableRoot));
ValidateCachingBehaviorUntrackedMount(nonhashableFile, nonhashableDir, undefinedMount: false);
}
/// <summary>
/// Indicates that no file accesses should be tracked under the AppData directory
/// </summary>
/// <remarks>This includes both ApplicationData and LocalApplicationData folders</remarks>
public void AddUntrackedAppDataDirectories()
{
if (!OperatingSystemHelper.IsUnixOS)
{
AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_pathTable, s_applicationDataPath));
AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_pathTable, s_localApplicationDataPath));
}
}
private EvaluationResult UnsafeExOutputDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
ThrowIfNotAllowed(context, ExOutputDirectory);
var path = Args.AsPath(args, 0);
return(EvaluationResult.Create(new StaticDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(path), SealDirectoryKind.Opaque, PipConstructionHelper.EmptyStaticDirContents)));
}
private static EvaluationResult GetDirectoryValue(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
AbsolutePath path = GetPathValue(context, args);
return(path.IsValid
? EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path))
: EvaluationResult.Undefined);
}
public virtual void ValidateCachingUndefinedMount_Bug1087986()
{
// Absent file and directory in an undefined mount
FileArtifact undefinedFile = new FileArtifact(CreateUniqueSourcePath(SourceRootPrefix, TemporaryDirectory));
DirectoryArtifact undefinedDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueSourcePath(SourceRootPrefix, TemporaryDirectory));
ValidateCachingBehaviorUntrackedMount(undefinedFile, undefinedDir, undefinedMount: true);
AssertWarningEventLogged(EventId.IgnoringUntrackedSourceFileNotUnderMount, 7);
}
public void ValidateCachingDirectoryEnumerationReadonlyMountAllowlistFile(bool cacheableAllowlist, int mountType)
{
// Read-only mounts are fingerprinted based on actual filesystem state
// Start with absent /allowlistFile under /dir
DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(m_mounts[mountType]));
FileArtifact allowlistFile = new FileArtifact(CreateUniquePath(SourceRootPrefix, ArtifactToString(dir)));
AddAllowlistEntry(allowlistFile, cacheableAllowlist: cacheableAllowlist);
FileArtifact outFile = CreateOutputFileArtifact();
Process pip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.EnumerateDir(dir),
Operation.WriteFile(outFile)
}).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
string outputWithAbsentAllowlistFile = File.ReadAllText(ArtifactToString(outFile));
// Create /allowlistFile
File.WriteAllText(ArtifactToString(allowlistFile), "abc");
switch (mountType)
{
case MountType.Readonly:
// allowlisted files still make it into directory fingerprints
// This is consistent with tracking existence of allowlisted files for absent file probes
RunScheduler().AssertCacheMiss(pip.PipId);
break;
case MountType.NonHashable:
// Files in nonhashable mounts never make it into directory fingerprints
break;
case MountType.Undefined:
// Files in undefined mounts never make it into directory fingerprints
// Bug 1134297: This is inconsistent with our tracking of absent file probes for allowlisted
// files under undefined mounts
break;
default:
return;
}
RunScheduler().AssertCacheHit(pip.PipId);
string outputWithExistingAllowlistFile = File.ReadAllText(ArtifactToString(outFile));
// Delete /allowlistFile
File.Delete(ArtifactToString(allowlistFile));
RunScheduler().AssertCacheHit(pip.PipId);
XAssert.AreEqual(File.ReadAllText(ArtifactToString(outFile)), outputWithAbsentAllowlistFile);
// Re-create /allowlistFile
File.WriteAllText(ArtifactToString(allowlistFile), "xyz");
RunScheduler().AssertCacheHit(pip.PipId);
XAssert.AreEqual(File.ReadAllText(ArtifactToString(outFile)), outputWithExistingAllowlistFile);
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_inputFiles =
new[]
{
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.Etc)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.TmpDir)),
};
// Sealed Source inputs
m_inputDirectories =
new[]
{
GetSourceSeal(pathTable, pipGraph, MacPaths.Applications),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrBin),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrInclude),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrLib),
GetSourceSeal(pathTable, pipGraph, MacPaths.Library),
GetSourceSeal(pathTable, pipGraph, MacPaths.UserProvisioning),
};
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChainsDb)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChains)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserCFTextEncoding)),
};
m_untrackedDirectories =
new[]
{
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Bin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Dev),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Private),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Sbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.SystemLibrary),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrLibexec),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrShare),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrStandalone),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrSbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Var),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UserPreferences),
};
}
/// <inheritdoc />
public DirectoryArtifact AddSealDirectory(SealDirectory sealDirectory, PipId valuePip)
{
Contract.Requires(sealDirectory != null);
sealDirectory.PipId = GetPipId();
sealDirectory.SetDirectoryArtifact(DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot));
Contract.Assume(sealDirectory.IsInitialized);
m_pips.Enqueue(sealDirectory);
return(sealDirectory.Directory);
}
private static EvaluationResult GetUserHomeDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
string homePath = Host.Current.CurrentOS == BuildXL.Interop.OperatingSystem.Win
? I($"{Environment.GetEnvironmentVariable(")HOMEDRIVE") ?? "C:"}{Path.DirectorySeparatorChar}{Environment.GetEnvironmentVariable("HOMEPATH") ?? "Users"}")
: Environment.GetEnvironmentVariable("HOME");
var directoryPath = AbsolutePath.Create(context.FrontEndContext.PathTable, homePath);
return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(directoryPath)));
}
private DirectoryArtifact Convert(DirectoryArtifact directory)
{
if (AreGraphsSame)
{
return(directory);
}
// TODO: This is wrong (loses the seal id but it probably doesn't matter since this isn't used for comparison)
return(DirectoryArtifact.CreateWithZeroPartialSealId(Convert(directory.Path)));
}
/// <summary>
/// Dumps file members into an array of file artifacts.
/// </summary>
public void GetArtifacts(Context context, EvaluationResult[] files, bool asDirectory)
{
for (int i = 0; i < m_files.Count; ++i)
{
var path = DirectoryPath.Combine(context.PathTable, m_files[i]);
files[Index + i] = asDirectory
? EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path))
: EvaluationResult.Create(FileArtifact.CreateSourceFile(path));
}
}