/// <summary>
/// Compute the intersection of the pip's dependencies and the affected contents of the build
/// </summary>
public IReadOnlyList <AbsolutePath> GetChangeAffectedInputs(Process process)
{
var changeAffectedInputs = new HashSet <AbsolutePath>(process.Dependencies.Select(f => f.Path).Where(p => m_sourceChangeAffectedFiles.Contains(p)));
foreach (var directory in process.DirectoryDependencies)
{
foreach (var file in m_fileContentManager.ListSealedDirectoryContents(directory))
{
if (m_sourceChangeAffectedFiles.Contains(file))
{
changeAffectedInputs.Add(file.Path);
}
}
if (m_fileContentManager.Host.TryGetSourceSealDirectory(directory, out var sourceSealWithPatterns))
{
var affectedMembers = m_sourceSealDirectoryAffectedMembers.GetOrAdd(
directory,
d => new List <AbsolutePath>(m_initialSourceChanges.Where(p => sourceSealWithPatterns.Contains(PathTable, p))));
changeAffectedInputs.AddRange(affectedMembers);
}
}
return(ReadOnlyArray <AbsolutePath> .FromWithoutCopy(changeAffectedInputs.ToArray()));
}
public void RunSingleBreakawayProcess()
{
var source = CreateSourceFile();
var output = CreateOutputFileArtifact();
var builder = CreatePipBuilder(new[]
{
Operation.Spawn(
Context.PathTable,
waitToFinish: true,
Operation.ReadFile(source),
Operation.WriteFile(output)),
Operation.AugmentedWrite(output),
Operation.AugmentedRead(source),
Operation.WriteFile(CreateOutputFileArtifact(root: null, prefix: "dummy"))
});
builder.AddInputFile(source);
builder.AddOutputFile(output.Path);
builder.Options |= Process.Options.RequiresAdmin;
// Configure the test process itself to escape the sandbox
builder.ChildProcessesToBreakawayFromSandbox = ReadOnlyArray <PathAtom> .FromWithoutCopy(new[] { PathAtom.Create(Context.StringTable, TestProcessToolName) });
SchedulePipBuilder(builder);
// run once and assert success
RunScheduler().AssertSuccess();
}
protected ProcessBuilder CreateTempDirProcessBuilder(Operation[] ops, int tempArtifactType, AbsolutePath tempRootPath, AbsolutePath tempOut)
{
var pipBuilder = CreatePipBuilder(ops);
switch (tempArtifactType)
{
case TempArtifactType.AdditionalTempDirectory:
var temp = new AbsolutePath[pipBuilder.AdditionalTempDirectories.Length + 1];
for (int i = 0; i < pipBuilder.AdditionalTempDirectories.Length; i++)
{
temp[i] = pipBuilder.AdditionalTempDirectories[i];
}
temp[pipBuilder.AdditionalTempDirectories.Length] = tempRootPath;
pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(temp);
break;
case TempArtifactType.TempDirectory:
pipBuilder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempRootPath));
break;
case TempArtifactType.TempFile:
pipBuilder.AddOutputFile(tempOut, FileExistence.Temporary);
break;
default:
XAssert.Fail("A test using a temporary file or directory didn't specify the temp artifact correctly.");
return(null);
}
return(pipBuilder);
}
/// <summary>
/// Constructor.
/// </summary>
public CounterCollectionInfo()
{
ulong min = EnumTraits <TEnum> .MinValue;
ulong max = EnumTraits <TEnum> .MaxValue;
Contract.Assume(max >= min);
ushort numValues = checked ((ushort)(max - min + 1));
var counterTypes = new CounterType[numValues];
var counterNames = new string[numValues];
foreach (FieldInfo field in typeof(TEnum).GetFields())
{
if (field.IsSpecialName)
{
continue;
}
Contract.Assume(field.FieldType == typeof(TEnum));
var attribute = field.GetCustomAttribute(typeof(CounterTypeAttribute)) as CounterTypeAttribute;
counterTypes[GetCounterIndex((TEnum)field.GetValue(null))] = attribute?.CounterType ?? CounterType.Numeric;
counterNames[GetCounterIndex((TEnum)field.GetValue(null))] = attribute?.CounterName;
}
CounterTypes = ReadOnlyArray <CounterType> .FromWithoutCopy(counterTypes);
CounterNames = ReadOnlyArray <string> .FromWithoutCopy(counterNames);
}
public void BreakawayProcessCompensatesWithAugmentedAccesses()
{
string sharedOpaqueDir = Path.Combine(ObjectRoot, "partialDir");
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
FileArtifact outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir);
FileArtifact source = CreateSourceFile();
var builder = CreatePipBuilder(new Operation[]
{
// We spawn a process, since breakaway happens for child processes only
Operation.Spawn(Context.PathTable, waitToFinish: true,
// Write a file. This access will not be observed
Operation.WriteFile(outputInSharedOpaque, doNotInfer: true)
),
// Report the augmented accesses (in the root process, which is detoured normally) without actually
// performing any IO
Operation.AugmentedWrite(outputInSharedOpaque, doNotInfer: true),
});
builder.AddOutputDirectory(sharedOpaqueDirPath, kind: SealDirectoryKind.SharedOpaque);
builder.AddInputFile(source);
// Configure the test process itself to escape the sandbox
builder.ChildProcessesToBreakawayFromSandbox = ReadOnlyArray <PathAtom> .FromWithoutCopy(new[] { PathAtom.Create(Context.StringTable, TestProcessToolName) });
var pip = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
XAssert.IsTrue(File.Exists(ArtifactToString(outputInSharedOpaque)));
// Make sure we can replay the file in the opaque directory. This means the write access reached detours via augmentation.
File.Delete(ArtifactToString(outputInSharedOpaque));
RunScheduler().AssertCacheHit(pip.Process.PipId);
XAssert.IsTrue(File.Exists(ArtifactToString(outputInSharedOpaque)));
}
public void BreakawayProcessesCanOutliveThePip()
{
var pidFile = CreateOutputFileArtifact(ObjectRoot, prefix: $"{nameof(BreakawayProcessesCanOutliveThePip)}.pid");
var builder = CreatePipBuilder(new Operation[]
{
Operation.SpawnAndWritePidFile(Context.PathTable, waitToFinish: false, pidFile: pidFile, doNotInfer: false,
Operation.Block())
});
// Configure the test process itself to escape the sandbox
builder.ChildProcessesToBreakawayFromSandbox = ReadOnlyArray <PathAtom> .FromWithoutCopy(new[] { PathAtom.Create(Context.StringTable, TestProcessToolName) });
var pip = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
var pidFilePath = ToString(pidFile);
XAssert.FileExists(pidFilePath);
var pidFileContent = File.ReadAllText(pidFilePath);
XAssert.IsTrue(int.TryParse(pidFileContent, out var pid), $"Cannot convert pid file content '{pidFileContent}' to integer");
var proc = TryGetProcessById(pid);
XAssert.IsNotNull(proc, $"Could not find the process (PID:{pid}) that was supposed to break away");
proc.Kill();
}
public void ProcessIsProperlyConfigured()
{
var project = CreateProjectWithPredictions("A.proj");
var testProj = Start()
.Add(project)
.ScheduleAll()
.AssertSuccess()
.RetrieveSuccessfulProcess(project);
// Undeclared sources are allowed as long as they are true sources
Assert.True(testProj.AllowUndeclaredSourceReads);
// Weak fingerprint augmentation should be enforced
Assert.True((testProj.ProcessOptions & Process.Options.EnforceWeakFingerprintAugmentation) != 0);
// Double writes are allowed as long as the written content is the same
Assert.True(testProj.DoubleWritePolicy == DoubleWritePolicy.AllowSameContentDoubleWrites);
// Working directory is the project directory
Assert.True(testProj.WorkingDirectory == project.FullPath.GetParent(PathTable));
// Log file is configured
testProj.GetOutputs().Any(fa => fa.Path.GetName(PathTable).ToString(PathTable.StringTable) == "msbuild.log");
// Surviving processes are configured
testProj.AllowedSurvivingChildProcessNames.ToReadOnlySet().SetEquals(ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(PathTable.StringTable, "mspdbsrv.exe"),
PathAtom.Create(PathTable.StringTable, "vctip.exe"),
PathAtom.Create(PathTable.StringTable, "conhost.exe")));
}
/// <summary>
/// Creates an output directory.
/// </summary>
protected ValueTuple <DirectoryArtifact, ReadOnlyArray <FileArtifactWithAttributes> > CreateOutputDirectory(
AbsolutePath rootPath = default(AbsolutePath),
RelativePath relativePathToDirectory = default(RelativePath),
RelativePath[] relativePathToMembers = null)
{
var directory = CreateDirectory(rootPath, relativePathToDirectory);
ReadOnlyArray <FileArtifactWithAttributes> members;
if (relativePathToMembers == null || relativePathToMembers.Length == 0)
{
members = ReadOnlyArray <FileArtifactWithAttributes> .Empty;
}
else
{
var fullMemberNames = new FileArtifactWithAttributes[relativePathToMembers.Length];
for (int i = 0; i < fullMemberNames.Length; ++i)
{
fullMemberNames[i] = FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(directory.Path.Combine(Context.PathTable, relativePathToMembers[i])), FileExistence.Required);
}
members = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(fullMemberNames);
}
return(directory, members);
}
/// <summary>
/// Writes a QualifierId using its underlying string representation
/// </summary>
public override void Write(QualifierId qualifierId)
{
Start <QualifierId>();
Write(qualifierId.IsValid);
if (qualifierId.IsValid)
{
var qualifier = m_qualifierTable.GetQualifier(qualifierId);
// the qualifier is stored as <key, value>[]
var keyValues = new Tuple <string, string> [qualifier.Keys.Count];
for (var i = 0; i < keyValues.Length; i++)
{
keyValues[i] = new Tuple <string, string>(qualifier.Keys[i].ToString(m_stringTable), qualifier.Values[i].ToString(m_stringTable));
}
Write(
ReadOnlyArray <Tuple <string, string> > .FromWithoutCopy(keyValues),
(writer, value) =>
{
writer.Write(value.Item1);
writer.Write(value.Item2);
});
}
End();
}
public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public void PreserveOutputsTestWithWhitelist()
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
var input = CreateSourceFile();
var outputPreserved = CreateOutputFileArtifact(Path.Combine(ObjectRoot, @"nested\out\filePreserved"));
var outputUnpreserved = CreateOutputFileArtifact(Path.Combine(ObjectRoot, @"nested\out\fileUnpreserved"));
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.WriteFile(outputPreserved, CONTENT),
Operation.WriteFile(outputUnpreserved, CONTENT)
});
builder.Options |= Process.Options.AllowPreserveOutputs;
builder.PreserveOutputWhitelist = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(outputPreserved);
var processAndOutputs = SchedulePipBuilder(builder);
var outputContent = RunSchedulerAndGetOutputContents(outputPreserved, false, processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT, outputContent);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnpreserved)));
ModifyFile(input);
outputContent = RunSchedulerAndGetOutputContents(outputPreserved, false, processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT_TWICE, outputContent);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnpreserved)));
outputContent = RunSchedulerAndGetOutputContents(outputPreserved, true, processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT_TWICE, outputContent);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnpreserved)));
}
public void InconsistentTempDirectoriesCreation(bool ensureTempDirectoriesCreation)
{
Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = ensureTempDirectoriesCreation;
AbsolutePath additionalTempDirectory = TempRootPath.Combine(
Context.PathTable,
nameof(InconsistentTempDirectoriesCreation) + "_" + ensureTempDirectoriesCreation);
var pipBuilder = CreatePipBuilder(new[]
{
Operation.ReadFile(CreateSourceFile()),
Operation.CreateDir(DirectoryArtifact.CreateWithZeroPartialSealId(additionalTempDirectory), additionalArgs: "--failIfExists"),
Operation.WriteFile(CreateOutputFileArtifact())
});
pipBuilder.EnableTempDirectory();
pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(new AbsolutePath[] { additionalTempDirectory });
var processWithOutputs = SchedulePipBuilder(pipBuilder);
var result = RunScheduler();
if (ensureTempDirectoriesCreation)
{
result.AssertFailure();
AssertErrorEventLogged(EventId.PipProcessError, 1);
}
else
{
result.AssertSuccess();
}
}
private async Task <CasHash> AddPathSet(ICacheSession session, params string[] thePaths)
{
var pathTable = new PathTable();
ObservedPathEntry[] paths = new ObservedPathEntry[thePaths.Length];
for (int i = 0; i < thePaths.Length; i++)
{
AbsolutePath absPath = AbsolutePath.Create(pathTable, thePaths[i]);
paths[i] = new ObservedPathEntry(absPath, false, false, false, null, false);
}
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
ObservedPathSet pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer)),
emptyObservedAccessFileNames,
null);
using (var pathSetBuffer = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false))
{
pathSet.Serialize(pathTable, writer, preserveCasing: false);
}
pathSetBuffer.Seek(0, SeekOrigin.Begin);
// Must await such that the dispose of the MemoryStream is only after the write completes
return(await session.AddToCasAsync(pathSetBuffer).SuccessAsync());
}
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
public void PreserveOutputsOpaqueTestWithWhitelist()
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
var input = CreateSourceFile();
var opaquePreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaquePreservedDir"));
var outputUnderPreservedOpaque = CreateOutputFileArtifact(opaquePreservedPath);
var createdDirectoryUnderPreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaquePreservedPath.Combine(Context.PathTable, "CreatedDir"));
var opaqueUnpreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaqueUnpreservedDir"));
var outputUnderUnpreservedOpaque = CreateOutputFileArtifact(opaqueUnpreservedPath);
var createdDirectoryUnderUnpreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaqueUnpreservedPath.Combine(Context.PathTable, "CreatedDir"));
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.WriteFile(outputUnderPreservedOpaque, CONTENT, doNotInfer: true),
Operation.CreateDir(createdDirectoryUnderPreservedOpaque, doNotInfer: true),
Operation.WriteFile(outputUnderUnpreservedOpaque, CONTENT, doNotInfer: true),
Operation.CreateDir(createdDirectoryUnderUnpreservedOpaque, doNotInfer: true),
});
builder.AddOutputDirectory(opaquePreservedPath);
builder.AddOutputDirectory(opaqueUnpreservedPath);
builder.Options |= Process.Options.AllowPreserveOutputs;
builder.PreserveOutputWhitelist = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(opaquePreservedPath);
var processAndOutputs = SchedulePipBuilder(builder);
// No cache hit
string outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Change input
ModifyFile(input);
// No cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
// As the opaque output is preserved, the pip appended the existing file.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
// For the file under unpreserved opaque directory, the file was created, so we did not append.
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: true, id: processAndOutputs.Process.PipId);
XAssert.IsTrue(Directory.Exists(createdDirectoryUnderPreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under preserved opaque should have existed.");
// Incremental scheduling doesn't replay the pip from cache and just leaves the filesystem as-is
if (!Configuration.Schedule.GraphAgnosticIncrementalScheduling && !Configuration.Schedule.GraphAgnosticIncrementalScheduling)
{
XAssert.IsFalse(Directory.Exists(createdDirectoryUnderUnpreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under non-preserved opaque should not exist.");
}
// The appended file (CONTENT_TWICE) should remain the same.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
}
public ChooseWorkerCacheLookup(
LoggingContext loggingContext,
IScheduleConfiguration scheduleConfig,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue) : base(loggingContext, workers, pipQueue, DispatcherKind.ChooseWorkerCacheLookup, scheduleConfig.MaxChooseWorkerCacheLookup, scheduleConfig.ModuleAffinityEnabled())
{
m_workerBalancedLoadFactors = ReadOnlyArray <double> .FromWithoutCopy(0.5, 1, 2, 3);
}
private (IIpcMoniker ipcMoniker, PipId servicePipId) CreateService(TestPipGraphFragment fragment)
{
var ipcMoniker = fragment.GetIpcMoniker();
var apiServerMoniker = fragment.GetApiServerMoniker();
var shutdownBuilder = fragment.GetProcessBuilder();
new ArgumentsBuilder(shutdownBuilder)
.AddIpcMonikerOption("--ipcMoniker", ipcMoniker)
.AddIpcMonikerOption("--serverMoniker", apiServerMoniker)
.AddOutputOption("--output", fragment.CreateOutputFile("shutdown.txt"));
shutdownBuilder.ServiceKind = global::BuildXL.Pips.Operations.ServicePipKind.ServiceShutdown;
(Process shutdownProcess, ProcessOutputs _) = fragment.ScheduleProcessBuilder(shutdownBuilder);
var finalProcessBuilder = fragment.GetIpcProcessBuilder();
new ArgumentsBuilder(finalProcessBuilder)
.AddOption("--command", "final")
.AddIpcMonikerOption("--ipcMoniker", ipcMoniker);
var finalOutputFile = fragment.CreateOutputFile("final.txt");
var finalizationPip = fragment.ScheduleIpcPip(
ipcMoniker,
null,
finalProcessBuilder,
finalOutputFile,
true);
XAssert.IsTrue(finalizationPip.IsValid);
var serviceProcessBuilder = fragment.GetProcessBuilder();
new ArgumentsBuilder(serviceProcessBuilder)
.AddIpcMonikerOption("--ipcMoniker", ipcMoniker)
.AddIpcMonikerOption("--serverMoniker", apiServerMoniker)
.AddOutputOption("--output", fragment.CreateOutputFile("service.txt"));
serviceProcessBuilder.ServiceKind = global::BuildXL.Pips.Operations.ServicePipKind.Service;
serviceProcessBuilder.ShutDownProcessPipId = shutdownProcess.PipId;
serviceProcessBuilder.FinalizationPipIds = ReadOnlyArray <PipId> .FromWithoutCopy(new[] { finalizationPip });
(Process serviceProcess, ProcessOutputs _) = fragment.ScheduleProcessBuilder(serviceProcessBuilder);
var createProcessBuilder = fragment.GetIpcProcessBuilder();
new ArgumentsBuilder(createProcessBuilder)
.AddOption("--command", "create")
.AddIpcMonikerOption("--ipcMoniker", ipcMoniker);
var createOutputFile = fragment.CreateOutputFile("create.txt");
var createPip = fragment.ScheduleIpcPip(
ipcMoniker,
serviceProcess.PipId,
createProcessBuilder,
createOutputFile,
false);
XAssert.IsTrue(createPip.IsValid);
return(ipcMoniker, serviceProcess.PipId);
}
public async Task CorrelateMoveFileAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
WriteFile(pathTable, sourceFile, "content");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: "CorrelateMoveFile",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
FileArtifactWithAttributes.Create(
FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressDest,
sourceFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressSource));
}
}
private void GenerateRandomFile(int size)
{
Random r = new Random();
var bytes = new byte[size];
r.NextBytes(bytes);
var content = ReadOnlyArray <byte> .FromWithoutCopy(bytes);
m_fileSystem[size + "_" + Guid.NewGuid().ToString()] = content;
}
/// <inheritdoc />
public async Task <Possible <ContentAvailabilityBatchResult, Failure> > TryLoadAvailableContentAsync(IReadOnlyList <ContentHash> hashes)
{
using (var hashAvailabilityMapWrapper = m_hashAvailabilityMapPool.GetInstance())
using (var hashListWrapper = m_hashListPool.GetInstance())
{
var hashAvailabilityMap = hashAvailabilityMapWrapper.Instance;
var hashList = hashListWrapper.Instance;
var uniqueUnknownAvailabilityHashes = DeduplicateAndGetUnknownAvailabilityHashes(hashes, hashAvailabilityMap, hashList);
bool allContentAvailable = true;
if (uniqueUnknownAvailabilityHashes.Count != 0)
{
// Only query inner cache if there are hashes whose availabilty is unknown
var possibleBatchResult = await m_innerCache.TryLoadAvailableContentAsync(uniqueUnknownAvailabilityHashes);
if (!possibleBatchResult.Succeeded || uniqueUnknownAvailabilityHashes == hashes)
{
// If not successful or the hashes are the same as original hashes just return the result
return(possibleBatchResult);
}
// Populate hash availability map with results from inner cache
foreach (var result in possibleBatchResult.Result.Results)
{
hashAvailabilityMap[result.Hash] = result;
if (!result.IsAvailable)
{
allContentAvailable = false;
}
else
{
// Mark the hash as available for subsequent operations
m_availableContent.TryAdd(result.Hash, result.SourceCache);
}
}
}
ContentAvailabilityResult[] results = new ContentAvailabilityResult[hashes.Count];
for (int i = 0; i < hashes.Count; i++)
{
var hash = hashes[i];
if (hashAvailabilityMap.TryGetValue(hash, out var result))
{
results[i] = result;
}
else
{
throw Contract.AssertFailure(I($"Hash {hash} should be present in availability map."));
}
}
return(new ContentAvailabilityBatchResult(ReadOnlyArray <ContentAvailabilityResult> .FromWithoutCopy(results), allContentAvailable));
}
}
private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath)
{
// Arguments to create an output file
var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) });
var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
return(pip);
}
private static Pip CreateCmdPip(BuildXLContext context, string tempDirectory, string outFile, bool is64Bit)
{
Contract.Requires(context != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
var pathTable = context.PathTable;
string executable = is64Bit ? CmdHelper.CmdX64 : CmdHelper.CmdX86;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
string workingDirectory = AssemblyDirectory;
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
AbsolutePath outFilePath = AbsolutePath.Create(pathTable, outFile);
FileArtifact outFileArtifact = FileArtifact.CreateSourceFile(outFilePath).CreateNextWrittenVersion();
var pip = new BuildXL.Pips.Operations.Process(
executableArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
"/d",
"/c",
"echo",
"hello",
">",
outFileArtifact),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
AbsolutePath.Create(pathTable, tempDirectory),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(outFileArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
return(pip);
}
/// <nodoc />
public static void AddTags(this ProcessBuilder processBuilder, StringTable stringTable, params string[] tags)
{
var tagIds = new StringId[tags.Length];
for (int i = 0; i < tags.Length; i++)
{
tagIds[i] = StringId.Create(stringTable, tags[i]);
}
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(tagIds);
}
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\");
var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);
var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
});
var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
new[]
{
new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
}
);
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
var pipExecutionResult = await RunProcess(context, pip);
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);
var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);
// Make sure outputs got redirected based on the configured isolation level
switch (containerIsolationLevel)
{
case ContainerIsolationLevel.IsolateOutputFiles:
XAssert.IsTrue(File.Exists(redirectedFile));
break;
case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
break;
}
}
}
private async Task <CacheEntry> CreateCacheEntryAndStoreContent(string metadata, params string[] content)
{
ContentHash metadataHash = await AddContent(metadata);
ContentHash[] hashes = new ContentHash[content.Length + 1];
hashes[0] = metadataHash;
for (int i = 0; i < content.Length; i++)
{
hashes[i + 1] = await AddContent(content[i]);
}
return(CacheEntry.FromArray(ReadOnlyArray <ContentHash> .FromWithoutCopy(hashes), null));
}
/// <summary>
/// Gets the ItemResources for the given process's semaphores using the given semaphore set
/// </summary>
public static ItemResources GetSemaphoreResources(
this Process process,
SemaphoreSet <StringId> semaphoreSet,
Func <ProcessSemaphoreInfo, int> getLimit = null,
IReadOnlyList <ProcessSemaphoreInfo> customSemaphores = null)
{
return(GetSemaphoreResources(
semaphoreSet,
semaphores: customSemaphores == null ?
process.Semaphores :
ReadOnlyArray <ProcessSemaphoreInfo> .FromWithoutCopy(process.Semaphores.ConcatAsArray(customSemaphores)),
getLimit: getLimit));
}
public void PreserveOutputsOpaqueTestWithWhitelist()
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
var input = CreateSourceFile();
var opaquePreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaquePreservedDir"));
var outputUnderPreservedOpaque = CreateOutputFileArtifact(opaquePreservedPath);
var opaqueUnpreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaqueUnpreservedDir"));
var outputUnderUnpreservedOpaque = CreateOutputFileArtifact(opaqueUnpreservedPath);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.WriteFile(outputUnderPreservedOpaque, CONTENT, doNotInfer: true),
Operation.WriteFile(outputUnderUnpreservedOpaque, CONTENT, doNotInfer: true)
});
builder.AddOutputDirectory(opaquePreservedPath);
builder.AddOutputDirectory(opaqueUnpreservedPath);
builder.Options |= Process.Options.AllowPreserveOutputs;
builder.PreserveOutputWhitelist = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(opaquePreservedPath);
var processAndOutputs = SchedulePipBuilder(builder);
// No cache hit
string outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
XAssert.AreEqual(CONTENT, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Change input
ModifyFile(input);
// No cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId);
// As the opaque output is preserved, the pip appended the existing file.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
// For the file under unpreserved opaque directory, the file was created, so we did not append.
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
// Cache hit
outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: true, id: processAndOutputs.Process.PipId);
// The appended file (CONTENT_TWICE) should remain the same.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque)));
}
/// <summary>
/// Creates a cacheable pip info for an ipc pip
/// </summary>
public static CacheablePipInfo GetIpcCacheInfo(IpcPip pip, PipExecutionContext context, bool omitLazilyMaterializedDependencies)
{
var dependencies = omitLazilyMaterializedDependencies && pip.LazilyMaterializedDependencies.Any()
? ReadOnlyArray <FileArtifact> .From(pip.FileDependencies.Except(pip.LazilyMaterializedDependencies))
: pip.FileDependencies;
return(new CacheablePipInfo(
pip: pip,
context: context,
allowPreserveOutputs: false,
outputs: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(pip.OutputFile.WithAttributes()),
dependencies: dependencies,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty));
}
public void TestSucceedFastPipFail_Early()
{
var opsFail = new[]
{
Operation.Echo("Foo"),
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.Fail()
};
var pipBuilderFail = CreatePipBuilder(opsFail);
pipBuilderFail.Priority = 0;
Process pipFail = SchedulePipBuilder(pipBuilderFail).Process;
var opsA = new[]
{
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.SucceedWithExitCode(1)
};
var pipBuilderA = CreatePipBuilder(opsA);
pipBuilderA.Priority = 1;
pipBuilderA.SucceedFastExitCodes = ReadOnlyArray <int> .FromWithoutCopy(new[] { 1 });
pipBuilderA.SuccessExitCodes = pipBuilderA.SucceedFastExitCodes;
Process pipA = SchedulePipBuilder(pipBuilderA).Process;
var opsB = new[]
{
Operation.WriteFile(CreateOutputFileArtifact())
};
var pipBuilderB = CreatePipBuilder(opsB);
pipBuilderB.Priority = 1;
pipBuilderB.AddInputFile(pipA.GetOutputs().First());
Process pipB = SchedulePipBuilder(pipBuilderB).Process;
var scheduleResult = RunScheduler();
scheduleResult.AssertFailure();
AssertErrorEventLogged(LogEventId.PipProcessError);
}
