public void ProcessMustRunLocalDueToTag()
{
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
// This configuration will test that must-run-local tags take precendence over can-run-remote tags.
const string MustRunLocalTag = nameof(MustRunLocalTag);
Configuration.Schedule.ProcessMustRunLocalTags = new List <string> {
MustRunLocalTag
};
Configuration.Schedule.ProcessCanRunRemoteTags = new List <string> {
MustRunLocalTag
};
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.AddTags(Context.StringTable, MustRunLocalTag);
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
public void ExecutionProcessReadingStdIn()
{
FileArtifact stdOut = CreateOutputFileArtifact();
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadStdIn() });
PipDataBuilder dataBuilder = new PipDataBuilder(Context.PathTable.StringTable);
dataBuilder.Add("Data0");
dataBuilder.Add("Data1");
dataBuilder.Add("Data2");
builder.StandardInput = global::BuildXL.Pips.StandardInput.CreateFromData(dataBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.NoEscaping));
builder.SetStandardOutputFile(stdOut.Path);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
string[] output = File.ReadAllLines(ArtifactToString(stdOut));
string actualContent = string.Join(Environment.NewLine, output);
XAssert.AreEqual(3, output.Length, "Actual content: {0}{1}", Environment.NewLine, string.Join(Environment.NewLine, output));
for (int i = 0; i < 3; ++i)
{
XAssert.AreEqual("Data" + i, output[i], "Actual content: {0}", output[i]);
}
}
public void RunSingleProcess()
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
}
public void ExecutionRespectFileAccessManifest()
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile(), doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 1);
AssertErrorEventLogged(EventId.FileMonitoringError, count: 1);
}
public void RunMultipleAdminRequiredProcesses()
{
for (int i = 0; i < 5; ++i)
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
}
RunScheduler().AssertSuccess();
}
public void RunProcessReferencingUnsetEnvironmentVariable()
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
// HOMEDRIVE is no longer set by CloudBuild. Test to make sure environment variable forwarding code can handle this
builder.SetPassthroughEnvironmentVariable(Context.StringTable.AddString("HOMEDRIVE"));
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
}
private void CreateAndScheduleProcess(out FileArtifact output, out ProcessWithOutputs scheduledProcess)
{
FileArtifact input = CreateSourceFile();
output = CreateOutputFileArtifact();
var pipBuilder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.WriteFile(output)
});
scheduledProcess = SchedulePipBuilder(pipBuilder);
}
public void ExecutionRespectFailure()
{
ProcessBuilder builder = CreatePipBuilder(new[] {
Operation.ReadFile(CreateSourceFile()),
Operation.Fail(),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertErrorEventLogged(EventId.PipProcessError, count: 1);
}
public void RunSingleProcessRemotely()
{
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
RunScheduler().AssertCacheHit(process.Process.PipId);
}
public void RemoteFallbackProcessRetryToRunLocally()
{
RemoteProcessManagerFactory.RemoteProcessManager = new Lazy <IRemoteProcessManager>(() => new TestRemoteProcessManager(shouldRunLocally: true));
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
public void ExternalToolPreserveWarning()
{
ProcessBuilder builder = CreatePipBuilder(new[] {
Operation.ReadFile(CreateSourceFile()),
Operation.Echo("WARN this is a warning"),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.Options |= Process.Options.RequiresAdmin;
builder.WarningRegex = new RegexDescriptor(StringId.Create(Context.StringTable, @"^WARN"), System.Text.RegularExpressions.RegexOptions.IgnoreCase);
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
AssertWarningEventLogged(EventId.PipProcessWarning, count: 1);
}
public void ExecutionRespectTimeout()
{
ProcessBuilder builder = CreatePipBuilder(new[] {
Operation.ReadFile(CreateSourceFile()),
Operation.Block(),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.Timeout = TimeSpan.FromSeconds(1);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertErrorEventLogged(ProcessesLogEventId.PipProcessTookTooLongError, count: 1);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessError, count: 1);
}
public void ExecutionRecordsReportedFileAccesses()
{
FileArtifact sourceFile = CreateSourceFile();
SealDirectory sourceDirectory = CreateAndScheduleSealDirectory(sourceFile.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories);
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.AddInputDirectory(sourceDirectory.Directory);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
File.WriteAllText(ArtifactToString(sourceFile), Guid.NewGuid().ToString());
RunScheduler().AssertCacheMiss(process.Process.PipId);
}
private void ScheduleDoubleWriteProducers(
AbsolutePath sharedOpaqueDirPath,
FileArtifact doubleWriteArtifact,
ContainerIsolationLevel containerIsolationLevel,
RewritePolicy doubleWritePolicy,
out ProcessWithOutputs firstProducer,
out ProcessWithOutputs secondProducer)
{
var firstProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "first", sharedOpaqueDirPath);
firstProducer = SchedulePipBuilder(firstProducerBuilder);
var secondProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "second", sharedOpaqueDirPath);
// Let's order this so who is the violator is deterministic
secondProducerBuilder.AddInputDirectory(firstProducer.Process.DirectoryOutputs.First());
secondProducer = SchedulePipBuilder(secondProducerBuilder);
}
public void ExecutionUntrackTempFolder()
{
AbsolutePath tempDirectory = CreateUniqueDirectory(ObjectRoot);
FileArtifact tempFile = CreateOutputFileArtifact(tempDirectory);
ProcessBuilder builder = CreatePipBuilder(new[]
{
Operation.ReadFile(CreateSourceFile()),
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.WriteFile(tempFile, doNotInfer: true),
Operation.ReadFile(tempFile, doNotInfer: true)
});
builder.Options |= Process.Options.RequiresAdmin;
builder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempDirectory));
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
}
public void ExecutionRespectTimeout()
{
ProcessBuilder builder = CreatePipBuilder(new[] {
Operation.ReadFile(CreateSourceFile()),
Operation.Block(),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.Timeout = TimeSpan.FromSeconds(1);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertErrorEventLogged(ProcessesLogEventId.PipProcessTookTooLongError, count: 1);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessError, count: 1);
if (OperatingSystemHelper.IsUnixOS)
{
// Creating dump is not supported on non-Windows.
AssertWarningEventLogged(ProcessesLogEventId.PipFailedToCreateDumpFile, count: 1);
}
}
/// <summary>
/// Creates a <paramref name="pip"/> that reads from <paramref name="input"/> and writes to <paramref name="output"/>.
/// The output id written in a <paramref name="sharedOpaque"/> directory with root <paramref name="root"/>.
/// </summary>
private void CreatePip(string root, out FileArtifact input, out FileArtifact output, out ProcessWithOutputs pip, out DirectoryArtifact sharedOpaque)
{
string sharedOpaqueDir = Path.Combine(ObjectRoot, root);
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
output = CreateOutputFileArtifact(sharedOpaqueDir);
input = CreateSourceFile();
pip = CreateAndScheduleSharedOpaqueProducer(sharedOpaqueDir, fileToProduceStatically: FileArtifact.Invalid, input, new KeyValuePair <FileArtifact, string>(output, null));
sharedOpaque = pip.ProcessOutputs.GetOutputDirectories().Single().Root;
}
