public void ProcessMustRunLocalDueToTag() { // Force run remotely. Configuration.Schedule.RemotingThresholdMultiplier = 0.0; // This configuration will test that must-run-local tags take precendence over can-run-remote tags. const string MustRunLocalTag = nameof(MustRunLocalTag); Configuration.Schedule.ProcessMustRunLocalTags = new List <string> { MustRunLocalTag }; Configuration.Schedule.ProcessCanRunRemoteTags = new List <string> { MustRunLocalTag }; ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.AddTags(Context.StringTable, MustRunLocalTag); ProcessWithOutputs process = SchedulePipBuilder(builder); ScheduleRunResult result = RunScheduler().AssertSuccess(); XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses)); XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker)); XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries)); }
public void ExecutionProcessReadingStdIn() { FileArtifact stdOut = CreateOutputFileArtifact(); ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadStdIn() }); PipDataBuilder dataBuilder = new PipDataBuilder(Context.PathTable.StringTable); dataBuilder.Add("Data0"); dataBuilder.Add("Data1"); dataBuilder.Add("Data2"); builder.StandardInput = global::BuildXL.Pips.StandardInput.CreateFromData(dataBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.NoEscaping)); builder.SetStandardOutputFile(stdOut.Path); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); string[] output = File.ReadAllLines(ArtifactToString(stdOut)); string actualContent = string.Join(Environment.NewLine, output); XAssert.AreEqual(3, output.Length, "Actual content: {0}{1}", Environment.NewLine, string.Join(Environment.NewLine, output)); for (int i = 0; i < 3; ++i) { XAssert.AreEqual("Data" + i, output[i], "Actual content: {0}", output[i]); } }
public void RunSingleProcess() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); RunScheduler().AssertCacheHit(process.Process.PipId); }
public void ExecutionRespectFileAccessManifest() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile(), doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 1); AssertErrorEventLogged(EventId.FileMonitoringError, count: 1); }
public void RunMultipleAdminRequiredProcesses() { for (int i = 0; i < 5; ++i) { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); } RunScheduler().AssertSuccess(); }
public void RunProcessReferencingUnsetEnvironmentVariable() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); // HOMEDRIVE is no longer set by CloudBuild. Test to make sure environment variable forwarding code can handle this builder.SetPassthroughEnvironmentVariable(Context.StringTable.AddString("HOMEDRIVE")); RunScheduler().AssertSuccess(); RunScheduler().AssertCacheHit(process.Process.PipId); }
private void CreateAndScheduleProcess(out FileArtifact output, out ProcessWithOutputs scheduledProcess) { FileArtifact input = CreateSourceFile(); output = CreateOutputFileArtifact(); var pipBuilder = CreatePipBuilder(new Operation[] { Operation.ReadFile(input), Operation.WriteFile(output) }); scheduledProcess = SchedulePipBuilder(pipBuilder); }
public void ExecutionRespectFailure() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.Fail(), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertErrorEventLogged(EventId.PipProcessError, count: 1); }
public void RunSingleProcessRemotely() { // Force run remotely. Configuration.Schedule.RemotingThresholdMultiplier = 0.0; ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); ProcessWithOutputs process = SchedulePipBuilder(builder); ScheduleRunResult result = RunScheduler().AssertSuccess(); XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses)); XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker)); XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries)); RunScheduler().AssertCacheHit(process.Process.PipId); }
public void RemoteFallbackProcessRetryToRunLocally() { RemoteProcessManagerFactory.RemoteProcessManager = new Lazy <IRemoteProcessManager>(() => new TestRemoteProcessManager(shouldRunLocally: true)); // Force run remotely. Configuration.Schedule.RemotingThresholdMultiplier = 0.0; ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) }); ProcessWithOutputs process = SchedulePipBuilder(builder); ScheduleRunResult result = RunScheduler().AssertSuccess(); XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses)); XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker)); XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries)); }
public void ExternalToolPreserveWarning() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.Echo("WARN this is a warning"), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Options |= Process.Options.RequiresAdmin; builder.WarningRegex = new RegexDescriptor(StringId.Create(Context.StringTable, @"^WARN"), System.Text.RegularExpressions.RegexOptions.IgnoreCase); ProcessWithOutputs process = SchedulePipBuilder(builder); ScheduleRunResult result = RunScheduler().AssertSuccess(); AssertWarningEventLogged(EventId.PipProcessWarning, count: 1); }
public void ExecutionRespectTimeout() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.Block(), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Timeout = TimeSpan.FromSeconds(1); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertErrorEventLogged(ProcessesLogEventId.PipProcessTookTooLongError, count: 1); AssertErrorEventLogged(ProcessesLogEventId.PipProcessError, count: 1); }
public void ExecutionRecordsReportedFileAccesses() { FileArtifact sourceFile = CreateSourceFile(); SealDirectory sourceDirectory = CreateAndScheduleSealDirectory(sourceFile.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories); ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.AddInputDirectory(sourceDirectory.Directory); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); RunScheduler().AssertCacheHit(process.Process.PipId); File.WriteAllText(ArtifactToString(sourceFile), Guid.NewGuid().ToString()); RunScheduler().AssertCacheMiss(process.Process.PipId); }
private void ScheduleDoubleWriteProducers( AbsolutePath sharedOpaqueDirPath, FileArtifact doubleWriteArtifact, ContainerIsolationLevel containerIsolationLevel, RewritePolicy doubleWritePolicy, out ProcessWithOutputs firstProducer, out ProcessWithOutputs secondProducer) { var firstProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "first", sharedOpaqueDirPath); firstProducer = SchedulePipBuilder(firstProducerBuilder); var secondProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "second", sharedOpaqueDirPath); // Let's order this so who is the violator is deterministic secondProducerBuilder.AddInputDirectory(firstProducer.Process.DirectoryOutputs.First()); secondProducer = SchedulePipBuilder(secondProducerBuilder); }
public void ExecutionUntrackTempFolder() { AbsolutePath tempDirectory = CreateUniqueDirectory(ObjectRoot); FileArtifact tempFile = CreateOutputFileArtifact(tempDirectory); ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()), Operation.WriteFile(tempFile, doNotInfer: true), Operation.ReadFile(tempFile, doNotInfer: true) }); builder.Options |= Process.Options.RequiresAdmin; builder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempDirectory)); ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); RunScheduler().AssertCacheHit(process.Process.PipId); }
public void ExecutionRespectTimeout() { ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.Block(), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.Timeout = TimeSpan.FromSeconds(1); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertErrorEventLogged(ProcessesLogEventId.PipProcessTookTooLongError, count: 1); AssertErrorEventLogged(ProcessesLogEventId.PipProcessError, count: 1); if (OperatingSystemHelper.IsUnixOS) { // Creating dump is not supported on non-Windows. AssertWarningEventLogged(ProcessesLogEventId.PipFailedToCreateDumpFile, count: 1); } }
/// <summary> /// Creates a <paramref name="pip"/> that reads from <paramref name="input"/> and writes to <paramref name="output"/>. /// The output id written in a <paramref name="sharedOpaque"/> directory with root <paramref name="root"/>. /// </summary> private void CreatePip(string root, out FileArtifact input, out FileArtifact output, out ProcessWithOutputs pip, out DirectoryArtifact sharedOpaque) { string sharedOpaqueDir = Path.Combine(ObjectRoot, root); AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir); output = CreateOutputFileArtifact(sharedOpaqueDir); input = CreateSourceFile(); pip = CreateAndScheduleSharedOpaqueProducer(sharedOpaqueDir, fileToProduceStatically: FileArtifact.Invalid, input, new KeyValuePair <FileArtifact, string>(output, null)); sharedOpaque = pip.ProcessOutputs.GetOutputDirectories().Single().Root; }