private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId) { SeparateExecutableFromCommands(node.Command, out string executable, out string args); if (!TryFindExecutablePath(executable, out AbsolutePath exePath)) { Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command); return(false); } FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath); processBuilder.Executable = prExeArtifact; processBuilder.AddInputFile(prExeArtifact); using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(args); processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } if (node.ResponseFile.HasValue) { using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(node.ResponseFile?.Content); PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping); // We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false) // because that information is already in node.Command var rspFileSpec = ResponseFileSpecification.Builder() .ExplicitData(responseFileData) .RequiresArgument(false) .ExplicitPath((AbsolutePath)node.ResponseFile?.Path) .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); } } // TODO: Maybe a better description. Add ninja description or change command for input/outputs processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]")); processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable; processBuilder.EnableTempDirectory(); // Working directory - the directory containing the ninja spec file // Ninja generators may express paths relative to this processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable)); // Untrack directories UntrackFilesAndDirectories(processBuilder); // Allow some surviving child process AddRequiredSurvivingChildren(processBuilder); // Environment variables SetEnvironmentVariables(processBuilder, node); return(true); }
private bool TryConfigureProcessBuilder( ProcessBuilder processBuilder, PipConstructionHelper pipConstructionHelper, ProjectWithPredictions project, QualifierId qualifierId, out AbsolutePath outputResultCacheFile, out string failureDetail) { outputResultCacheFile = AbsolutePath.Invalid; if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project)) { failureDetail = "Failed to construct tooldefinition"; return(false); } // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable)); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // Run in a container if specified if (m_resolverSettings.RunInContainer) { processBuilder.Options |= Process.Options.NeedsToRunInContainer; processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs; } // By default the double write policy is to allow same content double writes. processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites; SetUntrackedFilesAndDirectories(processBuilder); // Add the log directory and its corresponding files var qualifier = m_context.QualifierTable.GetQualifier(qualifierId); AbsolutePath logDirectory = GetLogDirectory(project, qualifier); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional); if (m_resolverSettings.EnableBinLogTracing == true) { processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional); } // Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore // it produces an output cache file. This file is placed on the (unique) object directory for this project if (m_resolverSettings.UseLegacyProjectIsolation != true) { var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable)); outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName)); processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required); } // Path to the project processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath)); // Response file with the rest of the arguments var rspFileSpec = ResponseFileSpecification.Builder() .AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor()) .ForceCreation(true) .Prefix("@") .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail)) { return(false); } // Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts. // It'll cause full cache misses if we try to hash it as an input, however, so exclude. processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar)); // GlobalUnsafePassthroughEnvironmentVariables processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable); // mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution. // However this process will live beyond the build.cmd or msbuild.exe call. // Allow the pip job object to clean the process without complaint. // // vctip.exe: On any compile error this telemetry upload exe will be run as a detached process. // Just let it be killed. // TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14 // // conhost.exe: This process needs a little bit more time to finish after the main process, but killing it right away // is inconsequential. // // All child processes: Don't wait to kill the processes. // CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero" processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy( PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"), PathAtom.Create(m_context.StringTable, "vctip.exe"), PathAtom.Create(m_context.StringTable, "conhost.exe"), PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe")); // There are some cases (e.g. a 64-bit MSBuild launched as a child process from a 32-bit MSBuild instance) where // processes need a little bit more time to finish. Increasing the timeout does not affect job objects where no child // processes survive, or job object where the only surviving processes are the ones explicitly allowed to survive (which // are killed immediately). So overall, this non-zero timeout will only make some pips that would have failed to take a little // bit longer (and hopefully succeed) processBuilder.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(500); SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder); failureDetail = string.Empty; return(true); }
private bool TryConfigureProcessBuilder( ProcessBuilder processBuilder, PipConstructionHelper pipConstructionHelper, ProjectWithPredictions project, QualifierId qualifierId, out AbsolutePath outputResultCacheFile, out string failureDetail) { outputResultCacheFile = AbsolutePath.Invalid; if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project)) { failureDetail = "Failed to construct tooldefinition"; return(false); } // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable)); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // Run in a container if specified if (m_resolverSettings.RunInContainer) { processBuilder.Options |= Process.Options.NeedsToRunInContainer; processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs; } // Until we can deal with double writes in a better way, this unsafe option allows the build to progress and // prints warnings processBuilder.DoubleWritePolicy |= DoubleWritePolicy.UnsafeFirstDoubleWriteWins; SetUntrackedFilesAndDirectories(processBuilder); // Add the log directory and its corresponding files var qualifier = m_context.QualifierTable.GetQualifier(qualifierId); AbsolutePath logDirectory = GetLogDirectory(project, qualifier); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional); if (m_resolverSettings.EnableBinLogTracing == true) { processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional); } // Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore // it produces an output cache file. This file is placed on the (unique) object directory for this project if (m_resolverSettings.UseLegacyProjectIsolation != true) { var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable)); outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName)); processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required); } // Path to the project processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath)); // Response file with the rest of the arguments var rspFileSpec = ResponseFileSpecification.Builder() .AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor()) .ForceCreation(true) .Prefix("@") .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); if (!TryAddMsBuildArguments(project, qualifier, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail)) { return(false); } // Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts. // It'll cause full cache misses if we try to hash it as an input, however, so exclude. processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar)); // mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution. // However this process will live beyond the build.cmd or msbuild.exe call. // Allow the pip job object to clean the process without complaint. // // vctip.exe: On any compile error this telemetry upload exe will be run as a detached process. // Just let it be killed. // TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14 // // conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing // this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other // arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is // being investigated. TODO: revisit this once this is fixed. // // All child processes: Don't wait to kill the processes. // CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero" processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy( PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"), PathAtom.Create(m_context.StringTable, "vctip.exe"), PathAtom.Create(m_context.StringTable, "conhost.exe")); processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero; SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder); failureDetail = string.Empty; return(true); }