private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId)
{
SeparateExecutableFromCommands(node.Command, out string executable, out string args);
if (!TryFindExecutablePath(executable, out AbsolutePath exePath))
{
Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command);
return(false);
}
FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath);
processBuilder.Executable = prExeArtifact;
processBuilder.AddInputFile(prExeArtifact);
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(args);
processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
if (node.ResponseFile.HasValue)
{
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(node.ResponseFile?.Content);
PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping);
// We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false)
// because that information is already in node.Command
var rspFileSpec = ResponseFileSpecification.Builder()
.ExplicitData(responseFileData)
.RequiresArgument(false)
.ExplicitPath((AbsolutePath)node.ResponseFile?.Path)
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
}
}
// TODO: Maybe a better description. Add ninja description or change command for input/outputs
processBuilder.ToolDescription = StringId.Create(m_context.StringTable,
I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]"));
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable;
processBuilder.EnableTempDirectory();
// Working directory - the directory containing the ninja spec file
// Ninja generators may express paths relative to this
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable));
// Untrack directories
UntrackFilesAndDirectories(processBuilder);
// Allow some surviving child process
AddRequiredSurvivingChildren(processBuilder);
// Environment variables
SetEnvironmentVariables(processBuilder, node);
return(true);
}
private bool TryConfigureProcessBuilder(
ProcessBuilder processBuilder,
PipConstructionHelper pipConstructionHelper,
ProjectWithPredictions project,
QualifierId qualifierId,
out AbsolutePath outputResultCacheFile,
out string failureDetail)
{
outputResultCacheFile = AbsolutePath.Invalid;
if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
{
failureDetail = "Failed to construct tooldefinition";
return(false);
}
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// Run in a container if specified
if (m_resolverSettings.RunInContainer)
{
processBuilder.Options |= Process.Options.NeedsToRunInContainer;
processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
}
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites;
SetUntrackedFilesAndDirectories(processBuilder);
// Add the log directory and its corresponding files
var qualifier = m_context.QualifierTable.GetQualifier(qualifierId);
AbsolutePath logDirectory = GetLogDirectory(project, qualifier);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);
if (m_resolverSettings.EnableBinLogTracing == true)
{
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
}
// Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
// it produces an output cache file. This file is placed on the (unique) object directory for this project
if (m_resolverSettings.UseLegacyProjectIsolation != true)
{
var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
}
// Path to the project
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
// Response file with the rest of the arguments
var rspFileSpec = ResponseFileSpecification.Builder()
.AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
.ForceCreation(true)
.Prefix("@")
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
{
return(false);
}
// Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
// It'll cause full cache misses if we try to hash it as an input, however, so exclude.
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));
// GlobalUnsafePassthroughEnvironmentVariables
processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable);
// mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
// However this process will live beyond the build.cmd or msbuild.exe call.
// Allow the pip job object to clean the process without complaint.
//
// vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
// Just let it be killed.
// TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
//
// conhost.exe: This process needs a little bit more time to finish after the main process, but killing it right away
// is inconsequential.
//
// All child processes: Don't wait to kill the processes.
// CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
PathAtom.Create(m_context.StringTable, "vctip.exe"),
PathAtom.Create(m_context.StringTable, "conhost.exe"),
PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe"));
// There are some cases (e.g. a 64-bit MSBuild launched as a child process from a 32-bit MSBuild instance) where
// processes need a little bit more time to finish. Increasing the timeout does not affect job objects where no child
// processes survive, or job object where the only surviving processes are the ones explicitly allowed to survive (which
// are killed immediately). So overall, this non-zero timeout will only make some pips that would have failed to take a little
// bit longer (and hopefully succeed)
processBuilder.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(500);
SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);
failureDetail = string.Empty;
return(true);
}
private bool TryConfigureProcessBuilder(
ProcessBuilder processBuilder,
PipConstructionHelper pipConstructionHelper,
ProjectWithPredictions project,
QualifierId qualifierId,
out AbsolutePath outputResultCacheFile,
out string failureDetail)
{
outputResultCacheFile = AbsolutePath.Invalid;
if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
{
failureDetail = "Failed to construct tooldefinition";
return(false);
}
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// Run in a container if specified
if (m_resolverSettings.RunInContainer)
{
processBuilder.Options |= Process.Options.NeedsToRunInContainer;
processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
}
// Until we can deal with double writes in a better way, this unsafe option allows the build to progress and
// prints warnings
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.UnsafeFirstDoubleWriteWins;
SetUntrackedFilesAndDirectories(processBuilder);
// Add the log directory and its corresponding files
var qualifier = m_context.QualifierTable.GetQualifier(qualifierId);
AbsolutePath logDirectory = GetLogDirectory(project, qualifier);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);
if (m_resolverSettings.EnableBinLogTracing == true)
{
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
}
// Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
// it produces an output cache file. This file is placed on the (unique) object directory for this project
if (m_resolverSettings.UseLegacyProjectIsolation != true)
{
var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
}
// Path to the project
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
// Response file with the rest of the arguments
var rspFileSpec = ResponseFileSpecification.Builder()
.AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
.ForceCreation(true)
.Prefix("@")
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
if (!TryAddMsBuildArguments(project, qualifier, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
{
return(false);
}
// Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
// It'll cause full cache misses if we try to hash it as an input, however, so exclude.
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));
// mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
// However this process will live beyond the build.cmd or msbuild.exe call.
// Allow the pip job object to clean the process without complaint.
//
// vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
// Just let it be killed.
// TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
//
// conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing
// this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other
// arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is
// being investigated. TODO: revisit this once this is fixed.
//
// All child processes: Don't wait to kill the processes.
// CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
PathAtom.Create(m_context.StringTable, "vctip.exe"),
PathAtom.Create(m_context.StringTable, "conhost.exe"));
processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero;
SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);
failureDetail = string.Empty;
return(true);
}
