private void ProcessOutputs(RushProject project, ProcessBuilder processBuilder)
{
// HACK HACK. We are missing output dirs with exclusions, so we don't include node_modules here, which is a pain for scrubbing.
// So let's add known common directories one by one
// Ideally we'd like to say 'the root of the project without node_modules'. No projects are supposed to write there.
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder.Combine(PathTable, "lib")), SealDirectoryKind.SharedOpaque);
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder.Combine(PathTable, "temp")), SealDirectoryKind.SharedOpaque);
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder.Combine(PathTable, "dist")), SealDirectoryKind.SharedOpaque);
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder.Combine(PathTable, "release")), SealDirectoryKind.SharedOpaque);
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder.Combine(PathTable, "src")), SealDirectoryKind.SharedOpaque);
// HACK HACK. Only non-test projects generate these files at the root of the project. So don't add them twice otherwise graph construction complains
if (!project.Name.EndsWith("_test"))
{
processBuilder.AddOutputFile(new FileArtifact(project.ProjectFolder.Combine(PathTable, "test-api.js"), 0), FileExistence.Optional);
processBuilder.AddOutputFile(new FileArtifact(project.ProjectFolder.Combine(PathTable, "test-api.d.ts"), 0), FileExistence.Optional);
}
// Some projects share their temp folder and some files might get consumed across. So treat it as a regular output directory
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.TempFolder), SealDirectoryKind.SharedOpaque);
// Add all the additional output directories that the rush graph knows about
foreach (var additionalOutput in project.AdditionalOutputDirectories)
{
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(additionalOutput), SealDirectoryKind.SharedOpaque);
}
// Add additional output directories configured in the main config file
AddAdditionalOutputDirectories(processBuilder, project.ProjectFolder);
}
private void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
RushProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
var logFile = logDirectory.Combine(PathTable, "build.log");
// Execute the build command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.BuildCommand));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(">"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(logFile));
processBuilder.AddOutputFile(logFile, FileExistence.Required);
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
/// <summary>
/// Adds output file option.
/// </summary>
public ArgumentsBuilder AddOutputOption(string optionName, AbsolutePath outputPath)
{
Contract.Requires(outputPath.IsValid);
AddOption(optionName, outputPath, (b, v) => b.Add(outputPath));
m_processBuilder.AddOutputFile(outputPath);
return(this);
}
private void AddOutputs(NinjaNode node, ProcessBuilder processBuilder)
{
foreach (AbsolutePath output in node.Outputs)
{
FileArtifact file;
if (m_outputFileArtifacts.TryGetValue(output, out file))
{
processBuilder.AddOutputFile(file, FileExistence.Required);
}
else
{
processBuilder.AddOutputFile(output, FileExistence.Required);
}
}
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(m_projectRoot), SealDirectoryKind.SharedOpaque);
}
private void AddOutputs(NinjaNode node, ProcessBuilder processBuilder)
{
foreach (AbsolutePath output in node.Outputs)
{
// TODO: outputs should be optional/required depending on the Ninja graph semantics instead of always optional
FileArtifact file;
if (m_outputFileArtifacts.TryGetValue(output, out file))
{
processBuilder.AddOutputFile(file, FileExistence.Optional);
}
else
{
processBuilder.AddOutputFile(output, FileExistence.Optional);
}
}
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(m_projectRoot), SealDirectoryKind.SharedOpaque);
}
/// <summary>
/// Adds output file option.
/// </summary>
public ArgumentsBuilder AddOutputFileOption(string optionName, AbsolutePath outputPath)
{
Contract.Requires(!string.IsNullOrEmpty(optionName));
Contract.Requires(outputPath.IsValid);
Contract.Assert(!m_finished);
m_dataBuilder.AddPathOption(optionName, outputPath);
m_processBuilder.AddOutputFile(outputPath);
return(this);
}
private bool TryConfigureProcessBuilder(
ProcessBuilder processBuilder,
PipConstructionHelper pipConstructionHelper,
ProjectWithPredictions project,
QualifierId qualifierId,
out AbsolutePath outputResultCacheFile,
out string failureDetail)
{
outputResultCacheFile = AbsolutePath.Invalid;
if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
{
failureDetail = "Failed to construct tooldefinition";
return(false);
}
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// Run in a container if specified
if (m_resolverSettings.RunInContainer)
{
processBuilder.Options |= Process.Options.NeedsToRunInContainer;
processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
}
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites;
SetUntrackedFilesAndDirectories(processBuilder);
// Add the log directory and its corresponding files
var qualifier = m_context.QualifierTable.GetQualifier(qualifierId);
AbsolutePath logDirectory = GetLogDirectory(project, qualifier);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);
if (m_resolverSettings.EnableBinLogTracing == true)
{
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
}
// Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
// it produces an output cache file. This file is placed on the (unique) object directory for this project
if (m_resolverSettings.UseLegacyProjectIsolation != true)
{
var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
}
// Path to the project
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
// Response file with the rest of the arguments
var rspFileSpec = ResponseFileSpecification.Builder()
.AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
.ForceCreation(true)
.Prefix("@")
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
{
return(false);
}
// Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
// It'll cause full cache misses if we try to hash it as an input, however, so exclude.
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));
// GlobalUnsafePassthroughEnvironmentVariables
processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable);
// mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
// However this process will live beyond the build.cmd or msbuild.exe call.
// Allow the pip job object to clean the process without complaint.
//
// vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
// Just let it be killed.
// TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
//
// conhost.exe: This process needs a little bit more time to finish after the main process, but killing it right away
// is inconsequential.
//
// All child processes: Don't wait to kill the processes.
// CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
PathAtom.Create(m_context.StringTable, "vctip.exe"),
PathAtom.Create(m_context.StringTable, "conhost.exe"),
PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe"));
// There are some cases (e.g. a 64-bit MSBuild launched as a child process from a 32-bit MSBuild instance) where
// processes need a little bit more time to finish. Increasing the timeout does not affect job objects where no child
// processes survive, or job object where the only surviving processes are the ones explicitly allowed to survive (which
// are killed immediately). So overall, this non-zero timeout will only make some pips that would have failed to take a little
// bit longer (and hopefully succeed)
processBuilder.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(500);
SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);
failureDetail = string.Empty;
return(true);
}
private bool TryConfigureProcessBuilder(
ProcessBuilder processBuilder,
PipConstructionHelper pipConstructionHelper,
ProjectWithPredictions project,
QualifierId qualifierId,
out AbsolutePath outputResultCacheFile,
out string failureDetail)
{
outputResultCacheFile = AbsolutePath.Invalid;
if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
{
failureDetail = "Failed to construct tooldefinition";
return(false);
}
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// Run in a container if specified
if (m_resolverSettings.RunInContainer)
{
processBuilder.Options |= Process.Options.NeedsToRunInContainer;
processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
}
// Until we can deal with double writes in a better way, this unsafe option allows the build to progress and
// prints warnings
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.UnsafeFirstDoubleWriteWins;
SetUntrackedFilesAndDirectories(processBuilder);
// Add the log directory and its corresponding files
var qualifier = m_context.QualifierTable.GetQualifier(qualifierId);
AbsolutePath logDirectory = GetLogDirectory(project, qualifier);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);
if (m_resolverSettings.EnableBinLogTracing == true)
{
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
}
// Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
// it produces an output cache file. This file is placed on the (unique) object directory for this project
if (m_resolverSettings.UseLegacyProjectIsolation != true)
{
var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
}
// Path to the project
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
// Response file with the rest of the arguments
var rspFileSpec = ResponseFileSpecification.Builder()
.AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
.ForceCreation(true)
.Prefix("@")
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
if (!TryAddMsBuildArguments(project, qualifier, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
{
return(false);
}
// Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
// It'll cause full cache misses if we try to hash it as an input, however, so exclude.
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));
// mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
// However this process will live beyond the build.cmd or msbuild.exe call.
// Allow the pip job object to clean the process without complaint.
//
// vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
// Just let it be killed.
// TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
//
// conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing
// this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other
// arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is
// being investigated. TODO: revisit this once this is fixed.
//
// All child processes: Don't wait to kill the processes.
// CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
PathAtom.Create(m_context.StringTable, "vctip.exe"),
PathAtom.Create(m_context.StringTable, "conhost.exe"));
processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero;
SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);
failureDetail = string.Empty;
return(true);
}