private void AddInputs(NinjaNode node, ProcessBuilder processBuilder)
{
foreach (AbsolutePath input in node.Inputs.Where(i => !m_outputFileArtifacts.ContainsKey(i))) // We only want source files. If the file is in m_outputFileArtifacts then it was output by someone else
{ // and we have that dependency expressed in node.
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(input));
}
foreach (NinjaNode dependency in node.Dependencies)
{
bool outputsPresent = m_processOutputs.TryGetValue(dependency, out ProcessOutputs processOutputs);
string ListOutputs(NinjaNode n) => string.Join(" ", n.Outputs.Select(x => x.GetName(m_context.PathTable).ToString(m_context.StringTable)).ToList());
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: [build { ListOutputs(dependency) }] missing, dependency of [build { ListOutputs(node)} ]");
}
foreach (FileArtifact output in processOutputs.GetOutputFiles())
{
if (node.Inputs.Contains(output.Path))
{
processBuilder.AddInputFile(output);
}
}
}
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
RushProject project,
ProcessBuilder processBuilder)
{
// Add all explicitly declared source files
foreach (AbsolutePath sourceFile in project.SourceFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(sourceFile));
}
// Add package.json, which should always be present at the root of the project
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.PackageJsonFile(PathTable)));
// If dependencies should be tracked via the project-level shrinkwrap-deps file, then force an input
// dependency on it
if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.ShrinkwrapDepsFile(PathTable)));
}
// In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for
// JavaScript projects.
var transitiveReferences = new HashSet <RushProject>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
IEnumerable <RushProject> references = transitiveReferences;
foreach (RushProject projectReference in references)
{
// If the project is referencing something that was not scheduled, just skip it
if (!projectReference.CanBeScheduled())
{
// We have already logged this case as an informational when building the project graph
continue;
}
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
// Add all known output files, but exclude logs
foreach (FileArtifact output in processOutputs.GetOutputFiles()
.Where(fa => !fa.Path.IsWithin(m_context.PathTable, LogDirectoryBase(m_frontEndHost.Configuration, m_context.PathTable))))
{
processBuilder.AddInputFile(output);
}
}
}
private bool TrySetBuildToolExecutor(
PipConstructionHelper pipConstructionHelper,
ProcessBuilder processBuilder,
ProjectWithPredictions project)
{
// If we should use the dotnet core version of msbuild, the executable for the pip is dotnet.exe instead of msbuild.exe, and
// the first argument is msbuild.dll
FileArtifact cmdExeArtifact;
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_dotnetExePath);
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(m_msBuildPath));
}
else
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildPath);
}
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}"));
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
RushProject project,
ProcessBuilder processBuilder)
{
var argumentsBuilder = processBuilder.ArgumentsBuilder;
IEnumerable <RushProject> references;
// In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for
// JavaScript projects.
var transitiveReferences = new HashSet <RushProject>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
foreach (RushProject projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
// Add all known output files
foreach (FileArtifact output in processOutputs.GetOutputFiles())
{
processBuilder.AddInputFile(output);
}
}
}
/// <summary>
/// Augments the processBuilder with the OS dependencies
/// </summary>
public void ProcessDefaults(ProcessBuilder processBuilder)
{
if ((processBuilder.Options & Process.Options.DependsOnCurrentOs) != 0)
{
foreach (var inputFile in m_inputFiles)
{
processBuilder.AddInputFile(inputFile);
}
foreach (var inputDirectory in m_inputDirectories)
{
processBuilder.AddInputDirectory(inputDirectory);
}
foreach (var untrackedFile in m_untrackedFiles)
{
processBuilder.AddUntrackedFile(untrackedFile);
}
foreach (var untrackedDirectory in m_untrackedDirectories)
{
processBuilder.AddUntrackedDirectoryScope(untrackedDirectory);
}
}
}
/// <summary>
/// Adds input file option.
/// </summary>
public ArgumentsBuilder AddInputOption(string optionName, FileArtifact inputFile)
{
Contract.Requires(inputFile.IsValid);
AddOption(optionName, inputFile, (b, v) => b.Add(inputFile.Path));
m_processBuilder.AddInputFile(inputFile);
return(this);
}
/// <summary>
/// Adds input file option.
/// </summary>
public ArgumentsBuilder AddInputFileOption(string optionName, FileArtifact inputFile)
{
Contract.Requires(!string.IsNullOrEmpty(optionName));
Contract.Requires(inputFile.IsValid);
Contract.Assert(!m_finished);
m_dataBuilder.AddPathOption(optionName, inputFile.Path);
m_processBuilder.AddInputFile(inputFile);
return(this);
}
/// <inheritdoc/>
protected override void ProcessInputs(
JavaScriptProject project,
ProcessBuilder processBuilder)
{
base.ProcessInputs(project, processBuilder);
// If dependencies should be tracked via the project-level shrinkwrap-deps file, then force an input
// dependency on it
if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.ShrinkwrapDepsFile(PathTable)));
}
}
private void SetCmdTool(
ProcessBuilder processBuilder,
RushProject project)
{
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.Name}"));
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out ProcessOutputs processOutputs);
Contract.Assert(outputsPresent, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
// Add all known opaque directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
}
}
private bool TrySetBuildToolExecutor(
PipConstructionHelper pipConstructionHelper,
ProcessBuilder processBuilder,
ProjectWithPredictions project)
{
FileArtifact cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildExePath);
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
AbsolutePath toolDir = m_msBuildExePath.GetParent(PathTable);
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}"));
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Predicted output directories for all direct dependencies, plus the output directories for the given project itself
var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders);
// Add all predicted inputs that are recognized as true source files
// This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means
// we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case
// of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up.
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
// If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file
// In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory
// dependency.
if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder)))
{
continue;
}
// If any of the predicted inputs is under an untracked directory scope, don't add it as an input
if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory)))
{
continue;
}
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => projectReference.PredictedTargetsToExecute.Targets.Count != 0);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId)
{
SeparateExecutableFromCommands(node.Command, out string executable, out string args);
if (!TryFindExecutablePath(executable, out AbsolutePath exePath))
{
Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command);
return(false);
}
FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath);
processBuilder.Executable = prExeArtifact;
processBuilder.AddInputFile(prExeArtifact);
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(args);
processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
if (node.ResponseFile.HasValue)
{
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(node.ResponseFile?.Content);
PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping);
// We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false)
// because that information is already in node.Command
var rspFileSpec = ResponseFileSpecification.Builder()
.ExplicitData(responseFileData)
.RequiresArgument(false)
.ExplicitPath((AbsolutePath)node.ResponseFile?.Path)
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
}
}
// TODO: Maybe a better description. Add ninja description or change command for input/outputs
processBuilder.ToolDescription = StringId.Create(m_context.StringTable,
I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]"));
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable;
processBuilder.EnableTempDirectory();
// Working directory - the directory containing the ninja spec file
// Ninja generators may express paths relative to this
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable));
// Untrack directories
UntrackFilesAndDirectories(processBuilder);
// Allow some surviving child process
AddRequiredSurvivingChildren(processBuilder);
// Environment variables
SetEnvironmentVariables(processBuilder, node);
return(true);
}