private void AddInputs(NinjaNode node, ProcessBuilder processBuilder) { foreach (AbsolutePath input in node.Inputs.Where(i => !m_outputFileArtifacts.ContainsKey(i))) // We only want source files. If the file is in m_outputFileArtifacts then it was output by someone else { // and we have that dependency expressed in node. processBuilder.AddInputFile(FileArtifact.CreateSourceFile(input)); } foreach (NinjaNode dependency in node.Dependencies) { bool outputsPresent = m_processOutputs.TryGetValue(dependency, out ProcessOutputs processOutputs); string ListOutputs(NinjaNode n) => string.Join(" ", n.Outputs.Select(x => x.GetName(m_context.PathTable).ToString(m_context.StringTable)).ToList()); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: [build { ListOutputs(dependency) }] missing, dependency of [build { ListOutputs(node)} ]"); } foreach (FileArtifact output in processOutputs.GetOutputFiles()) { if (node.Inputs.Contains(output.Path)) { processBuilder.AddInputFile(output); } } } }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( ProjectWithPredictions project, ProcessBuilder processBuilder) { // Add all predicted inputs foreach (AbsolutePath buildInput in project.PredictedInputFiles) { processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput)); } IEnumerable <ProjectWithPredictions> references; // The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in if (m_resolverSettings.EnableTransitiveProjectReferences == true) { // In this case all the transitive closure is automatically exposed to the project as direct references var transitiveReferences = new HashSet <ProjectWithPredictions>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); references = transitiveReferences; } else { // Only direct dependencies are declared. // Add all known explicit inputs from project references. But rule out // projects that have a known empty list of targets: those projects are not scheduled, so // there is nothing to consume from them. references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty); } var argumentsBuilder = processBuilder.ArgumentsBuilder; foreach (ProjectWithPredictions projectReference in references) { bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}"); } // Add all known output directories foreach (StaticDirectory output in projectOutputs.OutputDirectories) { processBuilder.AddInputDirectory(output.Root); } // If the dependency was built in isolation, this project needs to access the generated cache files if (projectOutputs.BuildsInIsolation) { var outputCache = projectOutputs.OutputCacheFile; processBuilder.AddInputFile(outputCache); // Instruct MSBuild to use the cache file from the associated dependency as an input. // Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode. using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty)) { argumentsBuilder.Add(PipDataAtom.FromString("/irc:")); argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache)); } } } }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( RushProject project, ProcessBuilder processBuilder) { // Add all explicitly declared source files foreach (AbsolutePath sourceFile in project.SourceFiles) { processBuilder.AddInputFile(FileArtifact.CreateSourceFile(sourceFile)); } // Add package.json, which should always be present at the root of the project processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.PackageJsonFile(PathTable))); // If dependencies should be tracked via the project-level shrinkwrap-deps file, then force an input // dependency on it if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true) { processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.ShrinkwrapDepsFile(PathTable))); } // In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for // JavaScript projects. var transitiveReferences = new HashSet <RushProject>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); IEnumerable <RushProject> references = transitiveReferences; foreach (RushProject projectReference in references) { // If the project is referencing something that was not scheduled, just skip it if (!projectReference.CanBeScheduled()) { // We have already logged this case as an informational when building the project graph continue; } bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}"); } // Add all known output directories foreach (StaticDirectory output in processOutputs.GetOutputDirectories()) { processBuilder.AddInputDirectory(output.Root); } // Add all known output files, but exclude logs foreach (FileArtifact output in processOutputs.GetOutputFiles() .Where(fa => !fa.Path.IsWithin(m_context.PathTable, LogDirectoryBase(m_frontEndHost.Configuration, m_context.PathTable)))) { processBuilder.AddInputFile(output); } } }
private bool TrySetBuildToolExecutor( PipConstructionHelper pipConstructionHelper, ProcessBuilder processBuilder, ProjectWithPredictions project) { // If we should use the dotnet core version of msbuild, the executable for the pip is dotnet.exe instead of msbuild.exe, and // the first argument is msbuild.dll FileArtifact cmdExeArtifact; if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) { cmdExeArtifact = FileArtifact.CreateSourceFile(m_dotnetExePath); processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(m_msBuildPath)); } else { cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildPath); } processBuilder.Executable = cmdExeArtifact; processBuilder.AddInputFile(cmdExeArtifact); processBuilder.AddCurrentHostOSDirectories(); processBuilder.AddUntrackedAppDataDirectories(); processBuilder.AddUntrackedProgramDataDirectories(); // Temp directory setup including setting TMP and TEMP env vars. The path to // the temp dir is generated in a consistent fashion between BuildXL runs to // ensure environment value (and hence pip hash) consistency. processBuilder.EnableTempDirectory(); processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}")); return(true); }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( RushProject project, ProcessBuilder processBuilder) { var argumentsBuilder = processBuilder.ArgumentsBuilder; IEnumerable <RushProject> references; // In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for // JavaScript projects. var transitiveReferences = new HashSet <RushProject>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); references = transitiveReferences; foreach (RushProject projectReference in references) { bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}"); } // Add all known output directories foreach (StaticDirectory output in processOutputs.GetOutputDirectories()) { processBuilder.AddInputDirectory(output.Root); } // Add all known output files foreach (FileArtifact output in processOutputs.GetOutputFiles()) { processBuilder.AddInputFile(output); } } }
/// <summary> /// Augments the processBuilder with the OS dependencies /// </summary> public void ProcessDefaults(ProcessBuilder processBuilder) { if ((processBuilder.Options & Process.Options.DependsOnCurrentOs) != 0) { foreach (var inputFile in m_inputFiles) { processBuilder.AddInputFile(inputFile); } foreach (var inputDirectory in m_inputDirectories) { processBuilder.AddInputDirectory(inputDirectory); } foreach (var untrackedFile in m_untrackedFiles) { processBuilder.AddUntrackedFile(untrackedFile); } foreach (var untrackedDirectory in m_untrackedDirectories) { processBuilder.AddUntrackedDirectoryScope(untrackedDirectory); } } }
/// <summary> /// Adds input file option. /// </summary> public ArgumentsBuilder AddInputOption(string optionName, FileArtifact inputFile) { Contract.Requires(inputFile.IsValid); AddOption(optionName, inputFile, (b, v) => b.Add(inputFile.Path)); m_processBuilder.AddInputFile(inputFile); return(this); }
/// <summary> /// Adds input file option. /// </summary> public ArgumentsBuilder AddInputFileOption(string optionName, FileArtifact inputFile) { Contract.Requires(!string.IsNullOrEmpty(optionName)); Contract.Requires(inputFile.IsValid); Contract.Assert(!m_finished); m_dataBuilder.AddPathOption(optionName, inputFile.Path); m_processBuilder.AddInputFile(inputFile); return(this); }
/// <inheritdoc/> protected override void ProcessInputs( JavaScriptProject project, ProcessBuilder processBuilder) { base.ProcessInputs(project, processBuilder); // If dependencies should be tracked via the project-level shrinkwrap-deps file, then force an input // dependency on it if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true) { processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.ShrinkwrapDepsFile(PathTable))); } }
private void SetCmdTool( ProcessBuilder processBuilder, RushProject project) { var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(PathTable, Environment.GetEnvironmentVariable("COMSPEC"))); processBuilder.Executable = cmdExeArtifact; processBuilder.AddInputFile(cmdExeArtifact); processBuilder.AddCurrentHostOSDirectories(); processBuilder.AddUntrackedAppDataDirectories(); processBuilder.AddUntrackedProgramDataDirectories(); // Temp directory setup including setting TMP and TEMP env vars. The path to // the temp dir is generated in a consistent fashion between BuildXL runs to // ensure environment value (and hence pip hash) consistency. processBuilder.EnableTempDirectory(); processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.Name}")); }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( ProjectWithPredictions project, ProcessBuilder processBuilder) { // Add all predicted inputs foreach (AbsolutePath buildInput in project.PredictedInputFiles) { processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput)); } IEnumerable <ProjectWithPredictions> references; // The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in if (m_resolverSettings.EnableTransitiveProjectReferences == true) { // In this case all the transitive closure is automatically exposed to the project as direct references var transitiveReferences = new HashSet <ProjectWithPredictions>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); references = transitiveReferences; } else { // Only direct dependencies are declared. // Add all known explicit inputs from project references. But rule out // projects that have a known empty list of targets: those projects are not scheduled, so // there is nothing to consume from them. references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty); } foreach (ProjectWithPredictions projectReference in references) { bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out ProcessOutputs processOutputs); Contract.Assert(outputsPresent, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}"); // Add all known opaque directories foreach (StaticDirectory output in processOutputs.GetOutputDirectories()) { processBuilder.AddInputDirectory(output.Root); } } }
private bool TrySetBuildToolExecutor( PipConstructionHelper pipConstructionHelper, ProcessBuilder processBuilder, ProjectWithPredictions project) { FileArtifact cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildExePath); processBuilder.Executable = cmdExeArtifact; processBuilder.AddInputFile(cmdExeArtifact); processBuilder.AddCurrentHostOSDirectories(); processBuilder.AddUntrackedAppDataDirectories(); processBuilder.AddUntrackedProgramDataDirectories(); // Temp directory setup including setting TMP and TEMP env vars. The path to // the temp dir is generated in a consistent fashion between BuildXL runs to // ensure environment value (and hence pip hash) consistency. processBuilder.EnableTempDirectory(); AbsolutePath toolDir = m_msBuildExePath.GetParent(PathTable); processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}")); return(true); }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( ProjectWithPredictions project, ProcessBuilder processBuilder) { // Predicted output directories for all direct dependencies, plus the output directories for the given project itself var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders); // Add all predicted inputs that are recognized as true source files // This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means // we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case // of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up. foreach (AbsolutePath buildInput in project.PredictedInputFiles) { // If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file // In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory // dependency. if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder))) { continue; } // If any of the predicted inputs is under an untracked directory scope, don't add it as an input if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory))) { continue; } processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput)); } IEnumerable <ProjectWithPredictions> references; // The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in if (m_resolverSettings.EnableTransitiveProjectReferences == true) { // In this case all the transitive closure is automatically exposed to the project as direct references var transitiveReferences = new HashSet <ProjectWithPredictions>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); references = transitiveReferences; } else { // Only direct dependencies are declared. // Add all known explicit inputs from project references. But rule out // projects that have a known empty list of targets: those projects are not scheduled, so // there is nothing to consume from them. references = project.ProjectReferences.Where(projectReference => projectReference.PredictedTargetsToExecute.Targets.Count != 0); } var argumentsBuilder = processBuilder.ArgumentsBuilder; foreach (ProjectWithPredictions projectReference in references) { bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}"); } // Add all known output directories foreach (StaticDirectory output in projectOutputs.OutputDirectories) { processBuilder.AddInputDirectory(output.Root); } // If the dependency was built in isolation, this project needs to access the generated cache files if (projectOutputs.BuildsInIsolation) { var outputCache = projectOutputs.OutputCacheFile; processBuilder.AddInputFile(outputCache); // Instruct MSBuild to use the cache file from the associated dependency as an input. // Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode. using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty)) { argumentsBuilder.Add(PipDataAtom.FromString("/irc:")); argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache)); } } } }
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId) { SeparateExecutableFromCommands(node.Command, out string executable, out string args); if (!TryFindExecutablePath(executable, out AbsolutePath exePath)) { Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command); return(false); } FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath); processBuilder.Executable = prExeArtifact; processBuilder.AddInputFile(prExeArtifact); using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(args); processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } if (node.ResponseFile.HasValue) { using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(node.ResponseFile?.Content); PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping); // We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false) // because that information is already in node.Command var rspFileSpec = ResponseFileSpecification.Builder() .ExplicitData(responseFileData) .RequiresArgument(false) .ExplicitPath((AbsolutePath)node.ResponseFile?.Path) .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); } } // TODO: Maybe a better description. Add ninja description or change command for input/outputs processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]")); processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable; processBuilder.EnableTempDirectory(); // Working directory - the directory containing the ninja spec file // Ninja generators may express paths relative to this processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable)); // Untrack directories UntrackFilesAndDirectories(processBuilder); // Allow some surviving child process AddRequiredSurvivingChildren(processBuilder); // Environment variables SetEnvironmentVariables(processBuilder, node); return(true); }