/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
/// <summary>
/// Augments the processBuilder with the OS dependencies
/// </summary>
public bool ProcessDefaults(ProcessBuilder processBuilder)
{
if ((processBuilder.Options & Process.Options.DependsOnCurrentOs) != 0)
{
var defaultSourceSealDirs = m_lazySourceSealDirectories.Value;
if (!defaultSourceSealDirs.IsValid)
{
return(false);
}
foreach (var inputDirectory in defaultSourceSealDirs.Directories)
{
processBuilder.AddInputDirectory(inputDirectory);
}
foreach (var untrackedFile in m_untrackedFiles)
{
processBuilder.AddUntrackedFile(untrackedFile);
}
foreach (var untrackedDirectory in m_untrackedDirectories)
{
processBuilder.AddUntrackedDirectoryScope(untrackedDirectory);
}
}
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
RushProject project,
ProcessBuilder processBuilder)
{
var argumentsBuilder = processBuilder.ArgumentsBuilder;
IEnumerable <RushProject> references;
// In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for
// JavaScript projects.
var transitiveReferences = new HashSet <RushProject>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
foreach (RushProject projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
// Add all known output files
foreach (FileArtifact output in processOutputs.GetOutputFiles())
{
processBuilder.AddInputFile(output);
}
}
}
/// <summary>
/// Augments the processBuilder with the OS dependencies
/// </summary>
public void ProcessDefaults(ProcessBuilder processBuilder)
{
if ((processBuilder.Options & Process.Options.DependsOnCurrentOs) != 0)
{
foreach (var inputFile in m_inputFiles)
{
processBuilder.AddInputFile(inputFile);
}
foreach (var inputDirectory in m_inputDirectories)
{
processBuilder.AddInputDirectory(inputDirectory);
}
foreach (var untrackedFile in m_untrackedFiles)
{
processBuilder.AddUntrackedFile(untrackedFile);
}
foreach (var untrackedDirectory in m_untrackedDirectories)
{
processBuilder.AddUntrackedDirectoryScope(untrackedDirectory);
}
}
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
RushProject project,
ProcessBuilder processBuilder)
{
// Add all explicitly declared source files
foreach (AbsolutePath sourceFile in project.SourceFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(sourceFile));
}
// Add package.json, which should always be present at the root of the project
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.PackageJsonFile(PathTable)));
// If dependencies should be tracked via the project-level shrinkwrap-deps file, then force an input
// dependency on it
if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(project.ShrinkwrapDepsFile(PathTable)));
}
// In this case all the transitive closure is automatically exposed to the project as direct references. This is standard for
// JavaScript projects.
var transitiveReferences = new HashSet <RushProject>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
IEnumerable <RushProject> references = transitiveReferences;
foreach (RushProject projectReference in references)
{
// If the project is referencing something that was not scheduled, just skip it
if (!projectReference.CanBeScheduled())
{
// We have already logged this case as an informational when building the project graph
continue;
}
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out var processOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.ProjectFolder.ToString(PathTable)} missing, dependency of {project.ProjectFolder.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
// Add all known output files, but exclude logs
foreach (FileArtifact output in processOutputs.GetOutputFiles()
.Where(fa => !fa.Path.IsWithin(m_context.PathTable, LogDirectoryBase(m_frontEndHost.Configuration, m_context.PathTable))))
{
processBuilder.AddInputFile(output);
}
}
}
/// <summary>
/// Adds input directory.
/// </summary>
public ArgumentsBuilder AddInputDirectoryOption(string optionName, DirectoryArtifact inputDirectory)
{
Contract.Requires(!string.IsNullOrEmpty(optionName));
Contract.Requires(inputDirectory.IsValid);
Contract.Assert(!m_finished);
m_dataBuilder.AddPathOption(optionName, inputDirectory.Path);
m_processBuilder.AddInputDirectory(inputDirectory);
return(this);
}
public void ExecutionRecordsReportedFileAccesses()
{
FileArtifact sourceFile = CreateSourceFile();
SealDirectory sourceDirectory = CreateAndScheduleSealDirectory(sourceFile.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories);
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.AddInputDirectory(sourceDirectory.Directory);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
File.WriteAllText(ArtifactToString(sourceFile), Guid.NewGuid().ToString());
RunScheduler().AssertCacheMiss(process.Process.PipId);
}
/// <summary>
/// Augments the processBuilder with the OS dependencies.
/// </summary>
/// <param name="processBuilder">builder to use</param>
/// <param name="untrackInsteadSourceSeal">when true, directories that are meant to be source sealed are untracked instead</param>
public bool ProcessDefaults(ProcessBuilder processBuilder, bool untrackInsteadSourceSeal = false)
{
if (processBuilder.Options.HasFlag(Process.Options.DependsOnCurrentOs))
{
// process source seal directories: either source seal them or untrack them, depending on 'untrackInsteadSourceSeal'
if (untrackInsteadSourceSeal)
{
foreach (var sourceSealDirPath in m_sourceSealDirectoryPaths)
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(sourceSealDirPath));
}
}
else
{
var defaultSourceSealDirs = m_lazySourceSealDirectories.Value;
if (!defaultSourceSealDirs.IsValid)
{
return(false);
}
foreach (var inputDirectory in defaultSourceSealDirs.Directories)
{
processBuilder.AddInputDirectory(inputDirectory);
}
}
// add untracked files
foreach (var untrackedFile in m_untrackedFiles)
{
processBuilder.AddUntrackedFile(untrackedFile);
}
// add untracked directories
foreach (var untrackedDirectory in m_untrackedDirectories)
{
processBuilder.AddUntrackedDirectoryScope(untrackedDirectory);
}
}
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out ProcessOutputs processOutputs);
Contract.Assert(outputsPresent, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
// Add all known opaque directories
foreach (StaticDirectory output in processOutputs.GetOutputDirectories())
{
processBuilder.AddInputDirectory(output.Root);
}
}
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Predicted output directories for all direct dependencies, plus the output directories for the given project itself
var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders);
// Add all predicted inputs that are recognized as true source files
// This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means
// we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case
// of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up.
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
// If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file
// In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory
// dependency.
if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder)))
{
continue;
}
// If any of the predicted inputs is under an untracked directory scope, don't add it as an input
if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory)))
{
continue;
}
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => projectReference.PredictedTargetsToExecute.Targets.Count != 0);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
