/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
private void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
RushProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
var logFile = logDirectory.Combine(PathTable, "build.log");
// Execute the build command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.BuildCommand));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(">"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(logFile));
processBuilder.AddOutputFile(logFile, FileExistence.Required);
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
private static void AddLogArgument(PipDataBuilder pipDataBuilder, int loggerNumber, AbsolutePath logFile, string verbosity)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(I($"/flp{loggerNumber}:logfile=")));
pipDataBuilder.Add((PipDataAtom.FromAbsolutePath(logFile)));
pipDataBuilder.Add(PipDataAtom.FromString(I($";{verbosity}")));
}
}
private static void AddMsBuildProperty(PipDataBuilder pipDataBuilder, string key, string value)
{
// Make sure properties are always quoted, since MSBuild doesn't handle semicolon separated
// properties without quotes
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/p:"));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(key));
}
pipDataBuilder.Add(PipDataAtom.FromString("=\""));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(value));
}
pipDataBuilder.Add(PipDataAtom.FromString("\""));
}
}
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Predicted output directories for all direct dependencies, plus the output directories for the given project itself
var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders);
// Add all predicted inputs that are recognized as true source files
// This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means
// we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case
// of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up.
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
// If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file
// In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory
// dependency.
if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder)))
{
continue;
}
// If any of the predicted inputs is under an untracked directory scope, don't add it as an input
if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory)))
{
continue;
}
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => projectReference.PredictedTargetsToExecute.Targets.Count != 0);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
public FileArtifact WriteFile(AbsolutePath destination, string content, WriteFileEncoding?encoding = null) => WriteFile(destination, PipDataAtom.FromString(content), encoding);
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}
private bool TryAddMsBuildArguments(ProjectWithPredictions <AbsolutePath> project, Qualifier qualifier, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// The specified qualifier, unless overridden by a global property, is turned into a property as well
// This means that:
// 1) if the project is not being referenced with a specific property that matters to the qualifier, the requested qualifier is used
// 2) if a particular property (e.g. platform) is set when referencing the project, that is honored
foreach (StringId key in qualifier.Keys)
{
string keyAsString = key.ToString(m_context.StringTable);
if (!project.GlobalProperties.ContainsKey(keyAsString))
{
var success = qualifier.TryGetValue(m_context.StringTable, keyAsString, out string value);
Contract.Assert(success);
AddMsBuildProperty(pipDataBuilder, keyAsString, value);
}
}
// Configure binary logger if specified
if (m_resolverSettings?.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
failureDetail = string.Empty;
return(true);
}