/// <summary>
/// Computes a build graph by calling an external tool in a sandboxed process. The particular tool and arguments are provided by implementors.
/// </summary>
protected override async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync(
BuildParameters.IBuildParameters buildParameters)
{
// We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// Make sure the directories are there
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));
Possible <(JavaScriptGraph <TGraphConfiguration> graph, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> flattenedGraph)> maybeResult = await ComputeBuildGraphAsync(outputFile, buildParameters);
if (!maybeResult.Succeeded)
{
// A more specific error has been logged already
return(maybeResult.Failure);
}
if (m_resolverSettings.KeepProjectGraphFile != true)
{
DeleteGraphBuilderRelatedFiles(outputFile);
}
else
{
// Graph-related files are requested to be left on disk. Let's print a message with their location.
Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable));
}
return(maybeResult);
}
private Task <SandboxedProcessResult> RunRushGraphBuilderAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
// We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work.
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
string nodeExe = m_resolverSettings.NodeExeLocation.HasValue ?
m_resolverSettings.NodeExeLocation.Value.Path.ToString(m_context.PathTable) :
"node.exe";
string pathToRushJson = m_resolverSettings.Root.Combine(m_context.PathTable, "rush.json").ToString(m_context.PathTable);
// TODO: add qualifier support.
// The graph construction tool expects: <path-to-rush.json> <path-to-output-graph> [<debug|release>]
string toolArguments = $@"/C """"{nodeExe}"" ""{toolPath.ToString(m_context.PathTable)}"" ""{pathToRushJson}"" ""{outputFile.ToString(m_context.PathTable)}"" debug""";
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_configuration.Layout.SourceDirectory.ToString(m_context.PathTable),
description: "Rush graph builder",
buildParameters));
}
private Task <SandboxedProcessResult> RunJavaScriptGraphBuilderAsync(
string nodeExeLocation,
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters,
AbsolutePath toolLocation)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
var toolArguments = GetGraphConstructionToolArguments(outputFile, toolLocation, toolPath, nodeExeLocation);
Tracing.Logger.Log.ConstructingGraphScript(m_context.LoggingContext, toolArguments);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_resolverSettings.Root.ToString(m_context.PathTable),
description: $"{Name} graph builder",
buildParameters));
}
/// <inheritdoc/>
protected override async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync()
{
// Get the locations where the MsBuild assemblies should be searched
if (!TryRetrieveMsBuildSearchLocations(out IEnumerable <AbsolutePath> msBuildSearchLocations))
{
// Errors should have been logged
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
if (!TryRetrieveParsingEntryPoint(out IEnumerable <AbsolutePath> parsingEntryPoints))
{
// Errors should have been logged
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If we should run the dotnet core version of MSBuild, let's retrieve the locations where dotnet.exe
// should be found
IEnumerable <AbsolutePath> dotNetSearchLocations = null;
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
if (!TryRetrieveDotNetSearchLocations(out dotNetSearchLocations))
{
// Errors should have been logged
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
}
BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters();
return(await TryComputeBuildGraphAsync(msBuildSearchLocations, dotNetSearchLocations, parsingEntryPoints, buildParameters));
}
private void TrackFilesAndEnvironment(BuildParameters.IBuildParameters buildParameters, ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Register all build parameters passed to the graph construction process
// TODO: we actually need the build parameters *used* by the graph construction process, but for now this is a compromise to keep
// graph caching sound. We need to modify this when MsBuild static graph API starts providing used env vars.
foreach (string key in buildParameters.ToDictionary().Keys)
{
Engine.TryGetBuildParameter(key, MsBuildFrontEnd.Name, out _);
}
FrontEndUtilities.TrackToolFileAccesses(Engine, Context, MsBuildFrontEnd.Name, fileAccesses, frontEndFolder);
}
private async Task <Possible <RushGraphResult> > TryComputeBuildGraphAsync(BuildParameters.IBuildParameters buildParameters)
{
// We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// Make sure the directories are there
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));
Possible <RushGraph> maybeRushGraph = await ComputeBuildGraphAsync(outputFile, buildParameters);
if (!maybeRushGraph.Succeeded)
{
// A more specific error has been logged already
return(maybeRushGraph.Failure);
}
var rushGraph = maybeRushGraph.Result;
if (m_resolverSettings.KeepProjectGraphFile != true)
{
DeleteGraphBuilderRelatedFiles(outputFile);
}
else
{
// Graph-related files are requested to be left on disk. Let's print a message with their location.
Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable));
}
// The module contains all project files that are part of the graph
var projectFiles = new HashSet <AbsolutePath>();
foreach (RushProject project in rushGraph.Projects)
{
projectFiles.Add(project.ProjectPath(m_context.PathTable));
}
var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this);
var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences(
moduleDescriptor,
m_resolverSettings.Root,
m_resolverSettings.File,
projectFiles,
allowedModuleDependencies: null, // no module policies
cyclicalFriendModules: null); // no whitelist of cycles
return(new RushGraphResult(rushGraph, moduleDefinition));
}
private async Task <Possible <RushGraph> > ComputeBuildGraphAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
SandboxedProcessResult result = await RunRushGraphBuilderAsync(outputFile, buildParameters);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
// In case of a cancellation, the tool may have exited with a non-zero
// code, but that's expected
if (!m_context.CancellationToken.IsCancellationRequested)
{
// This should never happen! Report the standard error and exit gracefully
Tracing.Logger.Log.GraphConstructionInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new RushGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer(s_jsonSerializerSettings);
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var flattenedRushGraph = serializer.Deserialize <GenericRushGraph <GenericRushProject <string> > >(reader);
RushGraph graph = ResolveDependencies(flattenedRushGraph);
return(graph);
}
}
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync(
AbsolutePath responseFile,
IEnumerable <AbsolutePath> projectEntryPoints,
AbsolutePath outputFile,
IEnumerable <AbsolutePath> searchLocations,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable);
string pathToTool = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
string outputFileString = outputFile.ToString(m_context.PathTable);
string enlistmentRoot = m_resolverSettings.Root.ToString(m_context.PathTable);
IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>();
var requestedQualifiers = m_requestedQualifiers.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList();
var arguments = new MSBuildGraphBuilderArguments(
enlistmentRoot,
projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(),
outputFileString,
new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()),
searchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(),
entryPointTargets,
requestedQualifiers,
m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true);
var responseFilePath = responseFile.ToString(m_context.PathTable);
SerializeResponseFile(responseFilePath, arguments);
Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool);
// Just being defensive, make sure there is not an old output file lingering around
File.Delete(outputFileString);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
pathToTool,
buildStorageDirectory: outputDirectory,
fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile),
arguments: I($"\"{responseFilePath}\""),
workingDirectory: outputDirectory,
description: "MsBuild graph builder",
buildParameters,
beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString)));
}
private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphIfNeededAsync()
{
if (m_projectGraph == null)
{
// Get the locations where the MsBuild assemblies should be searched
if (!TryRetrieveMsBuildSearchLocations(out IEnumerable <AbsolutePath> searchLocations))
{
// Errors should have been logged
return(new MsBuildGraphConstructionFailure(m_resolverSettings, Context.PathTable));
}
if (!TryRetrieveParsingEntryPoint(out IEnumerable <AbsolutePath> parsingEntryPoints))
{
// Errors should have been logged
return(new MsBuildGraphConstructionFailure(m_resolverSettings, Context.PathTable));
}
BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters();
m_projectGraph = await TryComputeBuildGraphAsync(searchLocations, parsingEntryPoints, buildParameters);
}
return(m_projectGraph.Value);
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(IYarnResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalYarnLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
if (resolverSettings.YarnLocation.HasValue)
{
finalYarnLocation = resolverSettings.YarnLocation.Value.Path;
failure = string.Empty;
return(true);
}
finalYarnLocation = AbsolutePath.Invalid;
// If the location was not provided, let's try to see if Yarn is under %PATH%
string paths = buildParameters["PATH"];
AbsolutePath foundPath = AbsolutePath.Invalid;
foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
var nonEscapedPath = path.Trim('"');
// Sometimes PATH is not well-formed, so make sure we can actually recognize an absolute path there
if (AbsolutePath.TryCreate(m_context.PathTable, nonEscapedPath, out var absolutePath))
{
AbsolutePath pathToYarn = absolutePath.Combine(m_context.PathTable, "yarn");
if (m_host.Engine.FileExists(pathToYarn))
{
foundPath = pathToYarn;
break;
}
// In some windows installations, only yarn.cmd exists
pathToYarn = absolutePath.Combine(m_context.PathTable, "yarn.cmd");
if (m_host.Engine.FileExists(pathToYarn))
{
foundPath = pathToYarn;
break;
}
}
}
if (!foundPath.IsValid)
{
failure = "A location for 'yarn' is not explicitly specified. However, 'yarn' doesn't seem to be part of PATH. You can either specify the location explicitly using 'yarnLocation' field in " +
$"the Yarn resolver configuration, or make sure 'yarn' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
failure = string.Empty;
finalYarnLocation = foundPath;
// Just verbose log this
Tracing.Logger.Log.UsingYarnAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalYarnLocation.ToString(m_context.PathTable));
return(true);
}
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync(
AbsolutePath responseFile,
IEnumerable <AbsolutePath> projectEntryPoints,
AbsolutePath outputFile,
IEnumerable <AbsolutePath> msBuildSearchLocations,
AbsolutePath dotnetExeLocation,
BuildParameters.IBuildParameters buildParameters)
{
Contract.Assert(!m_resolverSettings.ShouldRunDotNetCoreMSBuild() || dotnetExeLocation.IsValid);
AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
string outputFileString = outputFile.ToString(m_context.PathTable);
IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>();
var requestedQualifiers = m_host.QualifiersToEvaluate.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList();
var arguments = new MSBuildGraphBuilderArguments(
projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(),
outputFileString,
new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()),
msBuildSearchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(),
entryPointTargets,
requestedQualifiers,
m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true,
m_resolverSettings.ShouldRunDotNetCoreMSBuild());
var responseFilePath = responseFile.ToString(m_context.PathTable);
SerializeResponseFile(responseFilePath, arguments);
string graphConstructionToolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable);
string pathToTool;
string toolArguments;
// if we should call the dotnet core version of MSBuild, we need to actually call dotnet.exe and pass the tool itself as its first argument
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
pathToTool = dotnetExeLocation.ToString(m_context.PathTable);
toolArguments = I($"\"{graphConstructionToolPath}\" \"{responseFilePath}\"");
}
else
{
pathToTool = graphConstructionToolPath;
toolArguments = I($"\"{responseFilePath}\"");
}
Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool);
// Just being defensive, make sure there is not an old output file lingering around
File.Delete(outputFileString);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
pathToTool,
buildStorageDirectory: outputDirectory,
fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile),
arguments: toolArguments,
workingDirectory: outputDirectory,
description: "MsBuild graph builder",
buildParameters,
beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString)));
}
private async Task <Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > > ComputeBuildGraphAsync(
AbsolutePath responseFile,
IEnumerable <AbsolutePath> projectEntryPoints,
AbsolutePath outputFile,
IEnumerable <AbsolutePath> msBuidSearchLocations,
IEnumerable <AbsolutePath> dotnetSearchLocations,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath dotnetExeLocation = AbsolutePath.Invalid;
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
if (!TryFindDotNetExe(dotnetSearchLocations, out dotnetExeLocation, out string failure))
{
return(ProjectGraphWithPredictionsResult <AbsolutePath> .CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation(failure),
CollectionUtilities.EmptyDictionary <string, AbsolutePath>(), AbsolutePath.Invalid));
}
}
SandboxedProcessResult result = await RunMsBuildGraphBuilderAsync(responseFile, projectEntryPoints, outputFile, msBuidSearchLocations, dotnetExeLocation, buildParameters);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
// In case of a cancellation, the tool may have exited with a non-zero
// code, but that's expected
if (!m_context.CancellationToken.IsCancellationRequested)
{
// This should never happen! Report the standard error and exit gracefully
Tracing.Logger.Log.GraphConstructionInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer();
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var projectGraphWithPredictionsResult = serializer.Deserialize <ProjectGraphWithPredictionsResult <AbsolutePath> >(reader);
// A successfully constructed graph should always have a valid path to MsBuild
Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.PathToMsBuild.IsValid);
// A successfully constructed graph should always have at least one project node
Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.Result.ProjectNodes.Length > 0);
// A failed construction should always have a failure set
Contract.Assert(projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.Failure != null);
// Let's log the paths to the used MsBuild assemblies, just for debugging purposes
Tracing.Logger.Log.GraphConstructionToolCompleted(
m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable),
string.Join(",\n", projectGraphWithPredictionsResult.MsBuildAssemblyPaths.Select(kvp => I($"[{kvp.Key}]:{kvp.Value.ToString(m_context.PathTable)}"))),
projectGraphWithPredictionsResult.PathToMsBuild.ToString(m_context.PathTable));
return(m_resolverSettings.ShouldRunDotNetCoreMSBuild() ? projectGraphWithPredictionsResult.WithPathToDotNetExe(dotnetExeLocation) : projectGraphWithPredictionsResult);
}
}
private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync(IEnumerable <AbsolutePath> msBuildSearchLocations, IEnumerable <AbsolutePath> dotnetSearchLocations, IEnumerable <AbsolutePath> parsingEntryPoints, BuildParameters.IBuildParameters buildParameters)
{
// We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(MsBuildFrontEnd.Name);
AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// We create a unique response file that will contain the tool arguments
AbsolutePath responseFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// Make sure the directories are there
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));
Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, msBuildSearchLocations, dotnetSearchLocations, buildParameters);
if (!maybeProjectGraphResult.Succeeded)
{
// A more specific error has been logged already
return(maybeProjectGraphResult.Failure);
}
var projectGraphResult = maybeProjectGraphResult.Result;
if (m_resolverSettings.KeepProjectGraphFile != true)
{
DeleteGraphBuilderRelatedFiles(outputFile, responseFile);
}
else
{
// Graph-related files are requested to be left on disk. Let's print a message with their location.
Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable), responseFile.ToString(m_context.PathTable));
}
if (!projectGraphResult.Succeeded)
{
var failure = projectGraphResult.Failure;
Tracing.Logger.Log.ProjectGraphConstructionError(m_context.LoggingContext, failure.HasLocation ? failure.Location : m_resolverSettings.Location(m_context.PathTable), failure.Message);
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
ProjectGraphWithPredictions <AbsolutePath> projectGraph = projectGraphResult.Result;
// The module contains all project files that are part of the graph
var projectFiles = new HashSet <AbsolutePath>();
foreach (ProjectWithPredictions <AbsolutePath> node in projectGraph.ProjectNodes)
{
projectFiles.Add(node.FullPath);
}
var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this);
var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences(
moduleDescriptor,
m_resolverSettings.RootTraversal,
m_resolverSettings.File,
projectFiles,
allowedModuleDependencies: null, // no module policies
cyclicalFriendModules: null); // no whitelist of cycles
return(new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuild, projectGraphResult.PathToDotNetExe));
}
/// <inheritdoc/>
protected override Task <Possible <RushGraphResult> > TryComputeBuildGraphAsync()
{
BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters();
return(TryComputeBuildGraphAsync(buildParameters));
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(RushResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalRushLibBaseLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
if (resolverSettings.RushLibBaseLocation.HasValue)
{
finalRushLibBaseLocation = resolverSettings.RushLibBaseLocation.Value.Path;
failure = string.Empty;
return(true);
}
finalRushLibBaseLocation = AbsolutePath.Invalid;
// If the location was not provided, let's try to see if Rush is installed, since rush-lib comes as part of it
// Look in %PATH% (as exposed in build parameters) for rush
string paths = buildParameters["PATH"];
AbsolutePath foundPath = AbsolutePath.Invalid;
foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
var nonEscapedPath = path.Trim('"');
// Sometimes PATH is not well-formed, so make sure we can actually recognize an absolute path there
if (AbsolutePath.TryCreate(m_context.PathTable, nonEscapedPath, out var absolutePath))
{
if (m_host.Engine.FileExists(absolutePath.Combine(m_context.PathTable, "rush")))
{
foundPath = absolutePath;
break;
}
}
}
if (!foundPath.IsValid)
{
failure = "A location for 'rush-lib' is not explicitly specified, so trying to find a Rush installation to use instead. " +
"However, 'rush' doesn't seem to be part of PATH. You can either specify the location explicitly using 'rushLibBaseLocation' field in " +
$"the Rush resolver configuration, or make sure 'rush' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
// We found where Rush is located. So rush-lib is a known dependency of it, so should be nested within Rush module
// Observe that even if that's not the case the final validation will occur under the rush graph builder tool, when
// the module is tried to be loaded
failure = string.Empty;
finalRushLibBaseLocation = foundPath.Combine(m_context.PathTable,
RelativePath.Create(m_context.StringTable, "node_modules/@microsoft/rush/node_modules"));
// Just verbose log this
Tracing.Logger.Log.UsingRushLibBaseAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalRushLibBaseLocation.ToString(m_context.PathTable));
return(true);
}
/// <summary>
/// Runs the a tool in a sandboxed process and returns the result.
/// These optional callback Actions can be provided:
/// beforeLaunch is invoked right before the process is launched
/// onResult is invoked after getting a successful result
/// </summary>>
public static async Task <SandboxedProcessResult> RunSandboxedToolAsync(FrontEndContext context,
string pathToTool,
string buildStorageDirectory,
FileAccessManifest fileAccessManifest,
string arguments,
string workingDirectory,
string description,
BuildParameters.IBuildParameters buildParameters,
Action beforeLaunch = null, // Invoked right before the process starts
Action onResult = null // Action to be taken after getting a successful result
)
{
var info =
new SandboxedProcessInfo(
context.PathTable,
new ToolBuildStorage(buildStorageDirectory),
pathToTool,
fileAccessManifest,
disableConHostSharing: true,
ContainerConfiguration.DisabledIsolation,
loggingContext: context.LoggingContext)
{
Arguments = arguments,
WorkingDirectory = workingDirectory,
PipSemiStableHash = 0,
PipDescription = description,
EnvironmentVariables = buildParameters
};
var process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing : true);
var registration = context.CancellationToken.Register(
() =>
{
try
{
process.KillAsync().GetAwaiter().GetResult();
}
catch (TaskCanceledException)
{
// If the process has already terminated or doesn't exist, an TaskCanceledException is raised.
// In either case, we swallow the exception, cancellation is already requested by the user
}
});
beforeLaunch?.Invoke();
var result = process.GetResultAsync().ContinueWith(
r =>
{
// Dispose the registration for the cancellation token once the process is done
registration.Dispose();
//
onResult?.Invoke();
return(r.GetAwaiter().GetResult());
});
return(await result);
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(ILageResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath npmLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
if (resolverSettings.NpmLocation.HasValue)
{
npmLocation = resolverSettings.NpmLocation.Value.Path;
failure = string.Empty;
return(true);
}
// If the location was not provided, let's try to see if NPM is under %PATH%
string paths = buildParameters["PATH"];
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "npm", "npm.cmd" }, out npmLocation))
{
failure = "A location for 'npm' is not explicitly specified. However, 'npm' doesn't seem to be part of PATH. You can either specify the location explicitly using 'npmLocation' field in " +
$"the Lage resolver configuration, or make sure 'npm' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
failure = string.Empty;
// Just verbose log this
Tracing.Logger.Log.UsingNpmAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), npmLocation.ToString(m_context.PathTable));
return(true);
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(IYarnResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalYarnLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
string paths;
if (resolverSettings.YarnLocation != null)
{
var value = resolverSettings.YarnLocation.GetValue();
if (value is FileArtifact file)
{
finalYarnLocation = file;
failure = string.Empty;
return(true);
}
else
{
var pathCollection = ((IReadOnlyList <DirectoryArtifact>)value).Select(dir => dir.Path);
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation))
{
failure = $"'yarn' cannot be found under any of the provided paths '{string.Join(Path.PathSeparator.ToString(), pathCollection.Select(path => path.ToString(m_context.PathTable)))}'.";
return(false);
}
failure = string.Empty;
return(true);
}
}
// If the location was not provided, let's try to see if Yarn is under %PATH%
paths = buildParameters["PATH"];
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation))
{
failure = "A location for 'yarn' is not explicitly specified. However, 'yarn' doesn't seem to be part of PATH. You can either specify the location explicitly using 'yarnLocation' field in " +
$"the Yarn resolver configuration, or make sure 'yarn' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
failure = string.Empty;
// Just verbose log this
Tracing.Logger.Log.UsingYarnAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalYarnLocation.ToString(m_context.PathTable));
return(true);
}
/// <summary>
/// Compute the build graph by reading a user-specified package-to-package graph
/// </summary>
protected override Task <Possible <(JavaScriptGraph <YarnConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration>)> > ComputeBuildGraphAsync(BuildParameters.IBuildParameters buildParameters)
{
if (m_resolverSettings.CustomProjectGraph == null)
{
Tracing.Logger.Log.ErrorReadingCustomProjectGraph(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
"The custom project graph is undefined.");
var failure = new Possible <(JavaScriptGraph <YarnConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration>)>(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
return(Task.FromResult(failure));
}
// The graph may come from a file or from a DScript literal following the Yarn schema
Possible <GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration> > maybeGraph;
if (m_resolverSettings.CustomProjectGraph.GetValue() is AbsolutePath graphFile)
{
Contract.Assert(graphFile.IsValid);
maybeGraph = ReadGraphFromFile(graphFile);
}
else
{
var graphLiteral = m_resolverSettings.CustomProjectGraph.GetValue() as IReadOnlyDictionary <string, IJavaScriptCustomProjectGraphNode>;
maybeGraph = BuildGraphFromLiteral(graphLiteral);
}
// The graph is always resolved with execution semantics, since Yarn doesn't provide execution semantics
var maybeResult = maybeGraph
.Then(graph => ResolveGraphWithExecutionSemantics(graph))
.Then(resolvedGraph => (resolvedGraph, maybeGraph.Result));
// There is actually no graph file to 'keep' in this case, but in order to honor
// this option, let's serialize to a file the graph we just constructed
if (m_resolverSettings.KeepProjectGraphFile == true && maybeResult.Succeeded)
{
SerializeComputedGraph(maybeResult.Result.Result);
}
return(Task.FromResult(maybeResult));
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(ILageResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalLageLocation, out string failure)
{
// We use an npm call to invoke Lage
finalLageLocation = AbsolutePath.Invalid;
failure = null;
return(true);
}
private async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
// Determine the base location to use for finding the graph construction tool
if (!TryFindGraphBuilderToolLocation(
m_resolverSettings,
buildParameters,
out AbsolutePath foundLocation,
out string failure))
{
Tracing.Logger.Log.CannotFindGraphBuilderTool(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
failure);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
SandboxedProcessResult result = await RunJavaScriptGraphBuilderAsync(outputFile, buildParameters, foundLocation);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
Tracing.Logger.Log.ProjectGraphConstructionError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings);
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var flattenedJavaScriptGraph = serializer.Deserialize <GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> >(reader);
// If a custom script command callback is specified, give it a chance to alter the script commands of
// each package
if (m_resolverSettings.CustomScripts != null)
{
var projectsWithCustomScripts = new List <DeserializedJavaScriptProject>(flattenedJavaScriptGraph.Projects.Count);
foreach (var project in flattenedJavaScriptGraph.Projects)
{
m_resolverSettings.Root.TryGetRelative(m_context.PathTable, project.ProjectFolder, out var relativeFolder);
var maybeCustomScripts = ResolveCustomScripts(project.Name, relativeFolder);
if (!maybeCustomScripts.Succeeded)
{
return(maybeCustomScripts.Failure);
}
var customScripts = maybeCustomScripts.Result;
// A null customScript means the callback did not provide any customization
projectsWithCustomScripts.Add(customScripts == null ? project : project.WithCustomScripts(customScripts));
}
flattenedJavaScriptGraph = new GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>(
projectsWithCustomScripts, flattenedJavaScriptGraph.Configuration);
}
Possible <JavaScriptGraph <TGraphConfiguration> > graph = ApplyBxlExecutionSemantics() ? ResolveGraphWithExecutionSemantics(flattenedJavaScriptGraph) : ResolveGraphWithoutExecutionSemantics(flattenedJavaScriptGraph);
return(graph.Then(graph => new Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)>((graph, flattenedJavaScriptGraph))));
}
}
/// <summary>
/// Tries to find the JavaScript-based tool that will be pass as a parameter to the Bxl graph construction tool
/// </summary>
/// <remarks>
/// For example, for Rush this is the location of rush-lib. For Yarn, the location of yarn
/// </remarks>
protected abstract bool TryFindGraphBuilderToolLocation(
TResolverSettings resolverSettings,
BuildParameters.IBuildParameters buildParameters,
out AbsolutePath location,
out string failure);
private async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
// Determine the base location to use for finding the graph construction tool
if (!TryFindGraphBuilderToolLocation(
m_resolverSettings,
buildParameters,
out AbsolutePath foundLocation,
out string failure))
{
Tracing.Logger.Log.CannotFindGraphBuilderTool(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
failure);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
string nodeExeLocation;
if (m_resolverSettings.NodeExeLocation != null)
{
var specifiedNodeExe = m_resolverSettings.NodeExeLocation.GetValue();
AbsolutePath nodeExeLocationPath;
if (specifiedNodeExe is FileArtifact fileArtifact)
{
nodeExeLocationPath = fileArtifact.Path;
}
else
{
var pathCollection = ((IReadOnlyList <DirectoryArtifact>)specifiedNodeExe).Select(dir => dir.Path);
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "node", "node.exe" }, out nodeExeLocationPath))
{
failure = $"'node' cannot be found under any of the provided paths '{string.Join(";", pathCollection.Select(path => path.ToString(m_context.PathTable)))}'.";
Tracing.Logger.Log.CannotFindGraphBuilderTool(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
failure);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
}
nodeExeLocation = nodeExeLocationPath.ToString(m_context.PathTable);
// Most graph construction tools (yarn, rush, etc.) rely on node.exe being on the PATH. Make sure
// that's the case by appending the PATH exposed to the graph construction process with the location of the
// specified node.exe. By prepending PATH with it, we also make sure yarn/rush will be using the same version
// of node the user specified.
string pathWithNode = buildParameters.ContainsKey("PATH") ? buildParameters["PATH"] : string.Empty;
var nodeDirectory = nodeExeLocationPath.GetParent(m_context.PathTable);
if (nodeDirectory.IsValid)
{
pathWithNode = nodeDirectory.ToString(m_context.PathTable) + Path.PathSeparator + pathWithNode;
}
buildParameters = buildParameters.Override(new[] { new KeyValuePair <string, string>("PATH", pathWithNode) });
}
else
{
// We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work.
nodeExeLocation = "node.exe";
}
SandboxedProcessResult result = await RunJavaScriptGraphBuilderAsync(nodeExeLocation, outputFile, buildParameters, foundLocation);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
Tracing.Logger.Log.ProjectGraphConstructionError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings);
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var flattenedJavaScriptGraph = serializer.Deserialize <GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> >(reader);
// If a custom script command callback is specified, give it a chance to alter the script commands of
// each package
if (m_resolverSettings.CustomScripts != null)
{
var projectsWithCustomScripts = new List <DeserializedJavaScriptProject>(flattenedJavaScriptGraph.Projects.Count);
foreach (var project in flattenedJavaScriptGraph.Projects)
{
m_resolverSettings.Root.TryGetRelative(m_context.PathTable, project.ProjectFolder, out var relativeFolder);
var maybeCustomScripts = ResolveCustomScripts(project.Name, relativeFolder);
if (!maybeCustomScripts.Succeeded)
{
return(maybeCustomScripts.Failure);
}
var customScripts = maybeCustomScripts.Result;
// A null customScript means the callback did not provide any customization
projectsWithCustomScripts.Add(customScripts == null ? project : project.WithCustomScripts(customScripts));
}
flattenedJavaScriptGraph = new GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>(
projectsWithCustomScripts, flattenedJavaScriptGraph.Configuration);
}
Possible <JavaScriptGraph <TGraphConfiguration> > graph = ApplyBxlExecutionSemantics() ? ResolveGraphWithExecutionSemantics(flattenedJavaScriptGraph) : ResolveGraphWithoutExecutionSemantics(flattenedJavaScriptGraph);
return(graph.Then(graph => new Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)>((graph, flattenedJavaScriptGraph))));
}
}
