/// <summary> /// Computes a build graph by calling an external tool in a sandboxed process. The particular tool and arguments are provided by implementors. /// </summary> protected override async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync( BuildParameters.IBuildParameters buildParameters) { // We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name); AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString()); // Make sure the directories are there FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); Possible <(JavaScriptGraph <TGraphConfiguration> graph, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> flattenedGraph)> maybeResult = await ComputeBuildGraphAsync(outputFile, buildParameters); if (!maybeResult.Succeeded) { // A more specific error has been logged already return(maybeResult.Failure); } if (m_resolverSettings.KeepProjectGraphFile != true) { DeleteGraphBuilderRelatedFiles(outputFile); } else { // Graph-related files are requested to be left on disk. Let's print a message with their location. Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable)); } return(maybeResult); }
private Task <SandboxedProcessResult> RunRushGraphBuilderAsync( AbsolutePath outputFile, BuildParameters.IBuildParameters buildParameters) { AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool); string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable); // We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work. var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC"))); string nodeExe = m_resolverSettings.NodeExeLocation.HasValue ? m_resolverSettings.NodeExeLocation.Value.Path.ToString(m_context.PathTable) : "node.exe"; string pathToRushJson = m_resolverSettings.Root.Combine(m_context.PathTable, "rush.json").ToString(m_context.PathTable); // TODO: add qualifier support. // The graph construction tool expects: <path-to-rush.json> <path-to-output-graph> [<debug|release>] string toolArguments = $@"/C """"{nodeExe}"" ""{toolPath.ToString(m_context.PathTable)}"" ""{pathToRushJson}"" ""{outputFile.ToString(m_context.PathTable)}"" debug"""; return(FrontEndUtilities.RunSandboxedToolAsync( m_context, cmdExeArtifact.Path.ToString(m_context.PathTable), buildStorageDirectory: outputDirectory, fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)), arguments: toolArguments, workingDirectory: m_configuration.Layout.SourceDirectory.ToString(m_context.PathTable), description: "Rush graph builder", buildParameters)); }
private Task <SandboxedProcessResult> RunJavaScriptGraphBuilderAsync( string nodeExeLocation, AbsolutePath outputFile, BuildParameters.IBuildParameters buildParameters, AbsolutePath toolLocation) { AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool); string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable); var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC"))); var toolArguments = GetGraphConstructionToolArguments(outputFile, toolLocation, toolPath, nodeExeLocation); Tracing.Logger.Log.ConstructingGraphScript(m_context.LoggingContext, toolArguments); return(FrontEndUtilities.RunSandboxedToolAsync( m_context, cmdExeArtifact.Path.ToString(m_context.PathTable), buildStorageDirectory: outputDirectory, fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)), arguments: toolArguments, workingDirectory: m_resolverSettings.Root.ToString(m_context.PathTable), description: $"{Name} graph builder", buildParameters)); }
/// <inheritdoc/> protected override async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync() { // Get the locations where the MsBuild assemblies should be searched if (!TryRetrieveMsBuildSearchLocations(out IEnumerable <AbsolutePath> msBuildSearchLocations)) { // Errors should have been logged return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } if (!TryRetrieveParsingEntryPoint(out IEnumerable <AbsolutePath> parsingEntryPoints)) { // Errors should have been logged return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } // If we should run the dotnet core version of MSBuild, let's retrieve the locations where dotnet.exe // should be found IEnumerable <AbsolutePath> dotNetSearchLocations = null; if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) { if (!TryRetrieveDotNetSearchLocations(out dotNetSearchLocations)) { // Errors should have been logged return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } } BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters(); return(await TryComputeBuildGraphAsync(msBuildSearchLocations, dotNetSearchLocations, parsingEntryPoints, buildParameters)); }
private void TrackFilesAndEnvironment(BuildParameters.IBuildParameters buildParameters, ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder) { // Register all build parameters passed to the graph construction process // TODO: we actually need the build parameters *used* by the graph construction process, but for now this is a compromise to keep // graph caching sound. We need to modify this when MsBuild static graph API starts providing used env vars. foreach (string key in buildParameters.ToDictionary().Keys) { Engine.TryGetBuildParameter(key, MsBuildFrontEnd.Name, out _); } FrontEndUtilities.TrackToolFileAccesses(Engine, Context, MsBuildFrontEnd.Name, fileAccesses, frontEndFolder); }
private async Task <Possible <RushGraphResult> > TryComputeBuildGraphAsync(BuildParameters.IBuildParameters buildParameters) { // We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name); AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString()); // Make sure the directories are there FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); Possible <RushGraph> maybeRushGraph = await ComputeBuildGraphAsync(outputFile, buildParameters); if (!maybeRushGraph.Succeeded) { // A more specific error has been logged already return(maybeRushGraph.Failure); } var rushGraph = maybeRushGraph.Result; if (m_resolverSettings.KeepProjectGraphFile != true) { DeleteGraphBuilderRelatedFiles(outputFile); } else { // Graph-related files are requested to be left on disk. Let's print a message with their location. Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable)); } // The module contains all project files that are part of the graph var projectFiles = new HashSet <AbsolutePath>(); foreach (RushProject project in rushGraph.Projects) { projectFiles.Add(project.ProjectPath(m_context.PathTable)); } var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this); var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences( moduleDescriptor, m_resolverSettings.Root, m_resolverSettings.File, projectFiles, allowedModuleDependencies: null, // no module policies cyclicalFriendModules: null); // no whitelist of cycles return(new RushGraphResult(rushGraph, moduleDefinition)); }
private async Task <Possible <RushGraph> > ComputeBuildGraphAsync( AbsolutePath outputFile, BuildParameters.IBuildParameters buildParameters) { SandboxedProcessResult result = await RunRushGraphBuilderAsync(outputFile, buildParameters); string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult(); if (result.ExitCode != 0) { // In case of a cancellation, the tool may have exited with a non-zero // code, but that's expected if (!m_context.CancellationToken.IsCancellationRequested) { // This should never happen! Report the standard error and exit gracefully Tracing.Logger.Log.GraphConstructionInternalError( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } return(new RushGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } // If the tool exited gracefully, but standard error is not empty, that // is interpreted as a warning. We propagate that to the BuildXL log if (!string.IsNullOrEmpty(standardError)) { Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable)); JsonSerializer serializer = ConstructProjectGraphSerializer(s_jsonSerializerSettings); using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable))) using (var reader = new JsonTextReader(sr)) { var flattenedRushGraph = serializer.Deserialize <GenericRushGraph <GenericRushProject <string> > >(reader); RushGraph graph = ResolveDependencies(flattenedRushGraph); return(graph); } }
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync( AbsolutePath responseFile, IEnumerable <AbsolutePath> projectEntryPoints, AbsolutePath outputFile, IEnumerable <AbsolutePath> searchLocations, BuildParameters.IBuildParameters buildParameters) { AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable); string pathToTool = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable); string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable); string outputFileString = outputFile.ToString(m_context.PathTable); string enlistmentRoot = m_resolverSettings.Root.ToString(m_context.PathTable); IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>(); var requestedQualifiers = m_requestedQualifiers.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList(); var arguments = new MSBuildGraphBuilderArguments( enlistmentRoot, projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(), outputFileString, new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()), searchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(), entryPointTargets, requestedQualifiers, m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true); var responseFilePath = responseFile.ToString(m_context.PathTable); SerializeResponseFile(responseFilePath, arguments); Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool); // Just being defensive, make sure there is not an old output file lingering around File.Delete(outputFileString); return(FrontEndUtilities.RunSandboxedToolAsync( m_context, pathToTool, buildStorageDirectory: outputDirectory, fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile), arguments: I($"\"{responseFilePath}\""), workingDirectory: outputDirectory, description: "MsBuild graph builder", buildParameters, beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString))); }
private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphIfNeededAsync() { if (m_projectGraph == null) { // Get the locations where the MsBuild assemblies should be searched if (!TryRetrieveMsBuildSearchLocations(out IEnumerable <AbsolutePath> searchLocations)) { // Errors should have been logged return(new MsBuildGraphConstructionFailure(m_resolverSettings, Context.PathTable)); } if (!TryRetrieveParsingEntryPoint(out IEnumerable <AbsolutePath> parsingEntryPoints)) { // Errors should have been logged return(new MsBuildGraphConstructionFailure(m_resolverSettings, Context.PathTable)); } BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters(); m_projectGraph = await TryComputeBuildGraphAsync(searchLocations, parsingEntryPoints, buildParameters); } return(m_projectGraph.Value); }
/// <inheritdoc/> protected override bool TryFindGraphBuilderToolLocation(IYarnResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalYarnLocation, out string failure) { // If the base location was provided at configuration time, we honor it as is if (resolverSettings.YarnLocation.HasValue) { finalYarnLocation = resolverSettings.YarnLocation.Value.Path; failure = string.Empty; return(true); } finalYarnLocation = AbsolutePath.Invalid; // If the location was not provided, let's try to see if Yarn is under %PATH% string paths = buildParameters["PATH"]; AbsolutePath foundPath = AbsolutePath.Invalid; foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) { var nonEscapedPath = path.Trim('"'); // Sometimes PATH is not well-formed, so make sure we can actually recognize an absolute path there if (AbsolutePath.TryCreate(m_context.PathTable, nonEscapedPath, out var absolutePath)) { AbsolutePath pathToYarn = absolutePath.Combine(m_context.PathTable, "yarn"); if (m_host.Engine.FileExists(pathToYarn)) { foundPath = pathToYarn; break; } // In some windows installations, only yarn.cmd exists pathToYarn = absolutePath.Combine(m_context.PathTable, "yarn.cmd"); if (m_host.Engine.FileExists(pathToYarn)) { foundPath = pathToYarn; break; } } } if (!foundPath.IsValid) { failure = "A location for 'yarn' is not explicitly specified. However, 'yarn' doesn't seem to be part of PATH. You can either specify the location explicitly using 'yarnLocation' field in " + $"the Yarn resolver configuration, or make sure 'yarn' is part of your PATH. Current PATH is '{paths}'."; return(false); } failure = string.Empty; finalYarnLocation = foundPath; // Just verbose log this Tracing.Logger.Log.UsingYarnAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalYarnLocation.ToString(m_context.PathTable)); return(true); }
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync( AbsolutePath responseFile, IEnumerable <AbsolutePath> projectEntryPoints, AbsolutePath outputFile, IEnumerable <AbsolutePath> msBuildSearchLocations, AbsolutePath dotnetExeLocation, BuildParameters.IBuildParameters buildParameters) { Contract.Assert(!m_resolverSettings.ShouldRunDotNetCoreMSBuild() || dotnetExeLocation.IsValid); AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable); string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable); string outputFileString = outputFile.ToString(m_context.PathTable); IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>(); var requestedQualifiers = m_host.QualifiersToEvaluate.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList(); var arguments = new MSBuildGraphBuilderArguments( projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(), outputFileString, new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()), msBuildSearchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(), entryPointTargets, requestedQualifiers, m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true, m_resolverSettings.ShouldRunDotNetCoreMSBuild()); var responseFilePath = responseFile.ToString(m_context.PathTable); SerializeResponseFile(responseFilePath, arguments); string graphConstructionToolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable); string pathToTool; string toolArguments; // if we should call the dotnet core version of MSBuild, we need to actually call dotnet.exe and pass the tool itself as its first argument if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) { pathToTool = dotnetExeLocation.ToString(m_context.PathTable); toolArguments = I($"\"{graphConstructionToolPath}\" \"{responseFilePath}\""); } else { pathToTool = graphConstructionToolPath; toolArguments = I($"\"{responseFilePath}\""); } Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool); // Just being defensive, make sure there is not an old output file lingering around File.Delete(outputFileString); return(FrontEndUtilities.RunSandboxedToolAsync( m_context, pathToTool, buildStorageDirectory: outputDirectory, fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile), arguments: toolArguments, workingDirectory: outputDirectory, description: "MsBuild graph builder", buildParameters, beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString))); }
private async Task <Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > > ComputeBuildGraphAsync( AbsolutePath responseFile, IEnumerable <AbsolutePath> projectEntryPoints, AbsolutePath outputFile, IEnumerable <AbsolutePath> msBuidSearchLocations, IEnumerable <AbsolutePath> dotnetSearchLocations, BuildParameters.IBuildParameters buildParameters) { AbsolutePath dotnetExeLocation = AbsolutePath.Invalid; if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) { if (!TryFindDotNetExe(dotnetSearchLocations, out dotnetExeLocation, out string failure)) { return(ProjectGraphWithPredictionsResult <AbsolutePath> .CreateFailure( GraphConstructionError.CreateFailureWithoutLocation(failure), CollectionUtilities.EmptyDictionary <string, AbsolutePath>(), AbsolutePath.Invalid)); } } SandboxedProcessResult result = await RunMsBuildGraphBuilderAsync(responseFile, projectEntryPoints, outputFile, msBuidSearchLocations, dotnetExeLocation, buildParameters); string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult(); if (result.ExitCode != 0) { // In case of a cancellation, the tool may have exited with a non-zero // code, but that's expected if (!m_context.CancellationToken.IsCancellationRequested) { // This should never happen! Report the standard error and exit gracefully Tracing.Logger.Log.GraphConstructionInternalError( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } // If the tool exited gracefully, but standard error is not empty, that // is interpreted as a warning. We propagate that to the BuildXL log if (!string.IsNullOrEmpty(standardError)) { Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable)); JsonSerializer serializer = ConstructProjectGraphSerializer(); using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable))) using (var reader = new JsonTextReader(sr)) { var projectGraphWithPredictionsResult = serializer.Deserialize <ProjectGraphWithPredictionsResult <AbsolutePath> >(reader); // A successfully constructed graph should always have a valid path to MsBuild Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.PathToMsBuild.IsValid); // A successfully constructed graph should always have at least one project node Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.Result.ProjectNodes.Length > 0); // A failed construction should always have a failure set Contract.Assert(projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.Failure != null); // Let's log the paths to the used MsBuild assemblies, just for debugging purposes Tracing.Logger.Log.GraphConstructionToolCompleted( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), string.Join(",\n", projectGraphWithPredictionsResult.MsBuildAssemblyPaths.Select(kvp => I($"[{kvp.Key}]:{kvp.Value.ToString(m_context.PathTable)}"))), projectGraphWithPredictionsResult.PathToMsBuild.ToString(m_context.PathTable)); return(m_resolverSettings.ShouldRunDotNetCoreMSBuild() ? projectGraphWithPredictionsResult.WithPathToDotNetExe(dotnetExeLocation) : projectGraphWithPredictionsResult); } }
private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync(IEnumerable <AbsolutePath> msBuildSearchLocations, IEnumerable <AbsolutePath> dotnetSearchLocations, IEnumerable <AbsolutePath> parsingEntryPoints, BuildParameters.IBuildParameters buildParameters) { // We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(MsBuildFrontEnd.Name); AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString()); // We create a unique response file that will contain the tool arguments AbsolutePath responseFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString()); // Make sure the directories are there FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, msBuildSearchLocations, dotnetSearchLocations, buildParameters); if (!maybeProjectGraphResult.Succeeded) { // A more specific error has been logged already return(maybeProjectGraphResult.Failure); } var projectGraphResult = maybeProjectGraphResult.Result; if (m_resolverSettings.KeepProjectGraphFile != true) { DeleteGraphBuilderRelatedFiles(outputFile, responseFile); } else { // Graph-related files are requested to be left on disk. Let's print a message with their location. Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable), responseFile.ToString(m_context.PathTable)); } if (!projectGraphResult.Succeeded) { var failure = projectGraphResult.Failure; Tracing.Logger.Log.ProjectGraphConstructionError(m_context.LoggingContext, failure.HasLocation ? failure.Location : m_resolverSettings.Location(m_context.PathTable), failure.Message); return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } ProjectGraphWithPredictions <AbsolutePath> projectGraph = projectGraphResult.Result; // The module contains all project files that are part of the graph var projectFiles = new HashSet <AbsolutePath>(); foreach (ProjectWithPredictions <AbsolutePath> node in projectGraph.ProjectNodes) { projectFiles.Add(node.FullPath); } var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this); var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences( moduleDescriptor, m_resolverSettings.RootTraversal, m_resolverSettings.File, projectFiles, allowedModuleDependencies: null, // no module policies cyclicalFriendModules: null); // no whitelist of cycles return(new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuild, projectGraphResult.PathToDotNetExe)); }
/// <inheritdoc/> protected override Task <Possible <RushGraphResult> > TryComputeBuildGraphAsync() { BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters(); return(TryComputeBuildGraphAsync(buildParameters)); }
/// <inheritdoc/> protected override bool TryFindGraphBuilderToolLocation(RushResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalRushLibBaseLocation, out string failure) { // If the base location was provided at configuration time, we honor it as is if (resolverSettings.RushLibBaseLocation.HasValue) { finalRushLibBaseLocation = resolverSettings.RushLibBaseLocation.Value.Path; failure = string.Empty; return(true); } finalRushLibBaseLocation = AbsolutePath.Invalid; // If the location was not provided, let's try to see if Rush is installed, since rush-lib comes as part of it // Look in %PATH% (as exposed in build parameters) for rush string paths = buildParameters["PATH"]; AbsolutePath foundPath = AbsolutePath.Invalid; foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) { var nonEscapedPath = path.Trim('"'); // Sometimes PATH is not well-formed, so make sure we can actually recognize an absolute path there if (AbsolutePath.TryCreate(m_context.PathTable, nonEscapedPath, out var absolutePath)) { if (m_host.Engine.FileExists(absolutePath.Combine(m_context.PathTable, "rush"))) { foundPath = absolutePath; break; } } } if (!foundPath.IsValid) { failure = "A location for 'rush-lib' is not explicitly specified, so trying to find a Rush installation to use instead. " + "However, 'rush' doesn't seem to be part of PATH. You can either specify the location explicitly using 'rushLibBaseLocation' field in " + $"the Rush resolver configuration, or make sure 'rush' is part of your PATH. Current PATH is '{paths}'."; return(false); } // We found where Rush is located. So rush-lib is a known dependency of it, so should be nested within Rush module // Observe that even if that's not the case the final validation will occur under the rush graph builder tool, when // the module is tried to be loaded failure = string.Empty; finalRushLibBaseLocation = foundPath.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, "node_modules/@microsoft/rush/node_modules")); // Just verbose log this Tracing.Logger.Log.UsingRushLibBaseAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalRushLibBaseLocation.ToString(m_context.PathTable)); return(true); }
/// <summary> /// Runs the a tool in a sandboxed process and returns the result. /// These optional callback Actions can be provided: /// beforeLaunch is invoked right before the process is launched /// onResult is invoked after getting a successful result /// </summary>> public static async Task <SandboxedProcessResult> RunSandboxedToolAsync(FrontEndContext context, string pathToTool, string buildStorageDirectory, FileAccessManifest fileAccessManifest, string arguments, string workingDirectory, string description, BuildParameters.IBuildParameters buildParameters, Action beforeLaunch = null, // Invoked right before the process starts Action onResult = null // Action to be taken after getting a successful result ) { var info = new SandboxedProcessInfo( context.PathTable, new ToolBuildStorage(buildStorageDirectory), pathToTool, fileAccessManifest, disableConHostSharing: true, ContainerConfiguration.DisabledIsolation, loggingContext: context.LoggingContext) { Arguments = arguments, WorkingDirectory = workingDirectory, PipSemiStableHash = 0, PipDescription = description, EnvironmentVariables = buildParameters }; var process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing : true); var registration = context.CancellationToken.Register( () => { try { process.KillAsync().GetAwaiter().GetResult(); } catch (TaskCanceledException) { // If the process has already terminated or doesn't exist, an TaskCanceledException is raised. // In either case, we swallow the exception, cancellation is already requested by the user } }); beforeLaunch?.Invoke(); var result = process.GetResultAsync().ContinueWith( r => { // Dispose the registration for the cancellation token once the process is done registration.Dispose(); // onResult?.Invoke(); return(r.GetAwaiter().GetResult()); }); return(await result); }
/// <inheritdoc/> protected override bool TryFindGraphBuilderToolLocation(ILageResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath npmLocation, out string failure) { // If the base location was provided at configuration time, we honor it as is if (resolverSettings.NpmLocation.HasValue) { npmLocation = resolverSettings.NpmLocation.Value.Path; failure = string.Empty; return(true); } // If the location was not provided, let's try to see if NPM is under %PATH% string paths = buildParameters["PATH"]; if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "npm", "npm.cmd" }, out npmLocation)) { failure = "A location for 'npm' is not explicitly specified. However, 'npm' doesn't seem to be part of PATH. You can either specify the location explicitly using 'npmLocation' field in " + $"the Lage resolver configuration, or make sure 'npm' is part of your PATH. Current PATH is '{paths}'."; return(false); } failure = string.Empty; // Just verbose log this Tracing.Logger.Log.UsingNpmAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), npmLocation.ToString(m_context.PathTable)); return(true); }
/// <inheritdoc/> protected override bool TryFindGraphBuilderToolLocation(IYarnResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalYarnLocation, out string failure) { // If the base location was provided at configuration time, we honor it as is string paths; if (resolverSettings.YarnLocation != null) { var value = resolverSettings.YarnLocation.GetValue(); if (value is FileArtifact file) { finalYarnLocation = file; failure = string.Empty; return(true); } else { var pathCollection = ((IReadOnlyList <DirectoryArtifact>)value).Select(dir => dir.Path); if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation)) { failure = $"'yarn' cannot be found under any of the provided paths '{string.Join(Path.PathSeparator.ToString(), pathCollection.Select(path => path.ToString(m_context.PathTable)))}'."; return(false); } failure = string.Empty; return(true); } } // If the location was not provided, let's try to see if Yarn is under %PATH% paths = buildParameters["PATH"]; if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation)) { failure = "A location for 'yarn' is not explicitly specified. However, 'yarn' doesn't seem to be part of PATH. You can either specify the location explicitly using 'yarnLocation' field in " + $"the Yarn resolver configuration, or make sure 'yarn' is part of your PATH. Current PATH is '{paths}'."; return(false); } failure = string.Empty; // Just verbose log this Tracing.Logger.Log.UsingYarnAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalYarnLocation.ToString(m_context.PathTable)); return(true); }
/// <summary> /// Compute the build graph by reading a user-specified package-to-package graph /// </summary> protected override Task <Possible <(JavaScriptGraph <YarnConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration>)> > ComputeBuildGraphAsync(BuildParameters.IBuildParameters buildParameters) { if (m_resolverSettings.CustomProjectGraph == null) { Tracing.Logger.Log.ErrorReadingCustomProjectGraph( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), "The custom project graph is undefined."); var failure = new Possible <(JavaScriptGraph <YarnConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration>)>(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); return(Task.FromResult(failure)); } // The graph may come from a file or from a DScript literal following the Yarn schema Possible <GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration> > maybeGraph; if (m_resolverSettings.CustomProjectGraph.GetValue() is AbsolutePath graphFile) { Contract.Assert(graphFile.IsValid); maybeGraph = ReadGraphFromFile(graphFile); } else { var graphLiteral = m_resolverSettings.CustomProjectGraph.GetValue() as IReadOnlyDictionary <string, IJavaScriptCustomProjectGraphNode>; maybeGraph = BuildGraphFromLiteral(graphLiteral); } // The graph is always resolved with execution semantics, since Yarn doesn't provide execution semantics var maybeResult = maybeGraph .Then(graph => ResolveGraphWithExecutionSemantics(graph)) .Then(resolvedGraph => (resolvedGraph, maybeGraph.Result)); // There is actually no graph file to 'keep' in this case, but in order to honor // this option, let's serialize to a file the graph we just constructed if (m_resolverSettings.KeepProjectGraphFile == true && maybeResult.Succeeded) { SerializeComputedGraph(maybeResult.Result.Result); } return(Task.FromResult(maybeResult)); }
/// <inheritdoc/> protected override bool TryFindGraphBuilderToolLocation(ILageResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalLageLocation, out string failure) { // We use an npm call to invoke Lage finalLageLocation = AbsolutePath.Invalid; failure = null; return(true); }
private async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync( AbsolutePath outputFile, BuildParameters.IBuildParameters buildParameters) { // Determine the base location to use for finding the graph construction tool if (!TryFindGraphBuilderToolLocation( m_resolverSettings, buildParameters, out AbsolutePath foundLocation, out string failure)) { Tracing.Logger.Log.CannotFindGraphBuilderTool( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), failure); return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } SandboxedProcessResult result = await RunJavaScriptGraphBuilderAsync(outputFile, buildParameters, foundLocation); string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult(); if (result.ExitCode != 0) { Tracing.Logger.Log.ProjectGraphConstructionError( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } // If the tool exited gracefully, but standard error is not empty, that // is interpreted as a warning. We propagate that to the BuildXL log if (!string.IsNullOrEmpty(standardError)) { Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable)); JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings); using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable))) using (var reader = new JsonTextReader(sr)) { var flattenedJavaScriptGraph = serializer.Deserialize <GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> >(reader); // If a custom script command callback is specified, give it a chance to alter the script commands of // each package if (m_resolverSettings.CustomScripts != null) { var projectsWithCustomScripts = new List <DeserializedJavaScriptProject>(flattenedJavaScriptGraph.Projects.Count); foreach (var project in flattenedJavaScriptGraph.Projects) { m_resolverSettings.Root.TryGetRelative(m_context.PathTable, project.ProjectFolder, out var relativeFolder); var maybeCustomScripts = ResolveCustomScripts(project.Name, relativeFolder); if (!maybeCustomScripts.Succeeded) { return(maybeCustomScripts.Failure); } var customScripts = maybeCustomScripts.Result; // A null customScript means the callback did not provide any customization projectsWithCustomScripts.Add(customScripts == null ? project : project.WithCustomScripts(customScripts)); } flattenedJavaScriptGraph = new GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>( projectsWithCustomScripts, flattenedJavaScriptGraph.Configuration); } Possible <JavaScriptGraph <TGraphConfiguration> > graph = ApplyBxlExecutionSemantics() ? ResolveGraphWithExecutionSemantics(flattenedJavaScriptGraph) : ResolveGraphWithoutExecutionSemantics(flattenedJavaScriptGraph); return(graph.Then(graph => new Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)>((graph, flattenedJavaScriptGraph)))); } }
/// <summary> /// Tries to find the JavaScript-based tool that will be pass as a parameter to the Bxl graph construction tool /// </summary> /// <remarks> /// For example, for Rush this is the location of rush-lib. For Yarn, the location of yarn /// </remarks> protected abstract bool TryFindGraphBuilderToolLocation( TResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath location, out string failure);
private async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync( AbsolutePath outputFile, BuildParameters.IBuildParameters buildParameters) { // Determine the base location to use for finding the graph construction tool if (!TryFindGraphBuilderToolLocation( m_resolverSettings, buildParameters, out AbsolutePath foundLocation, out string failure)) { Tracing.Logger.Log.CannotFindGraphBuilderTool( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), failure); return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } string nodeExeLocation; if (m_resolverSettings.NodeExeLocation != null) { var specifiedNodeExe = m_resolverSettings.NodeExeLocation.GetValue(); AbsolutePath nodeExeLocationPath; if (specifiedNodeExe is FileArtifact fileArtifact) { nodeExeLocationPath = fileArtifact.Path; } else { var pathCollection = ((IReadOnlyList <DirectoryArtifact>)specifiedNodeExe).Select(dir => dir.Path); if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "node", "node.exe" }, out nodeExeLocationPath)) { failure = $"'node' cannot be found under any of the provided paths '{string.Join(";", pathCollection.Select(path => path.ToString(m_context.PathTable)))}'."; Tracing.Logger.Log.CannotFindGraphBuilderTool( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), failure); return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } } nodeExeLocation = nodeExeLocationPath.ToString(m_context.PathTable); // Most graph construction tools (yarn, rush, etc.) rely on node.exe being on the PATH. Make sure // that's the case by appending the PATH exposed to the graph construction process with the location of the // specified node.exe. By prepending PATH with it, we also make sure yarn/rush will be using the same version // of node the user specified. string pathWithNode = buildParameters.ContainsKey("PATH") ? buildParameters["PATH"] : string.Empty; var nodeDirectory = nodeExeLocationPath.GetParent(m_context.PathTable); if (nodeDirectory.IsValid) { pathWithNode = nodeDirectory.ToString(m_context.PathTable) + Path.PathSeparator + pathWithNode; } buildParameters = buildParameters.Override(new[] { new KeyValuePair <string, string>("PATH", pathWithNode) }); } else { // We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work. nodeExeLocation = "node.exe"; } SandboxedProcessResult result = await RunJavaScriptGraphBuilderAsync(nodeExeLocation, outputFile, buildParameters, foundLocation); string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult(); if (result.ExitCode != 0) { Tracing.Logger.Log.ProjectGraphConstructionError( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable)); } // If the tool exited gracefully, but standard error is not empty, that // is interpreted as a warning. We propagate that to the BuildXL log if (!string.IsNullOrEmpty(standardError)) { Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), standardError); } TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable)); JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings); using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable))) using (var reader = new JsonTextReader(sr)) { var flattenedJavaScriptGraph = serializer.Deserialize <GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> >(reader); // If a custom script command callback is specified, give it a chance to alter the script commands of // each package if (m_resolverSettings.CustomScripts != null) { var projectsWithCustomScripts = new List <DeserializedJavaScriptProject>(flattenedJavaScriptGraph.Projects.Count); foreach (var project in flattenedJavaScriptGraph.Projects) { m_resolverSettings.Root.TryGetRelative(m_context.PathTable, project.ProjectFolder, out var relativeFolder); var maybeCustomScripts = ResolveCustomScripts(project.Name, relativeFolder); if (!maybeCustomScripts.Succeeded) { return(maybeCustomScripts.Failure); } var customScripts = maybeCustomScripts.Result; // A null customScript means the callback did not provide any customization projectsWithCustomScripts.Add(customScripts == null ? project : project.WithCustomScripts(customScripts)); } flattenedJavaScriptGraph = new GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>( projectsWithCustomScripts, flattenedJavaScriptGraph.Configuration); } Possible <JavaScriptGraph <TGraphConfiguration> > graph = ApplyBxlExecutionSemantics() ? ResolveGraphWithExecutionSemantics(flattenedJavaScriptGraph) : ResolveGraphWithoutExecutionSemantics(flattenedJavaScriptGraph); return(graph.Then(graph => new Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)>((graph, flattenedJavaScriptGraph)))); } }