/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(ILageResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath npmLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
if (resolverSettings.NpmLocation.HasValue)
{
npmLocation = resolverSettings.NpmLocation.Value.Path;
failure = string.Empty;
return(true);
}
// If the location was not provided, let's try to see if NPM is under %PATH%
string paths = buildParameters["PATH"];
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "npm", "npm.cmd" }, out npmLocation))
{
failure = "A location for 'npm' is not explicitly specified. However, 'npm' doesn't seem to be part of PATH. You can either specify the location explicitly using 'npmLocation' field in " +
$"the Lage resolver configuration, or make sure 'npm' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
failure = string.Empty;
// Just verbose log this
Tracing.Logger.Log.UsingNpmAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), npmLocation.ToString(m_context.PathTable));
return(true);
}
private async Task <Possible <Unit> > GenerateBuildDirectoryAsync()
{
Contract.Assert(m_buildDirectory.IsValid);
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath argumentsFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
if (!TryRetrieveCMakeSearchLocations(out IEnumerable <AbsolutePath> searchLocations))
{
return(new CMakeGenerationError(m_resolverSettings.ModuleName, m_buildDirectory.ToString(m_context.PathTable)));
}
SandboxedProcessResult result = await ExecuteCMakeRunner(argumentsFile, searchLocations);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
if (!m_context.CancellationToken.IsCancellationRequested)
{
Tracing.Logger.Log.CMakeRunnerInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new CMakeGenerationError(m_resolverSettings.ModuleName, m_buildDirectory.ToString(m_context.PathTable)));
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses, outputDirectory);
return(Possible.Create(Unit.Void));
}
private void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
RushProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
var logFile = logDirectory.Combine(PathTable, "build.log");
// Execute the build command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.BuildCommand));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(">"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(logFile));
processBuilder.AddOutputFile(logFile, FileExistence.Required);
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
/// <inheritdoc />
public async Task <bool?> TryConvertModuleToEvaluationAsync(IModuleRegistry moduleRegistry, ParsedModule module, IWorkspace workspace)
{
if (!string.Equals(module.Descriptor.ResolverName, Name, StringComparison.Ordinal))
{
return(null);
}
var package = CreatePackage(module.Definition);
Contract.Assert(module.Specs.Count == 1, "This resolver generated the module, so we expect a single spec.");
var sourceKv = module.Specs.First();
// The in-memory generated spec is a regular DScript one, so run regular AST conversion
var result = await FrontEndUtilities.RunAstConversionAsync(m_frontEndHost, m_context, m_logger, m_frontEndStatistics, package, sourceKv.Key);
if (!result.Success)
{
return(false);
}
// Register the uninstantiated module
var moduleData = new UninstantiatedModuleInfo(
result.SourceFile,
result.Module,
result.QualifierSpaceId.IsValid ? result.QualifierSpaceId : m_context.QualifierTable.EmptyQualifierSpaceId);
m_frontEndHost.ModuleRegistry.AddUninstantiatedModuleInfo(moduleData);
return(true);
}
private void ConvertExportsFile(IModuleRegistry moduleRegistry, ParsedModule module, Package package)
{
var exportsFileModule = ModuleLiteral.CreateFileModule(
m_javaScriptWorkspaceResolver.ExportsFile,
moduleRegistry,
package,
module.Specs[m_javaScriptWorkspaceResolver.ExportsFile].LineMap);
// For each symbol defined in the resolver settings for exports, add all specified project outputs
int pos = 1;
foreach (var export in Exports)
{
FrontEndUtilities.AddEvaluationCallbackToFileModule(
exportsFileModule,
(context, moduleLiteral, evaluationStackFrame) =>
CollectProjectOutputsAsync(module.Definition.Specs, moduleLiteral.Qualifier.QualifierId, export, context.EvaluationScheduler),
export.FullSymbol,
pos);
pos += 2;
}
var moduleInfo = new UninstantiatedModuleInfo(
// We can register an empty one since we have the module populated properly
new Script.SourceFile(
m_javaScriptWorkspaceResolver.ExportsFile,
new Declaration[] { }),
exportsFileModule,
Context.QualifierTable.EmptyQualifierSpaceId);
moduleRegistry.AddUninstantiatedModuleInfo(moduleInfo);
}
private Task <SandboxedProcessResult> RunJavaScriptGraphBuilderAsync(
string nodeExeLocation,
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters,
AbsolutePath toolLocation)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
var toolArguments = GetGraphConstructionToolArguments(outputFile, toolLocation, toolPath, nodeExeLocation);
Tracing.Logger.Log.ConstructingGraphScript(m_context.LoggingContext, toolArguments);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_resolverSettings.Root.ToString(m_context.PathTable),
description: $"{Name} graph builder",
buildParameters));
}
private Task <SandboxedProcessResult> RunRushGraphBuilderAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
// We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work.
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
string nodeExe = m_resolverSettings.NodeExeLocation.HasValue ?
m_resolverSettings.NodeExeLocation.Value.Path.ToString(m_context.PathTable) :
"node.exe";
string pathToRushJson = m_resolverSettings.Root.Combine(m_context.PathTable, "rush.json").ToString(m_context.PathTable);
// TODO: add qualifier support.
// The graph construction tool expects: <path-to-rush.json> <path-to-output-graph> [<debug|release>]
string toolArguments = $@"/C """"{nodeExe}"" ""{toolPath.ToString(m_context.PathTable)}"" ""{pathToRushJson}"" ""{outputFile.ToString(m_context.PathTable)}"" debug""";
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_configuration.Layout.SourceDirectory.ToString(m_context.PathTable),
description: "Rush graph builder",
buildParameters));
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
private void TrackFilesAndEnvironment(BuildParameters.IBuildParameters buildParameters, ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Register all build parameters passed to the graph construction process
// TODO: we actually need the build parameters *used* by the graph construction process, but for now this is a compromise to keep
// graph caching sound. We need to modify this when MsBuild static graph API starts providing used env vars.
foreach (string key in buildParameters.ToDictionary().Keys)
{
Engine.TryGetBuildParameter(key, MsBuildFrontEnd.Name, out _);
}
FrontEndUtilities.TrackToolFileAccesses(Engine, Context, MsBuildFrontEnd.Name, fileAccesses, frontEndFolder);
}
/// <summary>
/// Retrieves a list of search locations for dotnet.exe
/// </summary>
/// <remarks>
/// First inspects the resolver configuration to check if these are defined explicitly. Otherwise, uses PATH environment variable.
/// </remarks>
private bool TryRetrieveDotNetSearchLocations(out IEnumerable <AbsolutePath> searchLocations)
{
return(FrontEndUtilities.TryRetrieveExecutableSearchLocations(
MsBuildFrontEnd.Name,
m_context,
m_host.Engine,
m_resolverSettings.DotNetSearchLocations?.SelectList(directoryLocation => directoryLocation.Path),
out searchLocations,
() => Tracing.Logger.Log.NoSearchLocationsSpecified(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), "dotnetSearchLocations"),
paths => Tracing.Logger.Log.CannotParseBuildParameterPath(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), paths)
));
}
private void TrackFilesAndEnvironment(ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Register all build parameters passed to the graph construction process
// Observe passthrough variables are explicitly skipped: we don't want the engine to track them
// TODO: we actually need the build parameters *used* by the graph construction process, but for now this is a compromise to keep
// graph caching sound. We need to modify this when MsBuild static graph API starts providing used env vars.
foreach (string key in m_userDefinedEnvironment.Keys)
{
m_host.Engine.TryGetBuildParameter(key, MsBuildFrontEnd.Name, out _);
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, MsBuildFrontEnd.Name, fileAccesses, frontEndFolder);
}
private async Task RunAstConverstionForFileAsync(Package package, AbsolutePath file)
{
var conversionResult = await FrontEndUtilities.RunAstConversionAsync(m_host, Context, m_logger, new FrontEndStatistics(), package, file);
Contract.Assert(conversionResult.Success);
var moduleData = new UninstantiatedModuleInfo(
conversionResult.SourceFile,
conversionResult.Module,
conversionResult.QualifierSpaceId.IsValid ? conversionResult.QualifierSpaceId : Context.QualifierTable.EmptyQualifierSpaceId);
m_host.ModuleRegistry.AddUninstantiatedModuleInfo(moduleData);
}
private async Task <Possible <NinjaGraphResult> > ComputeBuildGraphAsync()
{
AbsolutePath outputFile = SerializedGraphPath.Value;
SandboxedProcessResult result = await RunNinjaGraphBuilderAsync(outputFile);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
if (!m_context.CancellationToken.IsCancellationRequested)
{
Tracing.Logger.Log.GraphConstructionInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new NinjaGraphConstructionFailure(m_resolverSettings.ModuleName, ProjectRoot.ToString(m_context.PathTable)));
}
// If the tool exited gracefully, but standard error is not empty, that is interpreted as a warning
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
var serializer = JsonSerializer.Create(GraphSerializationSettings.Settings);
// Add custom deserializer for converting string arrays to AbsolutePath ReadOnlySets
serializer.Converters.Add(new RootAwareAbsolutePathConverter(m_context.PathTable, SpecFile.GetParent(m_context.PathTable)));
serializer.Converters.Add(new ToReadOnlySetJsonConverter <AbsolutePath>());
var outputFileString = outputFile.ToString(m_context.PathTable);
Tracing.Logger.Log.LeftGraphToolOutputAt(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), outputFileString);
NinjaGraphResult projectGraphWithPredictionResult;
using (var sr = new StreamReader(outputFileString))
using (var reader = new JsonTextReader(sr))
{
projectGraphWithPredictionResult = serializer.Deserialize <NinjaGraphResult>(reader);
}
return(projectGraphWithPredictionResult);
}
// Should be called from a Lazy
private IDictionary <string, string> GetAllEnvironmentVariables()
{
IDictionary <string, string> environment = FrontEndUtilities.GetEngineEnvironment(m_frontEndHost.Engine, m_frontEndName);
// Check if we are (supposedly) in the cloud (if the special folder exists)
if (!FileUtilities.Exists(m_manuallyDroppedDependenciesPath.Value.ToString(m_context.PathTable)))
{
return(environment);
}
else
{
return(SpecialCloudConfiguration.OverrideEnvironmentForCloud(environment, m_manuallyDroppedDependenciesPath.Value, m_context));
}
}
private Task <SandboxedProcessResult> ExecuteCMakeRunner(AbsolutePath argumentsFile, IEnumerable <AbsolutePath> searchLocations)
{
AbsolutePath pathToTool = Configuration.Layout.BuildEngineDirectory.Combine(Context.PathTable, m_relativePathToCMakeRunner);
string rootString = ProjectRoot.ToString(Context.PathTable);
AbsolutePath outputDirectory = argumentsFile.GetParent(Context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(Context.PathTable)); // Ensure it exists
SerializeToolArguments(argumentsFile, searchLocations);
void CleanUpOnResult()
{
try
{
FileUtilities.DeleteFile(argumentsFile.ToString(Context.PathTable));
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
Context.LoggingContext,
m_resolverSettings.Location(Context.PathTable),
argumentsFile.ToString(Context.PathTable),
e.Message);
}
}
var environment = FrontEndUtilities.GetEngineEnvironment(Engine, m_frontEnd.Name);
// TODO: This manual configuration is temporary. Remove after the cloud builders have the correct configuration
var pathToManuallyDroppedTools = Configuration.Layout.BuildEngineDirectory.Combine(Context.PathTable, RelativePath.Create(Context.StringTable, @"tools\CmakeNinjaPipEnvironment"));
if (FileUtilities.Exists(pathToManuallyDroppedTools.ToString(Context.PathTable)))
{
environment = SpecialCloudConfiguration.OverrideEnvironmentForCloud(environment, pathToManuallyDroppedTools, Context);
}
var buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(new ReadOnlyDictionary <string, string>(environment));
return(FrontEndUtilities.RunSandboxedToolAsync(
Context,
pathToTool.ToString(Context.PathTable),
buildStorageDirectory: outputDirectory.ToString(Context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(pathToTool.GetParent(Context.PathTable)),
arguments: I($@"""{argumentsFile.ToString(Context.PathTable)}"""),
workingDirectory: rootString,
description: "CMakeRunner",
buildParameters,
onResult: CleanUpOnResult));
}
/// <inheritdoc/>
public async Task <bool?> TryConvertModuleToEvaluationAsync(IModuleRegistry moduleRegistry, ParsedModule module, IWorkspace workspace)
{
// This resolver owns only one module.
if (!module.Definition.Equals(ModuleDefinition))
{
return(null);
}
var package = FrontEndUtilities.CreatePackage(module.Definition, Context.StringTable);
ConvertExportsFile(moduleRegistry, module, package);
await ConvertImportsFileAsync(package);
return(true);
}
/// <inheritdoc />
public Task <Possible <ISourceFile> > TryParseAsync(AbsolutePath pathToParse, AbsolutePath moduleOrConfigPathPromptingParse, ParsingOptions parsingOption = null)
{
Contract.Assume(m_descriptorsBySpecPath != null, "Init must have been called");
var pathToParseAsString = pathToParse.ToString(m_context.PathTable);
if (!m_descriptorsBySpecPath.TryGetValue(pathToParse, out var descriptor))
{
return(Task.FromResult <Possible <ISourceFile> >(new SpecNotOwnedByResolverFailure(pathToParseAsString)));
}
if (!Downloads.TryGetValue(descriptor.Name, out var downloadData))
{
Contract.Assert(false, "Inconsistent internal state of DownloadResolver");
return(Task.FromResult <Possible <ISourceFile> >(new SpecNotOwnedByResolverFailure(pathToParseAsString)));
}
string exeExtension = OperatingSystemHelper.IsWindowsOS ? ".exe" : string.Empty;
// CODESYNC: keep in sync with Public\Src\Tools\FileDownloader\Tool.FileDownloader.dsc deployment
AbsolutePath toolRootPath = m_host.Configuration.Layout.NormalizedBuildEngineDirectory.IsValid ? m_host.Configuration.Layout.NormalizedBuildEngineDirectory : m_host.Configuration.Layout.BuildEngineDirectory;
var pathToDownloader = toolRootPath.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, "/tools/FileDownloader/Downloader" + exeExtension));
var pathToExtractor = toolRootPath.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, "/tools/FileDownloader/Extractor" + exeExtension));
// Create a spec file that schedules two pips: a download one followed by an extract one. The latter only if extraction is specified
// CODESYNC: tools arguments and behavior defined in Public\Src\Tools\FileDownloader\Downloader.cs and \Public\Src\Tools\FileDownloader\Extractor.cs
var spec = $@"
export declare const qualifier: {{}};
const downloadTool = {CreateToolDefinition(pathToDownloader, dependsOnAppDataDirectory: true)}
const downloadResult = {CreateDownloadPip(downloadData)}
@@public export const {downloadData.DownloadedValueName} : File = downloadResult.getOutputFile(p`{downloadData.DownloadedFilePath.ToString(m_context.PathTable)}`);";
// The extract pip (and its corresponding public value) are only available if extraction needs to happen
if (downloadData.ShouldExtractBits)
{
spec += $@"
const extractTool = {CreateToolDefinition(pathToExtractor)}
const extractResult = {CreateExtractPip(downloadData)}
@@public export const {downloadData.ExtractedValueName} : OpaqueDirectory = extractResult.getOutputDirectory(d`{downloadData.ContentsFolder.Path.ToString(m_context.PathTable)}`);
";
}
// No need to check for errors here since they are embedded in the source file and downstream consumers check for those
FrontEndUtilities.TryParseSourceFile(m_context, pathToParse, spec, out var sourceFile);
return(Task.FromResult <Possible <ISourceFile> >(sourceFile));
}
private async Task ConvertImportsFileAsync(Package package)
{
if (m_javaScriptWorkspaceResolver.CustomSchedulingCallback != null)
{
// The imports file does not need any special callbacks and it is regular DScript. Run the normal AST conversion process on it.
var conversionResult = await FrontEndUtilities.RunAstConversionAsync(m_host, Context, m_logger, new FrontEndStatistics(), package, m_javaScriptWorkspaceResolver.ImportsFile);
Contract.Assert(conversionResult.Success);
var moduleData = new UninstantiatedModuleInfo(
conversionResult.SourceFile,
conversionResult.Module,
conversionResult.QualifierSpaceId.IsValid ? conversionResult.QualifierSpaceId : Context.QualifierTable.EmptyQualifierSpaceId);
m_host.ModuleRegistry.AddUninstantiatedModuleInfo(moduleData);
}
}
private void TrackFilesAndEnvironment(ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Register all build parameters passed to the graph construction process if they were retrieved from the process environment
// Otherwise, if build parameters were defined by the main config file, then there is nothing to register: if the definition
// in the config file actually accessed the environment, that was already registered during config evaluation.
// TODO: we actually need the build parameters *used* by the graph construction process, but for now this is a compromise to keep
// graph caching sound. We need to modify this when MsBuild static graph API starts providing used env vars.
if (m_processEnvironmentUsed)
{
foreach (string key in m_userDefinedEnvironment.Keys)
{
m_host.Engine.TryGetBuildParameter(key, MsBuildFrontEnd.Name, out _);
}
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, MsBuildFrontEnd.Name, fileAccesses, frontEndFolder);
}
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync(
AbsolutePath responseFile,
IEnumerable <AbsolutePath> projectEntryPoints,
AbsolutePath outputFile,
IEnumerable <AbsolutePath> searchLocations,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable);
string pathToTool = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
string outputFileString = outputFile.ToString(m_context.PathTable);
string enlistmentRoot = m_resolverSettings.Root.ToString(m_context.PathTable);
IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>();
var requestedQualifiers = m_requestedQualifiers.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList();
var arguments = new MSBuildGraphBuilderArguments(
enlistmentRoot,
projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(),
outputFileString,
new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()),
searchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(),
entryPointTargets,
requestedQualifiers,
m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true);
var responseFilePath = responseFile.ToString(m_context.PathTable);
SerializeResponseFile(responseFilePath, arguments);
Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool);
// Just being defensive, make sure there is not an old output file lingering around
File.Delete(outputFileString);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
pathToTool,
buildStorageDirectory: outputDirectory,
fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile),
arguments: I($"\"{responseFilePath}\""),
workingDirectory: outputDirectory,
description: "MsBuild graph builder",
buildParameters,
beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString)));
}
/// <inheritdoc/>
protected override bool TryFindGraphBuilderToolLocation(IYarnResolverSettings resolverSettings, BuildParameters.IBuildParameters buildParameters, out AbsolutePath finalYarnLocation, out string failure)
{
// If the base location was provided at configuration time, we honor it as is
string paths;
if (resolverSettings.YarnLocation != null)
{
var value = resolverSettings.YarnLocation.GetValue();
if (value is FileArtifact file)
{
finalYarnLocation = file;
failure = string.Empty;
return(true);
}
else
{
var pathCollection = ((IReadOnlyList <DirectoryArtifact>)value).Select(dir => dir.Path);
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation))
{
failure = $"'yarn' cannot be found under any of the provided paths '{string.Join(Path.PathSeparator.ToString(), pathCollection.Select(path => path.ToString(m_context.PathTable)))}'.";
return(false);
}
failure = string.Empty;
return(true);
}
}
// If the location was not provided, let's try to see if Yarn is under %PATH%
paths = buildParameters["PATH"];
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, paths, new[] { "yarn", "yarn.cmd" }, out finalYarnLocation))
{
failure = "A location for 'yarn' is not explicitly specified. However, 'yarn' doesn't seem to be part of PATH. You can either specify the location explicitly using 'yarnLocation' field in " +
$"the Yarn resolver configuration, or make sure 'yarn' is part of your PATH. Current PATH is '{paths}'.";
return(false);
}
failure = string.Empty;
// Just verbose log this
Tracing.Logger.Log.UsingYarnAt(m_context.LoggingContext, resolverSettings.Location(m_context.PathTable), finalYarnLocation.ToString(m_context.PathTable));
return(true);
}
private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
{
AbsolutePath pathToTool = Configuration.Layout.BuildEngineDirectory.Combine(Context.PathTable, m_relativePathToGraphConstructionTool);
string rootString = ProjectRoot.ToString(Context.PathTable);
AbsolutePath outputDirectory = outputFile.GetParent(Context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(Context.PathTable)); // Ensure it exists
AbsolutePath argumentsFile = outputDirectory.Combine(Context.PathTable, Guid.NewGuid().ToString());
SerializeToolArguments(outputFile, argumentsFile);
// After running the tool we'd like to remove some files
void CleanUpOnResult()
{
try
{
var shouldKeepArgs = m_resolverSettings.KeepToolFiles ?? false;
if (!shouldKeepArgs)
{
FileUtilities.DeleteFile(argumentsFile.ToString(Context.PathTable));
}
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
Context.LoggingContext,
m_resolverSettings.Location(Context.PathTable),
argumentsFile.ToString(Context.PathTable),
e.Message);
}
}
return(FrontEndUtilities.RunSandboxedToolAsync(
Context,
pathToTool.ToString(Context.PathTable),
buildStorageDirectory: outputDirectory.ToString(Context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(pathToTool.GetParent(Context.PathTable), outputFile),
arguments: I($@"""{argumentsFile.ToString(Context.PathTable)}"""),
workingDirectory: SpecFile.GetParent(Context.PathTable).ToString(Context.PathTable),
description: "Ninja graph builder",
BuildParameters.GetFactory().PopulateFromEnvironment(),
onResult: CleanUpOnResult));
}
private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
{
AbsolutePath outputDirectory = outputFile.GetParent(m_context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); // Ensure it exists
AbsolutePath argumentsFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
SerializeToolArguments(outputFile, argumentsFile);
// After running the tool we'd like to remove some files
void cleanUpOnResult()
{
try
{
var shouldKeepArgs = m_resolverSettings.KeepProjectGraphFile ?? false;
if (!shouldKeepArgs)
{
FileUtilities.DeleteFile(argumentsFile.ToString(m_context.PathTable));
}
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
argumentsFile.ToString(m_context.PathTable),
e.Message);
}
}
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
m_pathToTool.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory.ToString(m_context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(m_pathToTool.GetParent(m_context.PathTable), outputFile),
arguments: I($@"""{argumentsFile.ToString(m_context.PathTable)}"""),
workingDirectory: SpecFile.GetParent(m_context.PathTable).ToString(m_context.PathTable),
description: "Ninja graph builder",
RetrieveBuildParameters(),
onResult: cleanUpOnResult));
}
/// <inheritdoc/>
public Task <bool?> TryConvertModuleToEvaluationAsync(IModuleRegistry moduleRegistry, ParsedModule module, IWorkspace workspace)
{
// This resolver owns only one module.
if (!module.Definition.Equals(ModuleDefinition))
{
return(Task.FromResult <bool?>(null));
}
var exportsFileModule = ModuleLiteral.CreateFileModule(
m_javaScriptWorkspaceResolver.ExportsFile,
moduleRegistry,
FrontEndUtilities.CreatePackage(module.Definition, Context.StringTable),
module.Specs[m_javaScriptWorkspaceResolver.ExportsFile].LineMap);
// For each symbol defined in the resolver settings for exports, add all specified project outputs
int pos = 1;
foreach (var export in Exports)
{
FrontEndUtilities.AddEvaluationCallbackToFileModule(
exportsFileModule,
(context, moduleLiteral, evaluationStackFrame) =>
CollectProjectOutputsAsync(module.Definition.Specs, moduleLiteral.Qualifier.QualifierId, export),
export.FullSymbol,
pos);
pos += 2;
}
var moduleInfo = new UninstantiatedModuleInfo(
// We can register an empty one since we have the module populated properly
new SourceFile(
m_javaScriptWorkspaceResolver.ExportsFile,
new Declaration[] {}),
exportsFileModule,
Context.QualifierTable.EmptyQualifierSpaceId);
moduleRegistry.AddUninstantiatedModuleInfo(moduleInfo);
return(Task.FromResult <bool?>(true));
}
private Task <SandboxedProcessResult> RunMsBuildGraphBuilderAsync(
AbsolutePath responseFile,
IEnumerable <AbsolutePath> projectEntryPoints,
AbsolutePath outputFile,
IEnumerable <AbsolutePath> msBuildSearchLocations,
AbsolutePath dotnetExeLocation,
BuildParameters.IBuildParameters buildParameters)
{
Contract.Assert(!m_resolverSettings.ShouldRunDotNetCoreMSBuild() || dotnetExeLocation.IsValid);
AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
string outputFileString = outputFile.ToString(m_context.PathTable);
IReadOnlyCollection <string> entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray <string>();
var requestedQualifiers = m_host.QualifiersToEvaluate.Select(qualifierId => MsBuildResolverUtils.CreateQualifierAsGlobalProperties(qualifierId, m_context)).ToList();
var arguments = new MSBuildGraphBuilderArguments(
projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(),
outputFileString,
new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary <string, string>()),
msBuildSearchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(),
entryPointTargets,
requestedQualifiers,
m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true,
m_resolverSettings.ShouldRunDotNetCoreMSBuild());
var responseFilePath = responseFile.ToString(m_context.PathTable);
SerializeResponseFile(responseFilePath, arguments);
string graphConstructionToolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable);
string pathToTool;
string toolArguments;
// if we should call the dotnet core version of MSBuild, we need to actually call dotnet.exe and pass the tool itself as its first argument
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
pathToTool = dotnetExeLocation.ToString(m_context.PathTable);
toolArguments = I($"\"{graphConstructionToolPath}\" \"{responseFilePath}\"");
}
else
{
pathToTool = graphConstructionToolPath;
toolArguments = I($"\"{responseFilePath}\"");
}
Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool);
// Just being defensive, make sure there is not an old output file lingering around
File.Delete(outputFileString);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
pathToTool,
buildStorageDirectory: outputDirectory,
fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile),
arguments: toolArguments,
workingDirectory: outputDirectory,
description: "MsBuild graph builder",
buildParameters,
beforeLaunch: () => ConnectToServerPipeAndLogProgress(outputFileString)));
}
private async Task <Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)> > ComputeBuildGraphAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
// Determine the base location to use for finding the graph construction tool
if (!TryFindGraphBuilderToolLocation(
m_resolverSettings,
buildParameters,
out AbsolutePath foundLocation,
out string failure))
{
Tracing.Logger.Log.CannotFindGraphBuilderTool(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
failure);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
string nodeExeLocation;
if (m_resolverSettings.NodeExeLocation != null)
{
var specifiedNodeExe = m_resolverSettings.NodeExeLocation.GetValue();
AbsolutePath nodeExeLocationPath;
if (specifiedNodeExe is FileArtifact fileArtifact)
{
nodeExeLocationPath = fileArtifact.Path;
}
else
{
var pathCollection = ((IReadOnlyList <DirectoryArtifact>)specifiedNodeExe).Select(dir => dir.Path);
if (!FrontEndUtilities.TryFindToolInPath(m_context, m_host, pathCollection, new[] { "node", "node.exe" }, out nodeExeLocationPath))
{
failure = $"'node' cannot be found under any of the provided paths '{string.Join(";", pathCollection.Select(path => path.ToString(m_context.PathTable)))}'.";
Tracing.Logger.Log.CannotFindGraphBuilderTool(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
failure);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
}
nodeExeLocation = nodeExeLocationPath.ToString(m_context.PathTable);
// Most graph construction tools (yarn, rush, etc.) rely on node.exe being on the PATH. Make sure
// that's the case by appending the PATH exposed to the graph construction process with the location of the
// specified node.exe. By prepending PATH with it, we also make sure yarn/rush will be using the same version
// of node the user specified.
string pathWithNode = buildParameters.ContainsKey("PATH") ? buildParameters["PATH"] : string.Empty;
var nodeDirectory = nodeExeLocationPath.GetParent(m_context.PathTable);
if (nodeDirectory.IsValid)
{
pathWithNode = nodeDirectory.ToString(m_context.PathTable) + Path.PathSeparator + pathWithNode;
}
buildParameters = buildParameters.Override(new[] { new KeyValuePair <string, string>("PATH", pathWithNode) });
}
else
{
// We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work.
nodeExeLocation = "node.exe";
}
SandboxedProcessResult result = await RunJavaScriptGraphBuilderAsync(nodeExeLocation, outputFile, buildParameters, foundLocation);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
Tracing.Logger.Log.ProjectGraphConstructionError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
return(new JavaScriptGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings);
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var flattenedJavaScriptGraph = serializer.Deserialize <GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration> >(reader);
// If a custom script command callback is specified, give it a chance to alter the script commands of
// each package
if (m_resolverSettings.CustomScripts != null)
{
var projectsWithCustomScripts = new List <DeserializedJavaScriptProject>(flattenedJavaScriptGraph.Projects.Count);
foreach (var project in flattenedJavaScriptGraph.Projects)
{
m_resolverSettings.Root.TryGetRelative(m_context.PathTable, project.ProjectFolder, out var relativeFolder);
var maybeCustomScripts = ResolveCustomScripts(project.Name, relativeFolder);
if (!maybeCustomScripts.Succeeded)
{
return(maybeCustomScripts.Failure);
}
var customScripts = maybeCustomScripts.Result;
// A null customScript means the callback did not provide any customization
projectsWithCustomScripts.Add(customScripts == null ? project : project.WithCustomScripts(customScripts));
}
flattenedJavaScriptGraph = new GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>(
projectsWithCustomScripts, flattenedJavaScriptGraph.Configuration);
}
Possible <JavaScriptGraph <TGraphConfiguration> > graph = ApplyBxlExecutionSemantics() ? ResolveGraphWithExecutionSemantics(flattenedJavaScriptGraph) : ResolveGraphWithoutExecutionSemantics(flattenedJavaScriptGraph);
return(graph.Then(graph => new Possible <(JavaScriptGraph <TGraphConfiguration>, GenericJavaScriptGraph <DeserializedJavaScriptProject, TGraphConfiguration>)>((graph, flattenedJavaScriptGraph))));
}
}
