private static ProjectGraphWithPredictionsResult BuildGraph(
IMsBuildAssemblyLoader assemblyLoader,
GraphBuilderReporter reporter,
MSBuildGraphBuilderArguments arguments,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting)
{
reporter.ReportMessage("Looking for MSBuild toolset...");
if (!assemblyLoader.TryLoadMsBuildAssemblies(arguments.MSBuildSearchLocations, reporter, out string failure, out IReadOnlyDictionary <string, string> locatedAssemblyPaths, out string locatedMsBuildPath))
{
return(ProjectGraphWithPredictionsResult.CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation(failure),
locatedAssemblyPaths,
locatedMsBuildPath));
}
reporter.ReportMessage("Done looking for MSBuild toolset.");
return(BuildGraphInternal(
reporter,
locatedAssemblyPaths,
locatedMsBuildPath,
arguments,
projectPredictorsForTesting));
}
/// <inheritdoc/>
public bool TryLoadMsBuildAssemblies(
IEnumerable <string> searchLocations,
GraphBuilderReporter reporter,
out string failureReason,
out IReadOnlyDictionary <string, string> locatedAssemblyPaths,
out string locatedMsBuildExePath)
{
// We want to make sure the provided locations actually contain all the needed assemblies
if (!TryGetAssemblyLocations(searchLocations, reporter, out locatedAssemblyPaths, out IReadOnlyDictionary <string, string> missingAssemblyNames, out locatedMsBuildExePath))
{
var locationString = string.Join(", ", searchLocations.Select(location => location));
var missingAssemblyString = string.Join(", ", missingAssemblyNames.Select(nameAndReason => $"['{nameAndReason.Key}' Reason: {nameAndReason.Value}]"));
failureReason = $"Cannot find the required MSBuild toolset. Missing assemblies: {missingAssemblyString}. Searched locations: [{locationString}]";
return(false);
}
var assemblyPathsToLoad = locatedAssemblyPaths;
s_msBuildHandler = (_, eventArgs) =>
{
var assemblyNameString = eventArgs.Name;
var assemblyName = new AssemblyName(assemblyNameString).Name + ".dll";
// If we already loaded the requested assembly, just return it
if (m_loadedAssemblies.TryGetValue(assemblyName, out var assembly))
{
return(assembly);
}
if (assemblyPathsToLoad.TryGetValue(assemblyName, out string targetAssemblyPath))
{
assembly = Assembly.LoadFrom(targetAssemblyPath);
m_loadedAssemblies.TryAdd(assemblyNameString, assembly);
// Automatically un-register the handler once all supported assemblies have been loaded.
// No need to synchronize threads here, if the handler was already removed, nothing bad happens
if (m_loadedAssemblies.Count == m_assemblyNamesToLoad.Length)
{
AppDomain.CurrentDomain.AssemblyResolve -= s_msBuildHandler;
}
return(assembly);
}
return(null);
};
AppDomain.CurrentDomain.AssemblyResolve += s_msBuildHandler;
failureReason = string.Empty;
return(true);
}
/// <summary>
/// For tests only. Similar to <see cref="BuildGraphAndSerialize(MSBuildGraphBuilderArguments)"/>, but the assembly loader and reporter can be passed explicitly
/// </summary>
internal static void BuildGraphAndSerializeForTesting(
IMsBuildAssemblyLoader assemblyLoader,
GraphBuilderReporter reporter,
MSBuildGraphBuilderArguments arguments,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting = null)
{
DoBuildGraphAndSerialize(
assemblyLoader,
reporter,
arguments,
projectPredictorsForTesting);
}
/// <summary>
/// Makes sure the required MsBuild assemblies are loaded from, uses the MsBuild static graph API to get a build graph starting
/// at the project entry point and serializes it to an output file.
/// </summary>
/// <remarks>
/// Legit errors while trying to load the MsBuild assemblies or constructing the graph are represented in the serialized result
/// </remarks>
public static void BuildGraphAndSerialize(
MSBuildGraphBuilderArguments arguments)
{
Contract.Requires(arguments != null);
// Using the standard assembly loader and reporter
// The output file is used as a unique name to identify the pipe
using (var reporter = new GraphBuilderReporter(Path.GetFileName(arguments.OutputPath)))
{
DoBuildGraphAndSerialize(new MsBuildAssemblyLoader(arguments.MsBuildRuntimeIsDotNetCore), reporter, arguments);
}
}
private static void DoBuildGraphAndSerialize(
IMsBuildAssemblyLoader assemblyLoader,
GraphBuilderReporter reporter,
MSBuildGraphBuilderArguments arguments,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting = null)
{
reporter.ReportMessage("Starting MSBuild graph construction process...");
var stopwatch = Stopwatch.StartNew();
ProjectGraphWithPredictionsResult graphResult = BuildGraph(assemblyLoader, reporter, arguments, projectPredictorsForTesting);
SerializeGraph(graphResult, arguments.OutputPath, reporter);
reporter.ReportMessage($"Done constructing build graph in {stopwatch.ElapsedMilliseconds}ms.");
}
private bool TryGetAssemblyLocations(
IEnumerable <string> locations,
GraphBuilderReporter reporter,
out IReadOnlyDictionary </* assembly name */ string, /* full path */ string> assemblyPathsToLoad,
out IReadOnlyDictionary </* assembly name */ string, /* not found reason */ string> missingAssemblies,
out string locatedMsBuildExePath)
{
var foundAssemblies = new Dictionary <string, string>(m_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase);
var notFoundAssemblies = new Dictionary <string, string>(m_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase);
var assembliesToFind = new HashSet <string>(m_assemblyNamesToLoad, StringComparer.OrdinalIgnoreCase);
locatedMsBuildExePath = string.Empty;
foreach (var location in locations)
{
reporter.ReportMessage($"Looking for MSBuild toolset in '{location}'.");
Contract.Assert(!string.IsNullOrEmpty(location), "Specified search location must not be null or empty");
try
{
var dlls = Directory.EnumerateFiles(location, "*.dll", SearchOption.TopDirectoryOnly);
var msBuildExe = Directory.EnumerateFiles(location, m_msbuild, SearchOption.TopDirectoryOnly);
foreach (string fullPath in dlls.Union(msBuildExe))
{
var file = Path.GetFileName(fullPath);
if (assembliesToFind.Contains(Path.GetFileName(fullPath)))
{
var assemblyName = AssemblyName.GetAssemblyName(fullPath);
if (IsMsBuildAssembly(assemblyName, fullPath, out string notFoundReason))
{
assembliesToFind.Remove(file);
notFoundAssemblies.Remove(file);
// If the file found is msbuild.exe, we update the associated location but don't store the path
// in foundAssemblies, since this is not an assembly we really want to load
if (file.Equals(m_msbuild, StringComparison.OrdinalIgnoreCase))
{
locatedMsBuildExePath = fullPath;
}
else
{
// We store the first occurrence
if (!foundAssemblies.ContainsKey(file))
{
foundAssemblies[file] = fullPath;
}
}
// Shortcut the search if we already found all required ones
if (assembliesToFind.Count == 0)
{
break;
}
}
else
{
// We want to store the first reason for not being able to find an assembly that actually looks like the right one
if (!notFoundAssemblies.ContainsKey(file))
{
notFoundAssemblies.Add(file, notFoundReason);
}
}
}
}
// Shortcut the search if we already found all required ones
if (assembliesToFind.Count == 0)
{
break;
}
}
// Ignore unauthorized and IO exceptions: if we cannot get to the specified locations, we should skip those
catch (UnauthorizedAccessException ex)
{
reporter.ReportMessage($"Search location '{location}' is skipped because the process is not authorized to enumerate it. Details: {ex.Message}");
}
catch (IOException ex)
{
reporter.ReportMessage($"Search location '{location}' is skipped because an IO exception occurred while enumerating it. Details: {ex.Message}");
}
}
// When we are done enumerating the locations to search, if there are still assemblies we haven't found,
// add them to the not found set with the right reason
foreach (var assemblyName in assembliesToFind)
{
if (!notFoundAssemblies.ContainsKey(assemblyName))
{
notFoundAssemblies.Add(assemblyName, "Assembly not found.");
}
}
missingAssemblies = notFoundAssemblies;
assemblyPathsToLoad = foundAssemblies;
return(assembliesToFind.Count == 0);
}
private static bool TryConstructGraph(
ProjectGraph projectGraph,
GraphBuilderReporter reporter,
ConcurrentDictionary <ProjectInstance, Project> projectInstanceToProjectCache,
IReadOnlyCollection <string> entryPointTargets,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting,
bool allowProjectsWithoutTargetProtocol,
out ProjectGraphWithPredictions projectGraphWithPredictions,
out string failure)
{
Contract.Assert(projectGraph != null);
var projectNodes = new ProjectWithPredictions[projectGraph.ProjectNodes.Count];
var nodes = projectGraph.ProjectNodes.ToArray();
// Compute the list of targets to run per project
reporter.ReportMessage("Computing targets to execute for each project...");
// This dictionary should be exclusively read only at this point, and therefore thread safe
var targetsPerProject = projectGraph.GetTargetLists(entryPointTargets.ToArray());
// Bidirectional access from nodes with predictions to msbuild nodes in order to compute node references in the second pass
// TODO: revisit the structures, since the projects are known upfront we might be able to use lock-free structures
var nodeWithPredictionsToMsBuildNodes = new ConcurrentDictionary <ProjectWithPredictions, ProjectGraphNode>(Environment.ProcessorCount, projectNodes.Length);
var msBuildNodesToNodeWithPredictionIndex = new ConcurrentDictionary <ProjectGraphNode, ProjectWithPredictions>(Environment.ProcessorCount, projectNodes.Length);
reporter.ReportMessage("Statically predicting inputs and outputs...");
// Create the registered predictors and initialize the prediction executor
// The prediction executor potentially initializes third-party predictors, which may contain bugs. So let's be very defensive here
IReadOnlyCollection <IProjectPredictor> predictors;
try
{
predictors = projectPredictorsForTesting ?? ProjectPredictors.AllPredictors;
}
catch (Exception ex)
{
failure = $"Cannot create standard predictors. An unexpected error occurred. Please contact BuildPrediction project owners with this stack trace: {ex.ToString()}";
projectGraphWithPredictions = new ProjectGraphWithPredictions(new ProjectWithPredictions <string>[] { });
return(false);
}
// Using single-threaded prediction since we're parallelizing on project nodes instead.
var predictionExecutor = new ProjectPredictionExecutor(predictors, new ProjectPredictionOptions {
MaxDegreeOfParallelism = 1
});
// Each predictor may return unexpected/incorrect results and targets may not be able to be predicted. We put those failures here for post-processing.
ConcurrentQueue <(string predictorName, string failure)> predictionFailures = new ConcurrentQueue <(string, string)>();
var predictedTargetFailures = new ConcurrentQueue <string>();
// The predicted targets to execute (per project) go here
var computedTargets = new ConcurrentBigMap <ProjectGraphNode, PredictedTargetsToExecute>();
// When projects are allowed to not implement the target protocol, its references need default targets as a post-processing step
var pendingAddDefaultTargets = new ConcurrentBigSet <ProjectGraphNode>();
// First pass
// Predict all projects in the graph in parallel and populate ProjectNodes
Parallel.For(0, projectNodes.Length, (int i) => {
ProjectGraphNode msBuildNode = nodes[i];
ProjectInstance projectInstance = msBuildNode.ProjectInstance;
Project project = projectInstanceToProjectCache[projectInstance];
var outputFolderPredictions = new List <string>();
var predictionCollector = new MsBuildOutputPredictionCollector(outputFolderPredictions, predictionFailures);
try
{
// Again, be defensive when using arbitrary predictors
predictionExecutor.PredictInputsAndOutputs(project, predictionCollector);
}
catch (Exception ex)
{
predictionFailures.Enqueue((
"Unknown predictor",
$"Cannot run static predictor on project '{project.FullPath ?? "Unknown project"}'. An unexpected error occurred. Please contact BuildPrediction project owners with this stack trace: {ex.ToString()}"));
}
/// <summary>
/// Assumes the proper MsBuild assemblies are loaded already
/// </summary>
private static ProjectGraphWithPredictionsResult BuildGraphInternal(
GraphBuilderReporter reporter,
IReadOnlyDictionary <string, string> assemblyPathsToLoad,
string locatedMsBuildPath,
MSBuildGraphBuilderArguments graphBuildArguments,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting)
{
try
{
reporter.ReportMessage("Parsing MSBuild specs and constructing the build graph...");
var projectInstanceToProjectCache = new ConcurrentDictionary <ProjectInstance, Project>();
if (!TryBuildEntryPoints(
graphBuildArguments.ProjectsToParse,
graphBuildArguments.RequestedQualifiers,
graphBuildArguments.GlobalProperties,
out List <ProjectGraphEntryPoint> entryPoints,
out string failure))
{
return(ProjectGraphWithPredictionsResult.CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation(failure),
assemblyPathsToLoad,
locatedMsBuildPath));
}
var projectGraph = new ProjectGraph(
entryPoints,
// The project collection doesn't need any specific global properties, since entry points already contain all the ones that are needed, and the project graph will merge them
new ProjectCollection(),
(projectPath, globalProps, projectCollection) => ProjectInstanceFactory(projectPath, globalProps, projectCollection, projectInstanceToProjectCache));
// This is a defensive check to make sure the assembly loader actually honored the search locations provided by the user. The path of the assembly where ProjectGraph
// comes from has to be one of the provided search locations.
// If that's not the case, this is really an internal error. For example, the MSBuild dlls we use to compile against (that shouldn't be deployed) somehow sneaked into
// the deployment. This happened in the past, and it prevents the loader to redirect appropriately.
Assembly assembly = Assembly.GetAssembly(projectGraph.GetType());
string assemblylocation = assembly.Location;
if (!assemblyPathsToLoad.Values.Contains(assemblylocation, StringComparer.InvariantCultureIgnoreCase))
{
return(ProjectGraphWithPredictionsResult.CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation($"Internal error: the assembly '{assembly.GetName().Name}' was loaded from '{assemblylocation}'. This path doesn't match any of the provided search locations. Please contact the BuildXL team."),
assemblyPathsToLoad,
locatedMsBuildPath));
}
reporter.ReportMessage("Done parsing MSBuild specs.");
if (!TryConstructGraph(
projectGraph,
reporter,
projectInstanceToProjectCache,
graphBuildArguments.EntryPointTargets,
projectPredictorsForTesting,
graphBuildArguments.AllowProjectsWithoutTargetProtocol,
out ProjectGraphWithPredictions projectGraphWithPredictions,
out failure))
{
return(ProjectGraphWithPredictionsResult.CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation(failure),
assemblyPathsToLoad,
locatedMsBuildPath));
}
return(ProjectGraphWithPredictionsResult.CreateSuccessfulGraph(projectGraphWithPredictions, assemblyPathsToLoad, locatedMsBuildPath));
}
catch (InvalidProjectFileException e)
{
return(CreateFailureFromInvalidProjectFile(assemblyPathsToLoad, locatedMsBuildPath, e));
}
catch (AggregateException e)
{
// If there is an invalid project file exception, use that one since it contains the location.
var invalidProjectFileException = (InvalidProjectFileException)e.Flatten().InnerExceptions.FirstOrDefault(ex => ex is InvalidProjectFileException);
if (invalidProjectFileException != null)
{
return(CreateFailureFromInvalidProjectFile(assemblyPathsToLoad, locatedMsBuildPath, invalidProjectFileException));
}
// Otherwise, we don't have a location, so we use the message of the originating exception
return(ProjectGraphWithPredictionsResult.CreateFailure(
GraphConstructionError.CreateFailureWithoutLocation(
e.InnerException != null ? e.InnerException.Message : e.Message),
assemblyPathsToLoad,
locatedMsBuildPath));
}
}
