/// <summary> /// Creates a pip graph that corresponds to all the specified projects in the graph /// </summary> public async Task <Possible <ProjectGraphSchedulingResult <TProject> > > TrySchedulePipsForFilesAsync(IReadOnlySet <TProject> projectsToEvaluate, QualifierId qualifierId, Func <ProjectCreationResult <TProject>, Possible <ProcessOutputs> > customScheduler = null) { Contract.Requires(qualifierId.IsValid); if (!TryTopoSortProjectsAndComputeClosure(projectsToEvaluate, out PriorityQueue <(TProject, int tier)> topoSortedQueue, out IEnumerable <TProject> cycle)) { return(new Possible <ProjectGraphSchedulingResult <TProject> >(new CycleInProjectsFailure <TProject>(cycle))); } bool success = true; // Observe in case of multiple failure we non-deterministically report one Failure failure = null; var processOutputs = new ConcurrentDictionary <TProject, ProcessOutputs>(); ActionBlock <TProject> createActionBlockForTier() { return(new ActionBlock <TProject>( project => { // If a previous tier had any errors (which is guaranteed to be completed before the next tier is scheduled), just return, since the current tier may depend on pips that in the end // were not scheduled if (!success) { return; } // We only schedule the project if the project does not rule itself out from being added to the graph if (project.CanBeScheduled()) { var maybeResult = m_pipConstructor.TryCreatePipForProject(project, qualifierId); if (!maybeResult.Succeeded) { // Error is already logged success = false; failure = maybeResult.Failure; return; } // If a custom scheduler is not present, or it returns null when called, we add the resulting pip as the pip constructor created it if (customScheduler == null || (customScheduler(maybeResult.Result) is var maybeCustomSchedulingOutputs && maybeCustomSchedulingOutputs.Succeeded && maybeCustomSchedulingOutputs.Result == null)) { var maybeOutputs = m_pipConstructor.TrySchedulePipForProject(maybeResult.Result, qualifierId); if (!maybeOutputs.Succeeded) { // Error is already logged success = false; failure = maybeOutputs.Failure; return; } processOutputs.TryAdd(project, maybeOutputs.Result); } else if (maybeCustomSchedulingOutputs.Succeeded) { // In this case the custom scheduler should have already added the pip. Notify the pip constructor and keep track of the project -> output association m_pipConstructor.NotifyCustomProjectScheduled(project, maybeCustomSchedulingOutputs.Result, qualifierId); processOutputs.TryAdd(project, maybeCustomSchedulingOutputs.Result); } else { // In this case the custom scheduler evaluated to an error success = false; failure = maybeCustomSchedulingOutputs.Failure; return; } }
/// <summary> /// Creates a pip graph that corresponds to all the specified projects in the graph /// </summary> public async Task <bool> TrySchedulePipsForFilesAsync(IReadOnlySet <ProjectWithPredictions> projectsToEvaluate, QualifierId qualifierId) { Contract.Requires(qualifierId.IsValid); if (!TryTopoSortProjectsAndComputeClosure(projectsToEvaluate, out PriorityQueue <(ProjectWithPredictions, int tier)> topoSortedQueue, out IEnumerable <ProjectWithPredictions> cycle)) { var cycleDescription = string.Join(" -> ", cycle.Select(project => project.FullPath.ToString(m_context.PathTable))); Tracing.Logger.Log.CycleInBuildTargets(m_context.LoggingContext, cycleDescription); return(false); } bool success = true; ActionBlock <ProjectWithPredictions> CreateActionBlockForTier() { return(new ActionBlock <ProjectWithPredictions>( project => { // We only schedule the project if predicted target collection is non-empty if (project.PredictedTargetsToExecute.Targets.Count != 0) { if (!m_pipConstructor.TrySchedulePipForFile(project, qualifierId, out _, out _)) { // Error is already logged success = false; } } else { // Just log a verbose message indicating the project was not scheduled Tracing.Logger.Log.ProjectWithEmptyTargetsIsNotScheduled( m_context.LoggingContext, Location.FromFile(project.FullPath.ToString(m_context.PathTable)), project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable)); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = m_frontEndHost.FrontEndConfiguration.MaxFrontEndConcurrency() })); } int currentTier = 0; ActionBlock <ProjectWithPredictions> perTierParallelPipCreator = CreateActionBlockForTier(); while (topoSortedQueue.Count != 0 && success) { if (!success) { break; } var(project, tier) = topoSortedQueue.Top; topoSortedQueue.Pop(); if (tier != currentTier) { perTierParallelPipCreator.Complete(); await perTierParallelPipCreator.Completion; perTierParallelPipCreator = CreateActionBlockForTier(); currentTier++; } perTierParallelPipCreator.Post(project); } perTierParallelPipCreator.Complete(); await perTierParallelPipCreator.Completion; return(success); }
/// <summary> /// Creates a pip graph that corresponds to all the specified projects in the graph /// </summary> public async Task <Possible <ProjectGraphSchedulingResult <TProject> > > TrySchedulePipsForFilesAsync(IReadOnlySet <TProject> projectsToEvaluate, QualifierId qualifierId) { Contract.Requires(qualifierId.IsValid); if (!TryTopoSortProjectsAndComputeClosure(projectsToEvaluate, out PriorityQueue <(TProject, int tier)> topoSortedQueue, out IEnumerable <TProject> cycle)) { return(new Possible <ProjectGraphSchedulingResult <TProject> >(new CycleInProjectsFailure <TProject>(cycle))); } bool success = true; Failure failure = null; var processes = new ConcurrentDictionary <TProject, Pips.Operations.Process>(); ActionBlock <TProject> createActionBlockForTier() { return(new ActionBlock <TProject>( project => { // We only schedule the project if the project does not rule itself out from being added to the graph if (project.CanBeScheduled()) { var maybeProcess = m_pipConstructor.TrySchedulePipForProject(project, qualifierId); if (!maybeProcess.Succeeded) { // Error is already logged success = false; // Observe in case of multiple failure we non-deterministically report one failure = maybeProcess.Failure; } else { processes.TryAdd(project, maybeProcess.Result); } } else { m_pipConstructor.NotifyProjectNotScheduled(project); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = m_maxConcurrency })); } int currentTier = 0; ActionBlock <TProject> perTierParallelPipCreator = createActionBlockForTier(); while (topoSortedQueue.Count != 0 && success) { if (!success) { break; } var(project, tier) = topoSortedQueue.Top; topoSortedQueue.Pop(); if (tier != currentTier) { perTierParallelPipCreator.Complete(); await perTierParallelPipCreator.Completion; perTierParallelPipCreator = createActionBlockForTier(); currentTier++; } perTierParallelPipCreator.Post(project); } perTierParallelPipCreator.Complete(); await perTierParallelPipCreator.Completion; return(success? new ProjectGraphSchedulingResult <TProject>(processes) : (Possible <ProjectGraphSchedulingResult <TProject> >)failure); }