/// <summary> /// Choose a worker based on setup cost /// </summary> private Worker ChooseWorker(RunnablePip runnablePip, WorkerSetupCost[] workerSetupCosts, out WorkerResource?limitingResource) { if (MustRunOnMaster(runnablePip)) { // This is shortcut for the single-machine builds and distributed workers. return(LocalWorker.TryAcquire(runnablePip, out limitingResource, loadFactor: MaxLoadFactor) ? LocalWorker : null); } ResetStatus(); var pendingWorkerSelectionPipCount = PipQueue.GetNumQueuedByKind(DispatcherKind.ChooseWorkerCpu); bool loadBalanceWorkers = false; if (runnablePip.PipType == PipType.Process) { if (pendingWorkerSelectionPipCount + m_totalAcquiredProcessSlots < (m_totalProcessSlots / 2)) { // When there is a limited amount of work (less than half the total capacity of // the all the workers). We load balance so that each worker gets // its share of the work and the work can complete faster loadBalanceWorkers = true; } } long setupCostForBestWorker = workerSetupCosts[0].SetupBytes; limitingResource = null; foreach (var loadFactor in m_workerBalancedLoadFactors) { if (!loadBalanceWorkers && loadFactor < 1) { // Not load balancing so allow worker to be filled to capacity at least continue; } for (int i = 0; i < workerSetupCosts.Length; i++) { var worker = workerSetupCosts[i].Worker; if (worker.TryAcquire(runnablePip, out limitingResource, loadFactor: loadFactor)) { runnablePip.Performance.SetInputMaterializationCost(ByteSizeFormatter.ToMegabytes((ulong)setupCostForBestWorker), ByteSizeFormatter.ToMegabytes((ulong)workerSetupCosts[i].SetupBytes)); return(worker); } } } return(null); }
/// <summary> /// Choose a worker based on setup cost /// </summary> private Worker ChooseWorker(RunnablePip runnablePip, WorkerSetupCost[] workerSetupCosts, out WorkerResource?limitingResource) { if (MustRunOnMaster(runnablePip)) { // This is shortcut for the single-machine builds and distributed workers. return(LocalWorker.TryAcquire(runnablePip, out limitingResource, loadFactor: MaxLoadFactor) ? LocalWorker : null); } ResetStatus(); var pendingWorkerSelectionPipCount = PipQueue.GetNumQueuedByKind(DispatcherKind.ChooseWorkerCpu) + PipQueue.GetNumRunningByKind(DispatcherKind.ChooseWorkerCpu); bool loadBalanceWorkers = false; if (runnablePip.PipType == PipType.Process) { if (pendingWorkerSelectionPipCount + m_totalAcquiredProcessSlots < (m_totalProcessSlots / 2)) { // When there is a limited amount of work (less than half the total capacity of // the all the workers). We load balance so that each worker gets // its share of the work and the work can complete faster loadBalanceWorkers = true; } } double?disableLoadBalanceMultiplier = EngineEnvironmentSettings.DisableLoadBalanceMultiplier; // Disable load-balance if there is a multiplier specified including 0. loadBalanceWorkers &= !disableLoadBalanceMultiplier.HasValue; long setupCostForBestWorker = workerSetupCosts[0].SetupBytes; limitingResource = null; foreach (var loadFactor in m_workerBalancedLoadFactors) { if (!loadBalanceWorkers && loadFactor < 1) { // Not load balancing so allow worker to be filled to capacity at least continue; } for (int i = 0; i < workerSetupCosts.Length; i++) { var worker = workerSetupCosts[i].Worker; if (worker.TryAcquire(runnablePip, out limitingResource, loadFactor: loadFactor)) { runnablePip.Performance.SetInputMaterializationCost(ByteSizeFormatter.ToMegabytes((ulong)setupCostForBestWorker), ByteSizeFormatter.ToMegabytes((ulong)workerSetupCosts[i].SetupBytes)); return(worker); } // If the worker is not chosen due to the lack of process slots, // do not try the next worker immediately if 'BuildXLDisableLoadBalanceMultiplier' is specified. // We first check whether the number of pips waiting for a worker is less than the total slots times with the multiplier. // For example, if the multiplier is 1 and totalWorkerSlots is 100, then we do not try the next worker // if there are less than 100 pips waiting for a worker. // For Cosine builds, executing pips on a new worker is expensive due to the input materialization. // It is usually faster to wait for the busy worker to be available compared to trying on another worker. if (limitingResource == WorkerResource.AvailableProcessSlots && disableLoadBalanceMultiplier.HasValue && pendingWorkerSelectionPipCount < (worker.TotalProcessSlots * disableLoadBalanceMultiplier.Value)) { limitingResource = WorkerResource.DisableLoadBalance; return(null); } } } return(null); }