/// <summary> /// Adjusts the max parallel degree of the IO dispatcher queue /// </summary> /// <remarks> /// We introduce another limit for the IO queue, which is 'currentMax'. CurrentMax specifies the max parallel degree for the IO queue. /// CurrentMax initially equals to (maxIO + 1)/2. Then, based on the machine resources, it will vary between 1 and maxIO (both inclusive) during the build. /// This method will be called every second to adjust the IO limit. /// </remarks> public void AdjustIOParallelDegree(PerformanceCollector.MachinePerfInfo machinePerfInfo) { if (!IsDraining || !m_config.AdaptiveIO) { return; } var ioDispatcher = m_queuesByKind[DispatcherKind.IO]; int currentMax = ioDispatcher.MaxParallelDegree; // If numRunning is closer to the currentMax, consider increasing the limit based on the resource usage // We should not only look at the disk usage activity but also CPU, RAM as well because the pips running on the IO queue consume CPU and RAM resources as well. // TODO: Instead of looking at all disk usages, just look at the ones which are associated with the build files (both inputs and outputs). bool hasLowGlobalUsage = machinePerfInfo.CpuUsagePercentage < 90 && machinePerfInfo.RamUsagePercentage < 90 && machinePerfInfo.DiskUsagePercentages.All(a => a < 90); bool numRunningIsNearMax = ioDispatcher.NumRunning > currentMax * 0.8; if (numRunningIsNearMax && (currentMax < m_config.MaxIO) && hasLowGlobalUsage) { // The new currentMax will be the midpoint of currentMax and absoluteMax. currentMax = (m_config.MaxIO + currentMax + 1) / 2; ioDispatcher.AdjustParallelDegree(currentMax); TriggerDispatcher(); // After increasing the limit, trigger the dispatcher so that we can start new tasks. } else if (machinePerfInfo.DiskUsagePercentages.Any(a => a > 95)) { // If any of the disks usage is higher than 95%, then decrease the limit. // TODO: Should we look at the CPU or MEM usage as well to decrease the limit? currentMax = (currentMax + 1) / 2; ioDispatcher.AdjustParallelDegree(currentMax); } // TODO: Right now, we only care about the disk active time. We should also take the avg disk queue length into account. // Average disk queue length is a product of disk transfers/sec (response X I/O) and average disk sec/transfer. }
/// <inheritdoc/> public void AdjustIOParallelDegree(PerformanceCollector.MachinePerfInfo machinePerfInfo) { m_innerQueue.AdjustIOParallelDegree(machinePerfInfo); }