/// <summary>
/// Adjusts the max parallel degree of the IO dispatcher queue
/// </summary>
/// <remarks>
/// We introduce another limit for the IO queue, which is 'currentMax'. CurrentMax specifies the max parallel degree for the IO queue.
/// CurrentMax initially equals to (maxIO + 1)/2. Then, based on the machine resources, it will vary between 1 and maxIO (both inclusive) during the build.
/// This method will be called every second to adjust the IO limit.
/// </remarks>
public void AdjustIOParallelDegree(PerformanceCollector.MachinePerfInfo machinePerfInfo)
{
if (!IsDraining || !m_config.AdaptiveIO)
{
return;
}
var ioDispatcher = m_queuesByKind[DispatcherKind.IO];
int currentMax = ioDispatcher.MaxParallelDegree;
// If numRunning is closer to the currentMax, consider increasing the limit based on the resource usage
// We should not only look at the disk usage activity but also CPU, RAM as well because the pips running on the IO queue consume CPU and RAM resources as well.
// TODO: Instead of looking at all disk usages, just look at the ones which are associated with the build files (both inputs and outputs).
bool hasLowGlobalUsage = machinePerfInfo.CpuUsagePercentage < 90 &&
machinePerfInfo.RamUsagePercentage < 90 &&
machinePerfInfo.DiskUsagePercentages.All(a => a < 90);
bool numRunningIsNearMax = ioDispatcher.NumRunning > currentMax * 0.8;
if (numRunningIsNearMax && (currentMax < m_config.MaxIO) && hasLowGlobalUsage)
{
// The new currentMax will be the midpoint of currentMax and absoluteMax.
currentMax = (m_config.MaxIO + currentMax + 1) / 2;
ioDispatcher.AdjustParallelDegree(currentMax);
TriggerDispatcher(); // After increasing the limit, trigger the dispatcher so that we can start new tasks.
}
else if (machinePerfInfo.DiskUsagePercentages.Any(a => a > 95))
{
// If any of the disks usage is higher than 95%, then decrease the limit.
// TODO: Should we look at the CPU or MEM usage as well to decrease the limit?
currentMax = (currentMax + 1) / 2;
ioDispatcher.AdjustParallelDegree(currentMax);
}
// TODO: Right now, we only care about the disk active time. We should also take the avg disk queue length into account.
// Average disk queue length is a product of disk transfers/sec (response X I/O) and average disk sec/transfer.
}
/// <inheritdoc/>
public void AdjustIOParallelDegree(PerformanceCollector.MachinePerfInfo machinePerfInfo)
{
m_innerQueue.AdjustIOParallelDegree(machinePerfInfo);
}
