private void OnWorkerResourcesChanged(Worker worker, WorkerResource resourceKind, bool increased)
{
if (increased)
{
TogglePauseChooseWorkerQueue(pause: false);
}
}
/// <summary>
/// Try acquire given resources on the worker. This must be called from a thread-safe context to prevent race conditions.
/// </summary>
internal bool TryAcquire(RunnablePip runnablePip, out WorkerResource?limitingResource, double loadFactor = 1)
{
Contract.Requires(runnablePip.PipType == PipType.Ipc || runnablePip.PipType == PipType.Process);
Contract.Ensures(Contract.Result <bool>() == (limitingResource == null), "Must set a limiting resource when resources cannot be acquired");
using (EarlyReleaseLock.AcquireReadLock())
{
if (!IsAvailable)
{
limitingResource = WorkerResource.Status;
return(false);
}
if (runnablePip.PipType == PipType.Ipc)
{
Interlocked.Increment(ref m_acquiredIpcSlots);
runnablePip.AcquiredResourceWorker = this;
limitingResource = null;
return(true);
}
if (IsLocal)
{
// Local worker does not use load factor as it may be down throttle by the
// scheduler in order to handle remote requests.
loadFactor = 1;
}
var processRunnablePip = runnablePip as ProcessRunnablePip;
// If a process has a weight higher than the total number of process slots, still allow it to run as long as there are no other
// processes running (the number of acquired slots is 0)
if (AcquiredProcessSlots != 0 && AcquiredProcessSlots + processRunnablePip.Weight > (EffectiveTotalProcessSlots * loadFactor))
{
limitingResource = WorkerResource.AvailableProcessSlots;
return(false);
}
StringId limitingResourceName = StringId.Invalid;
if (processRunnablePip.TryAcquireResources(m_workerSemaphores, GetAdditionalResourceInfo(processRunnablePip), out limitingResourceName))
{
Interlocked.Add(ref m_acquiredProcessSlots, processRunnablePip.Weight);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: false);
runnablePip.AcquiredResourceWorker = this;
limitingResource = null;
return(true);
}
limitingResource = limitingResourceName == m_ramSemaphoreNameId
? WorkerResource.AvailableMemoryMb
: WorkerResource.CreateSemaphoreResource(limitingResourceName.ToString(runnablePip.Environment.Context.StringTable));
return(false);
}
}
/// <summary>
/// Try acquire given resources on the worker
/// </summary>
internal bool TryAcquire(RunnablePip runnablePip, out WorkerResource?limitingResource, double loadFactor = 1)
{
Contract.Ensures(Contract.Result <bool>() == (limitingResource == null), "Must set a limiting resource when resources cannot be acquired");
if (!IsAvailable)
{
limitingResource = WorkerResource.Status;
return(false);
}
if (runnablePip.PipType == PipType.Ipc)
{
Interlocked.Increment(ref m_acquiredIpcSlots);
runnablePip.AcquiredResourceWorker = this;
limitingResource = null;
return(true);
}
if (IsLocal)
{
// Local worker does not use load factor as it may be down throttle by the
// scheduler in order to handle remote requests.
loadFactor = 1;
}
if (AcquiredProcessSlots >= (EffectiveTotalProcessSlots * loadFactor))
{
limitingResource = WorkerResource.AvailableProcessSlots;
return(false);
}
var processRunnablePip = runnablePip as ProcessRunnablePip;
StringId limitingResourceName = StringId.Invalid;
if (processRunnablePip != null &&
processRunnablePip.TryAcquireResources(m_workerSemaphores, GetAdditionalResourceInfo(processRunnablePip), out limitingResourceName))
{
Interlocked.Increment(ref m_acquiredProcessSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: false);
runnablePip.AcquiredResourceWorker = this;
limitingResource = null;
return(true);
}
limitingResource = limitingResourceName == m_ramSemaphoreNameId
? WorkerResource.AvailableMemoryMb
: WorkerResource.CreateSemaphoreResource(limitingResourceName.ToString(runnablePip.Environment.Context.StringTable));
return(false);
}
/// <summary>
/// Try acquire given resources on the worker. This must be called from a thread-safe context to prevent race conditions.
/// </summary>
internal bool TryAcquire(RunnablePip runnablePip, out WorkerResource?limitingResource, double loadFactor = 1, bool moduleAffinityEnabled = false)
{
Contract.Requires(runnablePip.PipType == PipType.Ipc || runnablePip.PipType == PipType.Process);
Contract.Ensures(Contract.Result <bool>() == (limitingResource == null), "Must set a limiting resource when resources cannot be acquired");
using (EarlyReleaseLock.AcquireReadLock())
{
if (!IsAvailable)
{
limitingResource = WorkerResource.Status;
return(false);
}
if (runnablePip.PipType == PipType.Ipc)
{
Interlocked.Increment(ref m_acquiredLightSlots);
runnablePip.AcquiredResourceWorker = this;
limitingResource = null;
return(true);
}
if (IsLocal && !moduleAffinityEnabled)
{
// Local worker does not use load factor as it may be down throttle by the
// scheduler in order to handle remote requests.
// When ModuleAffinity is enabled, we want to oversubscribe the local worker as well
// to prevent a new worker from adding to the list.
loadFactor = 1;
}
var processRunnablePip = runnablePip as ProcessRunnablePip;
// If a process has a weight higher than the total number of process slots, still allow it to run as long as there are no other
// processes running (the number of acquired slots is 0)
// Light processes do not acquire process slots as they are not CPU-bound.
if (!processRunnablePip.Process.IsLight)
{
if (AcquiredProcessSlots != 0 && AcquiredProcessSlots + processRunnablePip.Weight > (EffectiveTotalProcessSlots * loadFactor))
{
limitingResource = MemoryResourceAvailable ? WorkerResource.AvailableProcessSlots : WorkerResource.MemoryResourceAvailable;
return(false);
}
}
if (AcquiredMaterializeInputSlots >= TotalMaterializeInputSlots)
{
limitingResource = WorkerResource.AvailableMaterializeInputSlots;
return(false);
}
StringId limitingResourceName = StringId.Invalid;
var expectedMemoryCounters = GetExpectedMemoryCounters(processRunnablePip);
if (processRunnablePip.TryAcquireResources(m_workerSemaphores, GetAdditionalResourceInfo(processRunnablePip, expectedMemoryCounters), out limitingResourceName))
{
if (processRunnablePip.Process.IsLight)
{
Interlocked.Increment(ref m_acquiredLightSlots);
}
else
{
Interlocked.Add(ref m_acquiredProcessSlots, processRunnablePip.Weight);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: false);
}
Interlocked.Add(ref m_acquiredPostProcessSlots, 1);
runnablePip.AcquiredResourceWorker = this;
processRunnablePip.ExpectedMemoryCounters = expectedMemoryCounters;
limitingResource = null;
if (runnablePip.Environment.InputsLazilyMaterialized)
{
// If inputs are lazily materialized, we need to acquire MaterializeInput slots.
// Then, we can stop ChooseWorkerCpu queue when the materialize slots are full.
// If inputs are not lazily materialized, there is no need to acquire MaterializeInput slots
// because we do not execute MaterializeInput step for those builds such as single-machine builds.
// Otherwise, the hang occurs for single machine builds where we do not lazily materialize inputs.
Interlocked.Add(ref m_acquiredMaterializeInputSlots, 1);
}
return(true);
}
if (limitingResourceName == m_ramSemaphoreNameId)
{
limitingResource = WorkerResource.AvailableMemoryMb;
}
else if (limitingResourceName == m_commitSemaphoreNameId)
{
limitingResource = WorkerResource.AvailableCommitMb;
}
else
{
limitingResource = WorkerResource.CreateSemaphoreResource(limitingResourceName.ToString(runnablePip.Environment.Context.StringTable));
}
return(false);
}
}
/// <summary>
/// Raises <see cref="ResourcesChanged"/> event.
/// </summary>
private void OnWorkerResourcesChanged(WorkerResource kind, bool increased)
{
ResourcesChanged?.Invoke(this, kind, increased);
}
