/// <summary>
/// Gets the estimated memory counters for the process
/// </summary>
public ProcessMemoryCounters GetExpectedMemoryCounters(ProcessRunnablePip runnableProcess)
{
if (TotalRamMb == null || TotalCommitMb == null)
{
return(ProcessMemoryCounters.CreateFromMb(0, 0, 0, 0));
}
if (runnableProcess.ExpectedMemoryCounters.HasValue)
{
// If there is already an expected memory counters for the process,
// it means that we retry the process with another worker due to
// several reasons including stopped worker, memory exhaustion.
// That's why, we should reuse the expected memory counters that
// are updated with recent data from last execution.
return(runnableProcess.ExpectedMemoryCounters.Value);
}
// If there is a historic perf data, use it.
if (runnableProcess.HistoricPerfData != null)
{
return(runnableProcess.HistoricPerfData.Value.MemoryCounters);
}
// If there is no historic perf data, use the defaults for the worker.
return(ProcessMemoryCounters.CreateFromMb(
peakWorkingSetMb: DefaultWorkingSetMbPerProcess,
averageWorkingSetMb: DefaultWorkingSetMbPerProcess,
peakCommitSizeMb: DefaultCommitSizeMbPerProcess,
averageCommitSizeMb: DefaultCommitSizeMbPerProcess));
}
private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess)
{
if (TotalRamMb == null || TotalCommitMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true)
{
// Not tracking working set or commit memory
return(null);
}
if (!m_ramSemaphoreNameId.IsValid)
{
m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName);
m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit);
}
if (!m_commitSemaphoreNameId.IsValid)
{
m_commitSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(CommitSemaphoreName);
m_commitSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_commitSemaphoreNameId, ProcessExtensions.PercentageResourceLimit);
}
var expectedMemoryCounters = GetExpectedMemoryCounters(runnableProcess);
return(new ProcessSemaphoreInfo[]
{
ProcessExtensions.GetNormalizedPercentageResource(
m_ramSemaphoreNameId,
usage: expectedMemoryCounters.PeakWorkingSetMb,
total: TotalRamMb.Value),
ProcessExtensions.GetNormalizedPercentageResource(
m_commitSemaphoreNameId,
usage: expectedMemoryCounters.PeakCommitUsageMb,
total: TotalCommitMb.Value),
});
}
/// <summary>
/// Gets the estimated memory counters for the process
/// </summary>
public ProcessMemoryCounters GetExpectedMemoryCounters(ProcessRunnablePip runnableProcess, bool isCancelledDueToResourceExhaustion = false)
{
if (TotalRamMb == null || TotalCommitMb == null)
{
return(ProcessMemoryCounters.CreateFromMb(0, 0, 0));
}
if (runnableProcess.ExpectedMemoryCounters == null)
{
return(ProcessMemoryCounters.CreateFromMb(
peakVirtualMemoryUsageMb: DefaultMemoryUsageMbPerProcess,
peakWorkingSetMb: DefaultMemoryUsageMbPerProcess,
peakCommitUsageMb: DefaultCommitUsageMbPerProcess));
}
var expectedMemoryCounters = runnableProcess.ExpectedMemoryCounters.Value;
// If the process is cancelled due to resource exhaustion, multiply the memory usage by 1.25; otherwise use the historic memory.
double multiplier = isCancelledDueToResourceExhaustion ? 1.25 : 1;
return(ProcessMemoryCounters.CreateFromMb(
peakVirtualMemoryUsageMb: (int)(expectedMemoryCounters.PeakVirtualMemoryUsageMb * multiplier),
peakWorkingSetMb: (int)(expectedMemoryCounters.PeakWorkingSetMb * multiplier),
peakCommitUsageMb: (int)(expectedMemoryCounters.PeakCommitUsageMb * multiplier)));
}
/// <inheritdoc />
public override async Task <ExecutionResult> ExecuteProcessAsync(ProcessRunnablePip processRunnable)
{
using (OnPipExecutionStarted(processRunnable))
{
RunningPipExecutorProcesses.TryAdd(processRunnable.PipId, Unit.Void);
var environment = processRunnable.Environment;
var process = processRunnable.Process;
var operationContext = processRunnable.OperationContext;
ContentFingerprint?fingerprint = processRunnable.CacheResult?.Fingerprint;
Transition(processRunnable.PipId, WorkerPipState.Executing);
ExecutionResult executionResult = await PipExecutor.ExecuteProcessAsync(
operationContext,
environment,
environment.State.GetScope(process),
process,
fingerprint,
processIdListener : UpdateCurrentlyRunningPipsCount,
expectedMemoryCounters : GetExpectedMemoryCounters(processRunnable));
processRunnable.SetExecutionResult(executionResult);
Unit ignore;
RunningPipExecutorProcesses.TryRemove(processRunnable.PipId, out ignore);
return(executionResult);
}
}
/// <summary>
/// Performs a cache lookup for the process on the worker
/// </summary>
public virtual Task <(RunnableFromCacheResult, PipResultStatus)> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
Contract.Requires(runnablePip.Step == PipExecutionStep.CacheLookup);
throw Contract.AssertFailure(I($"CacheLookupAsync is not supported for worker {Name}"));
}
/// <inheritdoc />
public override async Task <RunnableFromCacheResult> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
using (OnPipExecutionStarted(runnablePip))
{
return(await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess));
}
}
private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess, ProcessMemoryCounters expectedMemoryCounters)
{
using (var semaphoreInfoListWrapper = s_semaphoreInfoListPool.GetInstance())
{
var semaphores = semaphoreInfoListWrapper.Instance;
if (runnableProcess.Process.RequiresAdmin &&
runnableProcess.Environment.Configuration.Sandbox.AdminRequiredProcessExecutionMode.ExecuteExternalVm() &&
runnableProcess.Environment.Configuration.Sandbox.VmConcurrencyLimit > 0)
{
semaphores.Add(new ProcessSemaphoreInfo(
runnableProcess.Environment.Context.StringTable.AddString(PipInVmSemaphoreName),
value: 1,
limit: runnableProcess.Environment.Configuration.Sandbox.VmConcurrencyLimit));
}
if (TotalRamMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true)
{
// Not tracking working set
return(semaphores.ToArray());
}
if (!m_ramSemaphoreNameId.IsValid)
{
m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName);
m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit);
}
bool enableLessAggresiveMemoryProjection = runnableProcess.Environment.Configuration.Schedule.EnableLessAggresiveMemoryProjection;
var ramSemaphoreInfo = ProcessExtensions.GetNormalizedPercentageResource(
m_ramSemaphoreNameId,
usage: enableLessAggresiveMemoryProjection ? expectedMemoryCounters.AverageWorkingSetMb : expectedMemoryCounters.PeakWorkingSetMb,
total: TotalRamMb.Value);
semaphores.Add(ramSemaphoreInfo);
if (runnableProcess.Environment.Configuration.Schedule.EnableHistoricCommitMemoryProjection)
{
if (!m_commitSemaphoreNameId.IsValid)
{
m_commitSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(CommitSemaphoreName);
m_commitSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_commitSemaphoreNameId, ProcessExtensions.PercentageResourceLimit);
}
var commitSemaphoreInfo = ProcessExtensions.GetNormalizedPercentageResource(
m_commitSemaphoreNameId,
usage: enableLessAggresiveMemoryProjection ? expectedMemoryCounters.AverageCommitSizeMb : expectedMemoryCounters.PeakCommitSizeMb,
total: TotalCommitMb.Value);
semaphores.Add(commitSemaphoreInfo);
}
return(semaphores.ToArray());
}
}
/// <inheritdoc />
public override async Task <(RunnableFromCacheResult, PipResultStatus)> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
using (OnPipExecutionStarted(runnablePip))
{
var cacheResult = await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess);
return(ValueTuple.Create(cacheResult, cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded));
}
}
private void VerifyMultiplePips(string[] arr)
{
for (var i = 0; i < arr.Length; i++)
{
var jsonObj = JsonConvert.DeserializeObject <PerProcessPipPerformanceInformation>(arr[i]);
int index = jsonObj.IOReadMb;
ProcessRunnablePip runnablePip = (ProcessRunnablePip)m_runnablePips[index];
PerProcessPipPerformanceInformation pipInfo = GeneratePipInfoWithRunnablePipAndIndex(ref runnablePip, index);
Assert.Equal(PerProcessPipPerformanceInformationStore.SerializePipPerfInfo(pipInfo), arr[i]);
}
}
/// <summary>
/// Gets the estimated RAM usage for the process
/// </summary>
public int GetExpectedRamUsageMb(ProcessRunnablePip runnableProcess)
{
if (TotalMemoryMb == null)
{
return(0);
}
var defaultExpectedRamUsage = TotalMemoryMb.Value / Math.Max(TotalProcessSlots, Environment.ProcessorCount);
var expectedUsage = runnableProcess.ExpectedRamUsageMb != null
? (int)(runnableProcess.ExpectedRamUsageMb * 1.25) // Multiply by 1.25 to give some slack
: defaultExpectedRamUsage;
return(expectedUsage);
}
/// <inheritdoc />
public override async Task <ExecutionResult> PostProcessAsync(ProcessRunnablePip runnablePip)
{
using (OnPipExecutionStarted(runnablePip))
{
var pipScope = runnablePip.Environment.State.GetScope(runnablePip.Process);
var cacheableProcess = runnablePip.CacheableProcess ?? pipScope.GetCacheableProcess(runnablePip.Process, runnablePip.Environment);
return(await PipExecutor.PostProcessExecutionAsync(
operationContext : runnablePip.OperationContext,
environment : runnablePip.Environment,
state : pipScope,
cacheableProcess : cacheableProcess,
processExecutionResult : runnablePip.ExecutionResult));
}
}
/// <inheritdoc />
public override async Task <RunnableFromCacheResult> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
ExecutionResult result = await ExecutePipRemotely(runnablePip);
if (result.Result.IndicatesFailure())
{
return(null);
}
return(PipExecutor.TryConvertToRunnableFromCacheResult(
runnablePip.OperationContext,
runnablePip.Environment,
state,
cacheableProcess,
result));
}
/// <summary>
/// Reports the outputs of process execution after post process to distinguish between produced outputs and
/// outputs from cache when assigning affinity for IPC pips
/// </summary>
public void ReportProcessExecutionOutputs(ProcessRunnablePip runnableProcess, ExecutionResult executionResult)
{
Contract.Assert(runnableProcess.Step == PipExecutionStep.PostProcess);
if (executionResult.Converged || runnableProcess.Process.IsStartOrShutdownKind)
{
// Converged results are cached so they are not considered for as execution outputs
// Service start/shutdown pip outputs are not considered for IPC pip affinity
return;
}
foreach (var directoryOutput in executionResult.DirectoryOutputs)
{
m_executedProcessOutputs.Add(directoryOutput.directoryArtifact);
}
foreach (var output in executionResult.OutputContent)
{
m_executedProcessOutputs.Add(output.fileArtifact);
}
}
public bool TryAcquireCacheLookup(ProcessRunnablePip runnablePip, bool force)
{
using (EarlyReleaseLock.AcquireReadLock())
{
if (!IsAvailable)
{
return(false);
}
if (force)
{
Interlocked.Increment(ref m_acquiredCacheLookupSlots);
runnablePip.AcquiredResourceWorker = this;
return(true);
}
// Atomically acquire a cache lookup slot, being sure to not increase above the limit
while (true)
{
int acquiredCacheLookupSlots = AcquiredCacheLookupSlots;
if (acquiredCacheLookupSlots < TotalCacheLookupSlots)
{
if (Interlocked.CompareExchange(ref m_acquiredCacheLookupSlots, acquiredCacheLookupSlots + 1, acquiredCacheLookupSlots) == acquiredCacheLookupSlots)
{
OnWorkerResourcesChanged(WorkerResource.AvailableCacheLookupSlots, increased: false);
runnablePip.AcquiredResourceWorker = this;
return(true);
}
else
{
// Failed to update value. Retry.
}
}
else
{
return(false);
}
}
}
}
/// <summary>
/// Gets the estimated memory counters for the process
/// </summary>
public ProcessMemoryCounters GetExpectedMemoryCounters(ProcessRunnablePip runnableProcess)
{
if (TotalRamMb == null || TotalCommitMb == null)
{
return(ProcessMemoryCounters.CreateFromMb(0, 0, 0));
}
if (runnableProcess.ExpectedMemoryCounters == null)
{
return(ProcessMemoryCounters.CreateFromMb(
peakVirtualMemoryUsageMb: DefaultMemoryUsageMbPerProcess,
peakWorkingSetMb: DefaultMemoryUsageMbPerProcess,
peakCommitUsageMb: DefaultCommitUsageMbPerProcess));
}
var expectedMemoryCounters = runnableProcess.ExpectedMemoryCounters.Value;
// 5% more to give some slack
return(ProcessMemoryCounters.CreateFromMb(
peakVirtualMemoryUsageMb: (int)(expectedMemoryCounters.PeakVirtualMemoryUsageMb * 1.05),
peakWorkingSetMb: (int)(expectedMemoryCounters.PeakWorkingSetMb * 1.05),
peakCommitUsageMb: (int)(expectedMemoryCounters.PeakCommitUsageMb * 1.05)));
}
private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess)
{
if (TotalMemoryMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true)
{
// Not tracking total memory
return(null);
}
int expectedUsage = GetExpectedRamUsageMb(runnableProcess);
if (!m_ramSemaphoreNameId.IsValid)
{
m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName);
m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit);
}
return(new ProcessSemaphoreInfo[]
{
ProcessExtensions.GetNormalizedPercentageResource(
m_ramSemaphoreNameId,
usage: expectedUsage,
total: TotalMemoryMb.Value),
});
}
/// <inheritdoc />
public override async Task <ExecutionResult> PostProcessAsync(ProcessRunnablePip runnablePip)
{
var result = await ExecutePipRemotely(runnablePip);
return(result);
}
/// <inheritdoc />
public override async Task <ExecutionResult> ExecuteProcessAsync(ProcessRunnablePip processRunnable)
{
if (processRunnable.IsLight)
{
// Light process does not need remoting, and does not need a slot.
return(await base.ExecuteProcessAsync(processRunnable));
}
// Assume that the process is going to run locally (prefer local).
int runLocalCount = Interlocked.Increment(ref m_currentRunLocalCount);
// Run local criteria:
// - the process is forced to run locally, i.e., run location == ProcessRunLocation.Local, or
// - the process requires an admin privilege, or
// - the user specifies "MustRunLocal" tag, and the process has that tag, or
// - the user specifies "CanRunRemote" tag, and the process does not have that tag, or
// - the number of processes that are running locally (or are going to run locally)
// is below a threshold.
//
// When the process requires an admin privilege, then most likely it has to run in a VM hosted
// by the local worker. Thus, the parallelism of running such process should be the same as running
// the process on the local worker.
if (DisableRemoting ||
processRunnable.RunLocation == ProcessRunLocation.Local ||
ProcessRequiresAdminPrivilege(processRunnable.Process) ||
(ExistTags(m_processMustRunLocalTags) && HasTag(processRunnable.Process, m_processMustRunLocalTags)) ||
(ExistTags(m_processCanRunRemoteTags) && !HasTag(processRunnable.Process, m_processCanRunRemoteTags)) ||
runLocalCount <= m_remotingThreshold)
{
await m_localExecutionSemaphore.WaitAsync();
Interlocked.Increment(ref m_totalRunLocally);
if (processRunnable.ExecutionResult?.RetryInfo?.RetryReason == RetryReason.RemoteFallback)
{
Interlocked.Increment(ref m_totalRemoteFallbackRetryLocally);
}
try
{
return(await base.ExecuteProcessAsync(processRunnable));
}
finally
{
m_localExecutionSemaphore.Release();
Interlocked.Decrement(ref m_currentRunLocalCount);
}
}
else
{
// Retract the assumption that the process is going to run locally.
Interlocked.Decrement(ref m_currentRunLocalCount);
processRunnable.RunLocation = ProcessRunLocation.Remote;
Interlocked.Increment(ref m_currentRunRemoteCount);
Interlocked.Increment(ref m_totalRunRemote);
try
{
return(await base.ExecuteProcessAsync(processRunnable));
}
finally
{
Interlocked.Decrement(ref m_currentRunRemoteCount);
}
}
}
private PerProcessPipPerformanceInformation GeneratePipInfoWithRunnablePipAndIndex(ref ProcessRunnablePip runnablePip, int index)
{
return(new PerProcessPipPerformanceInformation(ref runnablePip, index, index, index, index));
}
/// <summary>
/// Executes PostProcess on the worker
/// </summary>
public virtual Task <ExecutionResult> PostProcessAsync(ProcessRunnablePip runnablePip)
{
Contract.Requires(runnablePip.Step == PipExecutionStep.PostProcess);
throw Contract.AssertFailure(I($"{nameof(PostProcessAsync)} is not supported for worker {Name}"));
}
