private void ReportResult(
OperationContext operationContext,
Pip pip,
ExecutionResult executionResult,
PipExecutionStep step)
{
var pipId = pip.PipId;
m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording);
if (executionResult.Result == PipResultStatus.Failed)
{
m_hasFailures = true;
}
bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion);
Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker");
pipCompletion.ExecutionResult = executionResult;
if (step == PipExecutionStep.MaterializeOutputs && m_config.FireForgetMaterializeOutput)
{
// We do not report 'MaterializeOutput' step results back to master.
Logger.Log.DistributionWorkerFinishedPipRequest(m_appLoggingContext, pipCompletion.SemiStableHash, step.ToString());
}
else
{
m_buildResults.Add(pipCompletion);
}
}
private void StartStep(RunnablePip runnablePip, PipExecutionStep step)
{
var pipId = runnablePip.PipId;
var processRunnable = runnablePip as ProcessRunnablePip;
Tracing.Logger.Log.DistributionWorkerExecutePipRequest(
runnablePip.LoggingContext,
runnablePip.Pip.SemiStableHash,
runnablePip.Description,
runnablePip.Step.AsString());
var completionData = m_pendingPipCompletions[pipId];
completionData.StepExecutionStarted.SetResult(true);
switch (step)
{
case PipExecutionStep.ExecuteProcess:
if (runnablePip.PipType == PipType.Process)
{
SinglePipBuildRequest pipBuildRequest;
bool found = m_pendingBuildRequests.TryGetValue(pipId, out pipBuildRequest);
Contract.Assert(found, "Could not find corresponding build request for executed pip on worker");
m_pendingBuildRequests[pipId] = null;
// Set the cache miss result with fingerprint so ExecuteProcess step can use it
var fingerprint = pipBuildRequest.Fingerprint.ToFingerprint();
processRunnable.SetCacheResult(RunnableFromCacheResult.CreateForMiss(new WeakContentFingerprint(fingerprint)));
processRunnable.ExpectedRamUsageMb = pipBuildRequest.ExpectedRamUsageMb;
}
break;
}
}
internal void Executed(PipExecutionStep step, TimeSpan duration)
{
lock (m_lock)
{
StepDurations[(int)step] += duration;
}
}
/// <summary>
/// Efficient toString implementation for PipExecutionStep.
/// </summary>
public static string AsString(this PipExecutionStep step)
{
switch (step)
{
case PipExecutionStep.CacheLookup:
return(nameof(PipExecutionStep.CacheLookup));
case PipExecutionStep.Cancel:
return(nameof(PipExecutionStep.Cancel));
case PipExecutionStep.CheckIncrementalSkip:
return(nameof(PipExecutionStep.CheckIncrementalSkip));
case PipExecutionStep.ChooseWorkerCacheLookup:
return(nameof(PipExecutionStep.ChooseWorkerCacheLookup));
case PipExecutionStep.ChooseWorkerCpu:
return(nameof(PipExecutionStep.ChooseWorkerCpu));
case PipExecutionStep.Done:
return(nameof(PipExecutionStep.Done));
case PipExecutionStep.ExecuteNonProcessPip:
return(nameof(PipExecutionStep.ExecuteNonProcessPip));
case PipExecutionStep.ExecuteProcess:
return(nameof(PipExecutionStep.ExecuteProcess));
case PipExecutionStep.HandleResult:
return(nameof(PipExecutionStep.HandleResult));
case PipExecutionStep.MaterializeInputs:
return(nameof(PipExecutionStep.MaterializeInputs));
case PipExecutionStep.MaterializeOutputs:
return(nameof(PipExecutionStep.MaterializeOutputs));
case PipExecutionStep.None:
return(nameof(PipExecutionStep.None));
case PipExecutionStep.PostProcess:
return(nameof(PipExecutionStep.PostProcess));
case PipExecutionStep.RunFromCache:
return(nameof(PipExecutionStep.RunFromCache));
case PipExecutionStep.Skip:
return(nameof(PipExecutionStep.Skip));
case PipExecutionStep.Start:
return(nameof(PipExecutionStep.Start));
case PipExecutionStep.DelayedCacheLookup:
return(nameof(PipExecutionStep.DelayedCacheLookup));
default:
throw new NotImplementedException("Unknown PipExecutionStep type: " + step);
}
}
public override void StartStep(RunnablePip runnablePip, PipExecutionStep step)
{
if (step == PipExecutionStep.PostProcess)
{
ExecutionResult executionResult;
var removed = m_processExecutionResult.TryRemove(runnablePip.PipId, out executionResult);
Contract.Assert(removed, "Execution result must be stored from ExecuteProcess step for PostProcess");
runnablePip.SetExecutionResult(executionResult);
}
m_workerService.StartStep(runnablePip, step);
}
internal void Executed(PipExecutionStep step, TimeSpan duration)
{
lock (m_lock)
{
StepDurations[(int)step] += duration;
}
if (step == PipExecutionStep.ExecuteProcess)
{
IsExecuted = true;
}
}
public override void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration)
{
if (step == PipExecutionStep.ExecuteProcess)
{
// For successful/unsuccessful results of ExecuteProcess, store so that when master calls worker for
// PostProcess it can reuse the result rather than sending it unnecessarily
// The unsuccessful results are stored as well to preserve the existing behavior where PostProcess is also done for such results.
// TODO: Should we skipped PostProcess when Process failed? In such a case then PipExecutor.ReportExecutionResultOutputContent should not be in PostProcess.
m_processExecutionResult[runnablePip.PipId] = runnablePip.ExecutionResult;
}
m_workerService.EndStep(runnablePip, step, duration);
}
/// <summary>
/// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout
/// </summary>
public static bool IncludeInTracer(this PipExecutionStep step)
{
switch (step)
{
case PipExecutionStep.ExecuteProcess:
case PipExecutionStep.MaterializeInputs:
case PipExecutionStep.CacheLookup:
return(true);
default:
return(false);
}
}
/// <summary>
/// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout
/// </summary>
public static bool IncludeInRunningTime(this PipExecutionStep step)
{
switch (step)
{
// These steps pertain to distribution and thus should not be considered for running time
// which is expected to be comparable whether the pip runs remotely or not
case PipExecutionStep.ChooseWorkerCpu:
case PipExecutionStep.ChooseWorkerCacheLookup:
return(false);
default:
return(true);
}
}
/// <summary>
/// Transition to another step
/// </summary>
public void Transition(PipExecutionStep toStep, bool force = false)
{
if (!force && !Step.CanTransitionTo(toStep))
{
Contract.Assert(false, I($"Cannot transition from {Step} to {toStep}"));
}
Step = toStep;
if (toStep == PipExecutionStep.Done)
{
End();
}
}
/// <summary>
/// Indicates if the pip execution step is mainly I/O related.
/// </summary>
public static bool IsIORelated(this PipExecutionStep step)
{
switch (step)
{
case PipExecutionStep.Start: // Hashing
case PipExecutionStep.CacheLookup:
case PipExecutionStep.MaterializeInputs:
case PipExecutionStep.MaterializeOutputs:
case PipExecutionStep.PostProcess:
return(true);
default:
return(false);
}
}
internal void RemoteExecuted(
uint workerId,
PipExecutionStep step,
TimeSpan remoteStepDuration,
TimeSpan remoteQueueDuration,
TimeSpan queueRequestDuration,
TimeSpan sendRequestDuration)
{
lock (m_lock)
{
Workers.Value[(int)step] = workerId;
RemoteStepDurations.Value[(int)step] += remoteStepDuration;
RemoteQueueDurations.Value[(int)step] += remoteQueueDuration;
QueueRequestDurations.Value[(int)step] += queueRequestDuration;
SendRequestDurations.Value[(int)step] += sendRequestDuration;
}
}
/// <summary>
/// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout
/// </summary>
public static bool IncludeInRunningTime(this PipExecutionStep step, IPipExecutionEnvironment environment)
{
switch (step)
{
// These steps pertain to distribution and thus should not be considered for running time
// which is expected to be comparable whether the pip runs remotely or not
case PipExecutionStep.ChooseWorkerCpu:
case PipExecutionStep.ChooseWorkerCacheLookup:
return(false);
case PipExecutionStep.MaterializeOutputs:
// If we materialize outputs in background, then do not include the duration in running time.
return(!environment.MaterializeOutputsInBackground);
default:
return(true);
}
}
private void ReportResult(
Pip pip,
ExecutionResult executionResult,
PipExecutionStep step)
{
var pipId = pip.PipId;
m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording);
if (executionResult.Result == PipResultStatus.Failed)
{
m_hasFailures = true;
}
bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion);
Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker");
pipCompletion.ExecutionResult = executionResult;
// To preserve the path set casing is an option only available for process pips
pipCompletion.PreservePathSetCasing = pip.PipType == PipType.Process ? ((Process)pip).PreservePathSetCasing : false;
if (step == PipExecutionStep.MaterializeOutputs && m_config.Distribution.FireForgetMaterializeOutput)
{
// We do not report 'MaterializeOutput' step results back to master.
Logger.Log.DistributionWorkerFinishedPipRequest(m_appLoggingContext, pipCompletion.SemiStableHash, step.ToString());
return;
}
try
{
m_buildResults.Add(pipCompletion);
}
catch (InvalidOperationException)
{
// m_buildResults is already marked as completed due to previously infrastructure errors reported (e.g., materialization errors).
// No need to report the other failed results as the build already failed
}
}
private void ReportResult(
OperationContext operationContext,
Pip pip,
ExecutionResult executionResult,
PipExecutionStep step)
{
var pipId = pip.PipId;
m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording);
if (executionResult.Result == PipResultStatus.Failed)
{
m_hasFailures = true;
}
bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion);
Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker");
pipCompletion.ExecutionResult = executionResult;
m_buildResults.Add(pipCompletion);
}
private void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration)
{
var pipId = runnablePip.PipId;
var loggingContext = runnablePip.LoggingContext;
var pip = runnablePip.Pip;
var description = runnablePip.Description;
var executionResult = runnablePip.ExecutionResult;
var completionData = m_pendingPipCompletions[pipId];
completionData.SerializedData.ExecuteStepTicks = duration.Ticks;
switch (step)
{
case PipExecutionStep.MaterializeInputs:
if (!runnablePip.Result.HasValue ||
!runnablePip.Result.Value.Status.IndicatesFailure())
{
m_workerPipStateManager.Transition(pipId, WorkerPipState.Prepped);
}
break;
case PipExecutionStep.ExecuteProcess:
case PipExecutionStep.ExecuteNonProcessPip:
executionResult.Seal();
m_workerPipStateManager.Transition(pipId, WorkerPipState.Executed);
if (!executionResult.Result.IndicatesFailure())
{
foreach (var outputContent in executionResult.OutputContent)
{
Tracing.Logger.Log.DistributionWorkerPipOutputContent(
loggingContext,
pip.SemiStableHash,
description,
outputContent.fileArtifact.Path.ToString(m_environment.Context.PathTable),
outputContent.fileInfo.Hash.ToHex());
}
}
break;
case PipExecutionStep.CacheLookup:
var runnableProcess = (ProcessRunnablePip)runnablePip;
executionResult = new ExecutionResult();
var cacheResult = runnableProcess.CacheResult;
executionResult.SetResult(
loggingContext,
status: cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded);
if (cacheResult != null)
{
executionResult.WeakFingerprint = cacheResult.WeakFingerprint;
if (cacheResult.CanRunFromCache)
{
var cacheHitData = cacheResult.GetCacheHitData();
if (m_environment.State.Cache.IsNewlyAdded(cacheHitData.PathSetHash))
{
executionResult.PathSet = cacheHitData.PathSet;
}
executionResult.PipCacheDescriptorV2Metadata = cacheHitData.Metadata;
executionResult.TwoPhaseCachingInfo = new TwoPhaseCachingInfo(
weakFingerprint: cacheResult.WeakFingerprint,
pathSetHash: cacheHitData.PathSetHash,
strongFingerprint: cacheHitData.StrongFingerprint,
// NOTE: This should not be used so we set it to default values except the metadata hash (it is used for HistoricMetadataCache).
cacheEntry: new CacheEntry(cacheHitData.MetadataHash, "unused", ArrayView <ContentHash> .Empty));
}
}
executionResult.CacheLookupPerfInfo = runnableProcess.CacheLookupPerfInfo;
executionResult.Seal();
break;
case PipExecutionStep.PostProcess:
// Execution result is already computed during ExecuteProcess.
Contract.Assert(executionResult != null);
break;
}
if (executionResult == null)
{
executionResult = new ExecutionResult();
// If no result is set, the step succeeded
executionResult.SetResult(loggingContext, runnablePip.Result?.Status ?? PipResultStatus.Succeeded);
executionResult.Seal();
}
completionData.StepExecutionCompleted.SetResult(executionResult);
}
/// <summary>
/// Release pip's resources after worker is done with the task
/// </summary>
public void ReleaseResources(RunnablePip runnablePip, PipExecutionStep nextStep)
{
Contract.Assert(runnablePip.AcquiredResourceWorker == this);
var stepCompleted = runnablePip.Step;
bool isCancelledOrFailed = nextStep == PipExecutionStep.ChooseWorkerCpu || nextStep == PipExecutionStep.HandleResult;
var processRunnablePip = runnablePip as ProcessRunnablePip;
if (processRunnablePip != null)
{
switch (stepCompleted)
{
case PipExecutionStep.CacheLookup:
{
Interlocked.Decrement(ref m_acquiredCacheLookupSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableCacheLookupSlots, increased: true);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
break;
}
case PipExecutionStep.MaterializeInputs:
{
Interlocked.Decrement(ref m_acquiredMaterializeInputSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableMaterializeInputSlots, increased: true);
if (isCancelledOrFailed)
{
releaseExecuteProcessSlots();
releasePostProcessSlots();
}
break;
}
case PipExecutionStep.ExecuteProcess:
{
releaseExecuteProcessSlots();
if (isCancelledOrFailed)
{
releasePostProcessSlots();
}
break;
}
case PipExecutionStep.PostProcess:
{
releasePostProcessSlots();
break;
}
}
}
if (runnablePip.PipType == PipType.Ipc)
{
if (stepCompleted == PipExecutionStep.ExecuteNonProcessPip || isCancelledOrFailed)
{
Interlocked.Decrement(ref m_acquiredLightSlots);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
}
}
if (AcquiredSlots == 0 && Status == WorkerNodeStatus.Stopping)
{
DrainCompletion.TrySetResult(true);
}
void releaseExecuteProcessSlots()
{
Contract.Assert(processRunnablePip.Resources.HasValue);
if (processRunnablePip.Process.IsLight)
{
Interlocked.Decrement(ref m_acquiredLightSlots);
}
else
{
Interlocked.Add(ref m_acquiredProcessSlots, -processRunnablePip.Weight);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: true);
}
var resources = processRunnablePip.Resources.Value;
m_workerSemaphores.ReleaseResources(resources);
}
void releasePostProcessSlots()
{
Interlocked.Decrement(ref m_acquiredPostProcessSlots);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
}
}
/// <summary>
/// Check whether it is valid to transition from one step to another pip execution step.
/// </summary>
public static bool CanTransitionTo(this PipExecutionStep fromStep, PipExecutionStep toStep)
{
if (toStep == PipExecutionStep.Cancel)
{
// You can transition to Cancel step from any step except None, Start, Done
return(fromStep != PipExecutionStep.None || fromStep != PipExecutionStep.Start || fromStep != PipExecutionStep.Done);
}
switch (fromStep)
{
case PipExecutionStep.None:
return(toStep == PipExecutionStep.Start);
case PipExecutionStep.Start:
return(toStep == PipExecutionStep.Skip ||
toStep == PipExecutionStep.CheckIncrementalSkip ||
toStep == PipExecutionStep.ExecuteNonProcessPip ||
toStep == PipExecutionStep.ChooseWorkerCpu ||
toStep == PipExecutionStep.HandleResult);
case PipExecutionStep.CheckIncrementalSkip:
return(toStep == PipExecutionStep.ExecuteNonProcessPip ||
toStep == PipExecutionStep.ChooseWorkerCacheLookup ||
toStep == PipExecutionStep.HandleResult);
case PipExecutionStep.ChooseWorkerCacheLookup:
return(toStep == PipExecutionStep.CacheLookup ||
toStep == PipExecutionStep.ChooseWorkerCacheLookup);
case PipExecutionStep.CacheLookup:
return(toStep == PipExecutionStep.RunFromCache ||
toStep == PipExecutionStep.ChooseWorkerCpu ||
toStep == PipExecutionStep.HandleResult ||
toStep == PipExecutionStep.Skip);
case PipExecutionStep.ChooseWorkerCpu:
return(toStep == PipExecutionStep.MaterializeInputs ||
toStep == PipExecutionStep.ExecuteNonProcessPip ||
toStep == PipExecutionStep.ExecuteProcess ||
toStep == PipExecutionStep.ChooseWorkerCpu);
case PipExecutionStep.ExecuteProcess:
return(toStep == PipExecutionStep.PostProcess ||
toStep == PipExecutionStep.ChooseWorkerCpu || /* retry */
toStep == PipExecutionStep.RunFromCache); /* determinism probe - deploy outputs from cache after executing process to enable downstream determinism */
case PipExecutionStep.ExecuteNonProcessPip:
case PipExecutionStep.PostProcess:
// May need to materialize outputs due to cache convergence
case PipExecutionStep.RunFromCache:
return(toStep == PipExecutionStep.MaterializeOutputs || /* lazy materialization off */
toStep == PipExecutionStep.HandleResult); /* lazy materialization on */
case PipExecutionStep.MaterializeInputs:
return(toStep == PipExecutionStep.ExecuteProcess ||
toStep == PipExecutionStep.ExecuteNonProcessPip ||
toStep == PipExecutionStep.HandleResult); /* failure */
case PipExecutionStep.Cancel:
case PipExecutionStep.Skip:
return(toStep == PipExecutionStep.HandleResult);
case PipExecutionStep.MaterializeOutputs:
return(toStep == PipExecutionStep.HandleResult ||
toStep == PipExecutionStep.Done); /* background output materialization */
case PipExecutionStep.HandleResult:
return(toStep == PipExecutionStep.Done ||
toStep == PipExecutionStep.MaterializeOutputs); /* background output materialization */
case PipExecutionStep.Done:
return(false);
default:
throw Contract.AssertFailure("Invalid step:" + fromStep);
}
}
/// <summary>
/// Gets pip execution step performance events by key
/// </summary>
public IEnumerable <PipExecutionStepPerformanceReportedEvent> GetPipExecutionStepPerformanceEventByKey(uint pipID, PipExecutionStep pipExecutionStep = 0, uint workerID = 0)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var eventKey = new EventKey
{
EventTypeID = ExecutionEventId.PipExecutionStepPerformanceReported,
WorkerID = workerID,
PipId = pipID,
PipExecutionStepPerformanceKey = pipExecutionStep
};
return(GetEventsByKey(eventKey).Cast <PipExecutionStepPerformanceReportedEvent>());
}
/// <summary>
/// Notification that the given runnable pip has started a particular step
/// </summary>
public virtual void StartStep(RunnablePip runnablePip, PipExecutionStep step)
{
}
/// <summary>
/// Notification that the given runnable pip has ended the pip step with the duration taken by that step
/// </summary>
public virtual void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration)
{
}
private void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration)
{
var pipId = runnablePip.PipId;
var loggingContext = runnablePip.LoggingContext;
var pip = runnablePip.Pip;
var description = runnablePip.Description;
var executionResult = runnablePip.ExecutionResult;
var completionData = m_pendingPipCompletions[pipId];
completionData.SerializedData.ExecuteStepTicks = duration.Ticks;
switch (step)
{
case PipExecutionStep.MaterializeInputs:
if (!runnablePip.Result.HasValue ||
!runnablePip.Result.Value.Status.IndicatesFailure())
{
m_workerPipStateManager.Transition(pipId, WorkerPipState.Prepped);
}
break;
case PipExecutionStep.ExecuteProcess:
case PipExecutionStep.ExecuteNonProcessPip:
executionResult.Seal();
m_workerPipStateManager.Transition(pipId, WorkerPipState.Executed);
if (!executionResult.Result.IndicatesFailure())
{
foreach (var outputContent in executionResult.OutputContent)
{
Tracing.Logger.Log.DistributionWorkerPipOutputContent(
loggingContext,
pip.SemiStableHash,
description,
outputContent.fileArtifact.Path.ToString(m_environment.Context.PathTable),
outputContent.fileInfo.Hash.ToHex());
}
}
break;
case PipExecutionStep.CacheLookup:
var runnableProcess = (ProcessRunnablePip)runnablePip;
executionResult = new ExecutionResult();
var cacheResult = runnableProcess.CacheResult;
executionResult.SetResult(
loggingContext,
status: cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded);
if (cacheResult != null)
{
executionResult.PopulateCacheInfoFromCacheResult(cacheResult);
}
executionResult.CacheLookupPerfInfo = runnableProcess.CacheLookupPerfInfo;
executionResult.Seal();
break;
case PipExecutionStep.PostProcess:
// Execution result is already computed during ExecuteProcess.
Contract.Assert(executionResult != null);
break;
}
if (executionResult == null)
{
executionResult = new ExecutionResult();
// If no result is set, the step succeeded
executionResult.SetResult(loggingContext, runnablePip.Result?.Status ?? PipResultStatus.Succeeded);
executionResult.Seal();
}
completionData.StepExecutionCompleted.SetResult(executionResult);
}
/// <inheritdoc />
public IEnumerable <PipExecutionStepPerformanceReportedEvent> GetPipExecutionStepPerformanceEventByKey(uint pipID, PipExecutionStep pipExecutionStep = 0, uint?workerID = null)
{
Contract.Requires(m_accessor != null, "XldbDataStore is not initialized");
var eventKey = new EventKey
{
EventTypeID = ExecutionEventId.PipExecutionStepPerformanceReported,
WorkerID = workerID ?? s_workerIDDefaultValue,
FileRewriteCount = s_fileRewriteCountDefaultValue,
PipId = pipID,
PipExecutionStepPerformanceKey = pipExecutionStep
};
return(GetEventsByKey(eventKey).Cast <PipExecutionStepPerformanceReportedEvent>());
}
