private ExecutionResult LogAndGetNetworkFailureResult(PipCompletionTask pipCompletionTask, string errorMessage)
{
var runnablePip = pipCompletionTask.RunnablePip;
var operationContext = runnablePip.OperationContext;
if (m_exitFailure == null)
{
m_exitFailure = errorMessage;
}
ExecutionResult result;
if (runnablePip.Step == PipExecutionStep.MaterializeOutputs)
{
// Output replication failures on workers due to connection issues do not fail the distributed build.
// Setting the exit failure above ensures that the worker will fail its build and not proceed.
Logger.Log.DistributionExecutePipFailedNetworkFailureWarning(
operationContext,
runnablePip.Description,
Name,
errorMessage: errorMessage,
step: runnablePip.Step.AsString());
// Return success result
result = new ExecutionResult();
result.SetResult(operationContext, PipResultStatus.NotMaterialized);
result.Seal();
pipCompletionTask.Set(result);
}
else
{
Logger.Log.DistributionExecutePipFailedNetworkFailure(
operationContext,
runnablePip.Description,
Name,
errorMessage: errorMessage,
step: runnablePip.Step.AsString());
result = ExecutionResult.GetFailureNotRunResult(operationContext);
}
return(result);
}
public async Task SendToRemote(OperationContext operationContext, RunnablePip runnable)
{
Contract.Assert(m_workerClient != null, "Calling SendToRemote before the worker is initialized");
Contract.Assert(m_attachCompletion.IsValid, "Remote worker not started");
var attachCompletionResult = await m_attachCompletion.Task;
var environment = runnable.Environment;
var pipId = runnable.PipId;
var description = runnable.Description;
var pip = runnable.Pip;
var processRunnable = runnable as ProcessRunnablePip;
var fingerprint = processRunnable?.CacheResult?.Fingerprint ?? ContentFingerprint.Zero;
var pipCompletionTask = new PipCompletionTask(runnable.OperationContext, runnable);
m_pipCompletionTasks.Add(pipId, pipCompletionTask);
if (!attachCompletionResult)
{
FailRemotePip(
pipCompletionTask,
"Worker did not attach");
return;
}
var pipBuildRequest = new SinglePipBuildRequest
{
ActivityId = operationContext.LoggingContext.ActivityId.ToString(),
PipIdValue = pipId.Value,
Fingerprint = fingerprint.Hash.ToBondFingerprint(),
Priority = runnable.Priority,
Step = (int)runnable.Step,
ExpectedRamUsageMb = processRunnable?.ExpectedRamUsageMb,
SequenceNumber = Interlocked.Increment(ref m_nextSequenceNumber),
};
m_buildRequests.Add(ValueTuple.Create(pipCompletionTask, pipBuildRequest));
}
private void ReportRemoteExecutionStepDuration(OperationContext operationContext, RunnablePip runnablePip, PipCompletionTask completionTask)
{
var remoteStepDuration = TimeSpan.FromTicks(completionTask.ExecuteStepTicks ?? 0);
var remoteQueueDuration = TimeSpan.FromTicks(completionTask.QueueTicks ?? 0);
var queueRequestDuration = completionTask.QueueRequestDuration;
var sendRequestDuration = completionTask.SendRequestDuration;
operationContext.ReportExternalOperation(PipExecutorCounter.RemoteWorkerReportedExecutionDuration, remoteStepDuration);
runnablePip.LogRemoteExecutionStepPerformance(WorkerId, runnablePip.Step, remoteStepDuration, remoteQueueDuration, queueRequestDuration, sendRequestDuration);
}
private void FailRemotePip(PipCompletionTask pipCompletionTask, string errorMessage)
{
var result = LogAndGetNetworkFailureResult(pipCompletionTask, errorMessage);
pipCompletionTask.Set(result);
}
