private ExecutionResult LogAndGetNetworkFailureResult(PipCompletionTask pipCompletionTask, string errorMessage) { var runnablePip = pipCompletionTask.RunnablePip; var operationContext = runnablePip.OperationContext; if (m_exitFailure == null) { m_exitFailure = errorMessage; } ExecutionResult result; if (runnablePip.Step == PipExecutionStep.MaterializeOutputs) { // Output replication failures on workers due to connection issues do not fail the distributed build. // Setting the exit failure above ensures that the worker will fail its build and not proceed. Logger.Log.DistributionExecutePipFailedNetworkFailureWarning( operationContext, runnablePip.Description, Name, errorMessage: errorMessage, step: runnablePip.Step.AsString()); // Return success result result = new ExecutionResult(); result.SetResult(operationContext, PipResultStatus.NotMaterialized); result.Seal(); pipCompletionTask.Set(result); } else { Logger.Log.DistributionExecutePipFailedNetworkFailure( operationContext, runnablePip.Description, Name, errorMessage: errorMessage, step: runnablePip.Step.AsString()); result = ExecutionResult.GetFailureNotRunResult(operationContext); } return(result); }
public async Task SendToRemote(OperationContext operationContext, RunnablePip runnable) { Contract.Assert(m_workerClient != null, "Calling SendToRemote before the worker is initialized"); Contract.Assert(m_attachCompletion.IsValid, "Remote worker not started"); var attachCompletionResult = await m_attachCompletion.Task; var environment = runnable.Environment; var pipId = runnable.PipId; var description = runnable.Description; var pip = runnable.Pip; var processRunnable = runnable as ProcessRunnablePip; var fingerprint = processRunnable?.CacheResult?.Fingerprint ?? ContentFingerprint.Zero; var pipCompletionTask = new PipCompletionTask(runnable.OperationContext, runnable); m_pipCompletionTasks.Add(pipId, pipCompletionTask); if (!attachCompletionResult) { FailRemotePip( pipCompletionTask, "Worker did not attach"); return; } var pipBuildRequest = new SinglePipBuildRequest { ActivityId = operationContext.LoggingContext.ActivityId.ToString(), PipIdValue = pipId.Value, Fingerprint = fingerprint.Hash.ToBondFingerprint(), Priority = runnable.Priority, Step = (int)runnable.Step, ExpectedRamUsageMb = processRunnable?.ExpectedRamUsageMb, SequenceNumber = Interlocked.Increment(ref m_nextSequenceNumber), }; m_buildRequests.Add(ValueTuple.Create(pipCompletionTask, pipBuildRequest)); }
private void ReportRemoteExecutionStepDuration(OperationContext operationContext, RunnablePip runnablePip, PipCompletionTask completionTask) { var remoteStepDuration = TimeSpan.FromTicks(completionTask.ExecuteStepTicks ?? 0); var remoteQueueDuration = TimeSpan.FromTicks(completionTask.QueueTicks ?? 0); var queueRequestDuration = completionTask.QueueRequestDuration; var sendRequestDuration = completionTask.SendRequestDuration; operationContext.ReportExternalOperation(PipExecutorCounter.RemoteWorkerReportedExecutionDuration, remoteStepDuration); runnablePip.LogRemoteExecutionStepPerformance(WorkerId, runnablePip.Step, remoteStepDuration, remoteQueueDuration, queueRequestDuration, sendRequestDuration); }
private void FailRemotePip(PipCompletionTask pipCompletionTask, string errorMessage) { var result = LogAndGetNetworkFailureResult(pipCompletionTask, errorMessage); pipCompletionTask.Set(result); }