private void SerializeExecutionResult(ExtendedPipCompletionData completionData)
{
using (var pooledWriter = m_writerPool.GetInstance())
{
var writer = pooledWriter.Instance;
PipId pipId = completionData.PipId;
m_resultSerializer.Serialize(writer, completionData.ExecutionResult);
// TODO: ToArray is expensive here. Think about alternatives.
var dataByte = ((MemoryStream)writer.BaseStream).ToArray();
completionData.SerializedData.ResultBlob = new ArraySegment <byte>(dataByte);
m_workerPipStateManager.Transition(pipId, WorkerPipState.Reporting);
m_environment.Counters.AddToCounter(m_pipTable.GetPipType(pipId) == PipType.Process ? PipExecutorCounter.ProcessExecutionResultSize : PipExecutorCounter.IpcExecutionResultSize, dataByte.Length);
}
}
public void ReportResult(ExtendedPipCompletionData pipCompletion)
{
try
{
using (DistributionServices.Counters.StartStopwatch(DistributionCounter.WorkerServiceResultSerializationDuration))
{
SerializeExecutionResult(pipCompletion);
}
ReadyToSendResultList.Add(pipCompletion);
}
catch (InvalidOperationException)
{
// ReadyToSendResultList is already marked as completed (due to cancellation or early exit)
// No need to report the other results as the build already failed or the orchestrator doesn't
// care about the results (this is the case with early release).
}
}
internal void ExecutePipsCore(PipBuildRequest request)
{
var reportInputsResult = TryReportInputs(request.Hashes);
for (int i = 0; i < request.Pips.Count; i++)
{
SinglePipBuildRequest pipBuildRequest = request.Pips[i];
// Start the pip. Handle the case of a retry - the pip may be already started by a previous call.
if (m_handledBuildRequests.Add(pipBuildRequest.SequenceNumber))
{
var pipId = new PipId(pipBuildRequest.PipIdValue);
var pip = m_pipTable.HydratePip(pipId, PipQueryContext.HandlePipStepOnWorker);
m_pendingBuildRequests[pipId] = pipBuildRequest;
var pipCompletionData = new ExtendedPipCompletionData(new PipCompletionData()
{
PipIdValue = pipId.Value, Step = pipBuildRequest.Step
})
{
SemiStableHash = m_pipTable.GetPipSemiStableHash(pipId)
};
m_pendingPipCompletions[pipId] = pipCompletionData;
HandlePipStepAsync(pip, pipCompletionData, pipBuildRequest, reportInputsResult).Forget((ex) =>
{
Scheduler.Tracing.Logger.Log.HandlePipStepOnWorkerFailed(
m_appLoggingContext,
pip.GetDescription(m_environment.Context),
ex.ToString());
// HandlePipStep might throw an exception after we remove pipCompletionData from m_pendingPipCompletions.
// That's why, we check whether the pipCompletionData still exists there.
if (m_pendingPipCompletions.ContainsKey(pip.PipId))
{
ReportResult(
pip,
ExecutionResult.GetFailureNotRunResult(m_appLoggingContext),
(PipExecutionStep)pipBuildRequest.Step);
}
});
}
}
}
public void ReportResult(ExtendedPipCompletionData pipCompletion)
{
try
{
using (DistributionService.Counters.StartStopwatch(DistributionCounter.WorkerServiceResultSerializationDuration))
{
m_resultSerializer.SerializeExecutionResult(pipCompletion);
DistributionService.Counters.AddToCounter(pipCompletion.PipType == PipType.Process ? DistributionCounter.ProcessExecutionResultSize : DistributionCounter.IpcExecutionResultSize, pipCompletion.SerializedData.ResultBlob.Count);
m_notificationManager.ExecutionService.Transition(pipCompletion.PipId, WorkerPipState.Reporting);
}
ReadyToSendResultList.Add(pipCompletion);
}
catch (InvalidOperationException)
{
// ReadyToSendResultList is already marked as completed (due to cancellation or early exit)
// No need to report the other results as the build already failed or the orchestrator doesn't
// care about the results (this is the case with early release).
}
}
private async Task HandlePipStepAsync(SinglePipBuildRequest pipBuildRequest, Possible <Unit> reportInputsResult)
{
// Do not block the caller.
await Task.Yield();
var pipId = new PipId(pipBuildRequest.PipIdValue);
var pip = m_pipTable.HydratePip(pipId, PipQueryContext.LoggingPipFailedOnWorker);
var pipType = pip.PipType;
var step = (PipExecutionStep)pipBuildRequest.Step;
if (!(pipType == PipType.Process || pipType == PipType.Ipc || step == PipExecutionStep.MaterializeOutputs))
{
throw Contract.AssertFailure(I($"Workers can only execute process or IPC pips for steps other than MaterializeOutputs: Step={step}, PipId={pipId}, Pip={pip.GetDescription(m_environment.Context)}"));
}
m_pendingBuildRequests[pipId] = pipBuildRequest;
var pipCompletionData = new ExtendedPipCompletionData(new PipCompletionData()
{
PipIdValue = pipId.Value, Step = (int)step
})
{
SemiStableHash = m_pipTable.GetPipSemiStableHash(pipId)
};
m_pendingPipCompletions[pipId] = pipCompletionData;
using (var operationContext = m_operationTracker.StartOperation(PipExecutorCounter.WorkerServiceHandlePipStepDuration, pipId, pipType, m_appLoggingContext))
using (operationContext.StartOperation(step))
{
var pipInfo = new PipInfo(pip, m_environment.Context);
if (!reportInputsResult.Succeeded)
{
// Could not report inputs due to input mismatch. Fail the pip
BuildXL.Scheduler.Tracing.Logger.Log.PipMaterializeDependenciesFailureUnrelatedToCache(
m_appLoggingContext,
pipInfo.Description,
ArtifactMaterializationResult.VerifySourceFilesFailed.ToString(),
reportInputsResult.Failure.DescribeIncludingInnerFailures());
ReportResult(
operationContext,
pip,
ExecutionResult.GetFailureNotRunResult(m_appLoggingContext),
step);
return;
}
if (step == PipExecutionStep.CacheLookup)
{
// Directory dependencies need to be registered for cache lookup.
// For process execution, the input materialization guarantees the registration.
m_environment.State.FileContentManager.RegisterDirectoryDependencies(pipInfo.UnderlyingPip);
}
m_workerPipStateManager.Transition(pipId, WorkerPipState.Queued);
m_scheduler.HandlePipRequest(pipId, m_workerRunnablePipObserver, step, pipBuildRequest.Priority);
// Track how much time the request spent queued
using (var op = operationContext.StartOperation(PipExecutorCounter.WorkerServiceQueuedPipStepDuration))
{
await pipCompletionData.StepExecutionStarted.Task;
pipCompletionData.SerializedData.QueueTicks = op.Duration.Value.Ticks;
}
ExecutionResult executionResult;
// Track how much time the request spent executing
using (operationContext.StartOperation(PipExecutorCounter.WorkerServiceExecutePipStepDuration))
{
executionResult = await pipCompletionData.StepExecutionCompleted.Task;
}
ReportResult(
operationContext,
pip,
executionResult,
step);
}
}
internal void ReportResult(ExtendedPipCompletionData pipCompletion) => m_pipResultListener.ReportResult(pipCompletion);
