public static PipBuildRequest ToGrpc(this OpenBond.PipBuildRequest message)
{
var pipBuildRequest = new PipBuildRequest();
foreach (var i in message.Hashes)
{
var fileArtifactKeyedHash = new FileArtifactKeyedHash()
{
ContentHash = i.ContentHash.Data.ToByteString(),
FileName = i.FileName ?? string.Empty,
Length = i.Length,
PathString = i.PathString ?? string.Empty,
PathValue = i.PathValue,
ReparsePointType = (FileArtifactKeyedHash.Types.GrpcReparsePointType)i.ReparsePointType,
RewriteCount = i.RewriteCount,
IsSourceAffected = i.IsSourceAffected,
};
if (i.ReparsePointTarget != null)
{
fileArtifactKeyedHash.ReparsePointTarget = i.ReparsePointTarget;
}
if (i.AssociatedDirectories != null)
{
foreach (var j in i.AssociatedDirectories)
{
fileArtifactKeyedHash.AssociatedDirectories.Add(new GrpcDirectoryArtifact()
{
DirectoryPathValue = j.DirectoryPathValue,
DirectorySealId = j.DirectorySealId,
IsDirectorySharedOpaque = j.IsDirectorySharedOpaque,
});
}
}
pipBuildRequest.Hashes.Add(fileArtifactKeyedHash);
}
foreach (var i in message.Pips)
{
var singlePipBuildRequest = new SinglePipBuildRequest()
{
ActivityId = i.ActivityId,
ExpectedPeakWorkingSetMb = i.ExpectedPeakWorkingSetMb,
ExpectedAverageWorkingSetMb = i.ExpectedAverageWorkingSetMb,
ExpectedPeakCommitSizeMb = i.ExpectedPeakCommitSizeMb,
ExpectedAverageCommitSizeMb = i.ExpectedAverageCommitSizeMb,
Fingerprint = i.Fingerprint.Data.ToByteString(),
PipIdValue = i.PipIdValue,
Priority = i.Priority,
SequenceNumber = i.SequenceNumber,
Step = i.Step
};
pipBuildRequest.Pips.Add(singlePipBuildRequest);
}
return(pipBuildRequest);
}
internal void ExecutePipsCore(PipBuildRequest request)
{
var reportInputsResult = TryReportInputs(request.Hashes);
for (int i = 0; i < request.Pips.Count; i++)
{
SinglePipBuildRequest pipBuildRequest = request.Pips[i];
// Start the pip. Handle the case of a retry - the pip may be already started by a previous call.
if (m_handledBuildRequests.Add(pipBuildRequest.SequenceNumber))
{
HandlePipStepAsync(pipBuildRequest, reportInputsResult).Forget();
}
}
}
internal void ExecutePipsCore(PipBuildRequest request)
{
var reportInputsResult = TryReportInputs(request.Hashes);
for (int i = 0; i < request.Pips.Count; i++)
{
SinglePipBuildRequest pipBuildRequest = request.Pips[i];
// Start the pip. Handle the case of a retry - the pip may be already started by a previous call.
if (m_handledBuildRequests.Add(pipBuildRequest.SequenceNumber))
{
var pipId = new PipId(pipBuildRequest.PipIdValue);
var pip = m_pipTable.HydratePip(pipId, PipQueryContext.HandlePipStepOnWorker);
m_pendingBuildRequests[pipId] = pipBuildRequest;
var pipCompletionData = new ExtendedPipCompletionData(new PipCompletionData()
{
PipIdValue = pipId.Value, Step = pipBuildRequest.Step
})
{
SemiStableHash = m_pipTable.GetPipSemiStableHash(pipId)
};
m_pendingPipCompletions[pipId] = pipCompletionData;
HandlePipStepAsync(pip, pipCompletionData, pipBuildRequest, reportInputsResult).Forget((ex) =>
{
Scheduler.Tracing.Logger.Log.HandlePipStepOnWorkerFailed(
m_appLoggingContext,
pip.GetDescription(m_environment.Context),
ex.ToString());
// HandlePipStep might throw an exception after we remove pipCompletionData from m_pendingPipCompletions.
// That's why, we check whether the pipCompletionData still exists there.
if (m_pendingPipCompletions.ContainsKey(pip.PipId))
{
ReportResult(
pip,
ExecutionResult.GetFailureNotRunResult(m_appLoggingContext),
(PipExecutionStep)pipBuildRequest.Step);
}
});
}
}
}
public async Task SendToRemote(OperationContext operationContext, RunnablePip runnable)
{
Contract.Assert(m_workerClient != null, "Calling SendToRemote before the worker is initialized");
Contract.Assert(m_attachCompletion.IsValid, "Remote worker not started");
var attachCompletionResult = await m_attachCompletion.Task;
var environment = runnable.Environment;
var pipId = runnable.PipId;
var description = runnable.Description;
var pip = runnable.Pip;
var processRunnable = runnable as ProcessRunnablePip;
var fingerprint = processRunnable?.CacheResult?.Fingerprint ?? ContentFingerprint.Zero;
var pipCompletionTask = new PipCompletionTask(runnable.OperationContext, runnable);
m_pipCompletionTasks.Add(pipId, pipCompletionTask);
if (!attachCompletionResult)
{
FailRemotePip(
pipCompletionTask,
"Worker did not attach");
return;
}
var pipBuildRequest = new SinglePipBuildRequest
{
ActivityId = operationContext.LoggingContext.ActivityId.ToString(),
PipIdValue = pipId.Value,
Fingerprint = fingerprint.Hash.ToBondFingerprint(),
Priority = runnable.Priority,
Step = (int)runnable.Step,
ExpectedRamUsageMb = processRunnable?.ExpectedRamUsageMb,
SequenceNumber = Interlocked.Increment(ref m_nextSequenceNumber),
};
m_buildRequests.Add(ValueTuple.Create(pipCompletionTask, pipBuildRequest));
}
private async Task HandlePipStepAsync(SinglePipBuildRequest pipBuildRequest, Possible <Unit> reportInputsResult)
{
// Do not block the caller.
await Task.Yield();
var pipId = new PipId(pipBuildRequest.PipIdValue);
var pip = m_pipTable.HydratePip(pipId, PipQueryContext.LoggingPipFailedOnWorker);
var pipType = pip.PipType;
var step = (PipExecutionStep)pipBuildRequest.Step;
if (!(pipType == PipType.Process || pipType == PipType.Ipc || step == PipExecutionStep.MaterializeOutputs))
{
throw Contract.AssertFailure(I($"Workers can only execute process or IPC pips for steps other than MaterializeOutputs: Step={step}, PipId={pipId}, Pip={pip.GetDescription(m_environment.Context)}"));
}
m_pendingBuildRequests[pipId] = pipBuildRequest;
var pipCompletionData = new ExtendedPipCompletionData(new PipCompletionData()
{
PipIdValue = pipId.Value, Step = (int)step
})
{
SemiStableHash = m_pipTable.GetPipSemiStableHash(pipId)
};
m_pendingPipCompletions[pipId] = pipCompletionData;
using (var operationContext = m_operationTracker.StartOperation(PipExecutorCounter.WorkerServiceHandlePipStepDuration, pipId, pipType, m_appLoggingContext))
using (operationContext.StartOperation(step))
{
var pipInfo = new PipInfo(pip, m_environment.Context);
if (!reportInputsResult.Succeeded)
{
// Could not report inputs due to input mismatch. Fail the pip
BuildXL.Scheduler.Tracing.Logger.Log.PipMaterializeDependenciesFailureUnrelatedToCache(
m_appLoggingContext,
pipInfo.Description,
ArtifactMaterializationResult.VerifySourceFilesFailed.ToString(),
reportInputsResult.Failure.DescribeIncludingInnerFailures());
ReportResult(
operationContext,
pip,
ExecutionResult.GetFailureNotRunResult(m_appLoggingContext),
step);
return;
}
if (step == PipExecutionStep.CacheLookup)
{
// Directory dependencies need to be registered for cache lookup.
// For process execution, the input materialization guarantees the registration.
m_environment.State.FileContentManager.RegisterDirectoryDependencies(pipInfo.UnderlyingPip);
}
m_workerPipStateManager.Transition(pipId, WorkerPipState.Queued);
m_scheduler.HandlePipRequest(pipId, m_workerRunnablePipObserver, step, pipBuildRequest.Priority);
// Track how much time the request spent queued
using (var op = operationContext.StartOperation(PipExecutorCounter.WorkerServiceQueuedPipStepDuration))
{
await pipCompletionData.StepExecutionStarted.Task;
pipCompletionData.SerializedData.QueueTicks = op.Duration.Value.Ticks;
}
ExecutionResult executionResult;
// Track how much time the request spent executing
using (operationContext.StartOperation(PipExecutorCounter.WorkerServiceExecutePipStepDuration))
{
executionResult = await pipCompletionData.StepExecutionCompleted.Task;
}
ReportResult(
operationContext,
pip,
executionResult,
step);
}
}
