/// <summary>
/// Checks that
/// - a sideband file for a given process exists
/// - the sideband file is not corrupt (its checksum checks out)
/// - the process metadata recorded in the sideband file matches the metadata expected for this process
/// </summary>
private bool ValidateSidebandFileForProcess(Process process)
{
var sidebandFile = GetSidebandFile(process);
if (!FileUtilities.FileExistsNoFollow(sidebandFile))
{
return(failed(SidebandIntegrityCheckFailReason.FileNotFound));
}
using (var reader = new SidebandReader(sidebandFile))
{
if (!reader.ReadHeader(ignoreChecksum: false))
{
return(failed(SidebandIntegrityCheckFailReason.ChecksumMismatch));
}
var metadata = reader.ReadMetadata();
var expected = PipExecutor.CreateSidebandMetadata(Scheduler, process);
if (!metadata.Equals(expected))
{
return(failed(SidebandIntegrityCheckFailReason.MetadataMismatch, $"Expected: {expected}. Actual: {metadata}"));
}
return(true);
}
bool failed(SidebandIntegrityCheckFailReason reason, string details = "")
{
Logger.Log.SidebandIntegrityCheckForProcessFailed(LoggingContext, process.FormattedSemiStableHash, sidebandFile, reason.ToString(), details);
return(false);
}
}
/// <inheritdoc />
public override async Task <ExecutionResult> ExecuteProcessAsync(ProcessRunnablePip processRunnable)
{
using (OnPipExecutionStarted(processRunnable))
{
RunningPipExecutorProcesses.TryAdd(processRunnable.PipId, Unit.Void);
var environment = processRunnable.Environment;
var process = processRunnable.Process;
var operationContext = processRunnable.OperationContext;
ContentFingerprint?fingerprint = processRunnable.CacheResult?.Fingerprint;
Transition(processRunnable.PipId, WorkerPipState.Executing);
ExecutionResult executionResult = await PipExecutor.ExecuteProcessAsync(
operationContext,
environment,
environment.State.GetScope(process),
process,
fingerprint,
processIdListener : UpdateCurrentlyRunningPipsCount,
expectedMemoryCounters : GetExpectedMemoryCounters(processRunnable));
processRunnable.SetExecutionResult(executionResult);
Unit ignore;
RunningPipExecutorProcesses.TryRemove(processRunnable.PipId, out ignore);
return(executionResult);
}
}
/// <summary>
/// Checks that
/// - a sideband file for a given process exists
/// - the sideband file is not corrupt (its checksum checks out)
/// - the process metadata recorded in the sideband file matches the metadata expected for this process
/// Returns true on success and the sideband state in the out parameter
/// </summary>
private bool TryGetAndValidateSidebandStateForProcess(Process process, out IReadOnlyCollection <AbsolutePath> paths)
{
var sidebandFile = GetSidebandFile(process);
paths = null;
if (!FileUtilities.FileExistsNoFollow(sidebandFile))
{
return(failed(SidebandIntegrityCheckFailReason.FileNotFound));
}
using (var reader = new SidebandReader(sidebandFile))
{
if (!reader.ReadHeader(ignoreChecksum: false))
{
return(failed(SidebandIntegrityCheckFailReason.ChecksumMismatch));
}
var metadata = reader.ReadMetadata();
var expected = PipExecutor.CreateSidebandMetadata(Scheduler, process);
if (!metadata.Equals(expected))
{
return(failed(SidebandIntegrityCheckFailReason.MetadataMismatch, $"Expected: {expected}. Actual: {metadata}"));
}
paths = reader.ReadRecordedPaths().Select(p => AbsolutePath.Create(Context.PathTable, p)).ToHashSet();
return(true);
}
bool failed(SidebandIntegrityCheckFailReason reason, string details = "")
{
Logger.Log.SidebandIntegrityCheckForProcessFailed(LoggingContext, process.FormattedSemiStableHash, sidebandFile, reason.ToString(), details);
return(false);
}
}
/// <inheritdoc />
public override async Task <PipResultStatus> MaterializeInputsAsync(RunnablePip runnablePip)
{
using (OnPipExecutionStarted(runnablePip))
{
var result = await PipExecutor.MaterializeInputsAsync(runnablePip.OperationContext, runnablePip.Environment, runnablePip.Pip);
return(result);
}
}
/// <inheritdoc />
public override async Task <RunnableFromCacheResult> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
using (OnPipExecutionStarted(runnablePip))
{
return(await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess));
}
}
/// <inheritdoc />
public override async Task <(RunnableFromCacheResult, PipResultStatus)> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
using (OnPipExecutionStarted(runnablePip))
{
var cacheResult = await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess);
return(ValueTuple.Create(cacheResult, cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded));
}
}
/// <inheritdoc />
public override async Task <PipResultStatus> MaterializeOutputsAsync(RunnablePip runnablePip)
{
// Need to create separate operation context since there may be concurrent operations on representing executions on remote workers
using (var operationContext = runnablePip.OperationContext.StartAsyncOperation(OperationKind.PassThrough))
using (OnPipExecutionStarted(runnablePip, operationContext))
{
var cachingInfo = runnablePip.ExecutionResult?.TwoPhaseCachingInfo;
Task cachingInfoAvailableCompletion = Unit.VoidTask;
PipResultStatus result = await PipExecutor.MaterializeOutputsAsync(operationContext, runnablePip.Environment, runnablePip.Pip);
return(result);
}
}
/// <inheritdoc />
public override async Task <ExecutionResult> PostProcessAsync(ProcessRunnablePip runnablePip)
{
using (OnPipExecutionStarted(runnablePip))
{
var pipScope = runnablePip.Environment.State.GetScope(runnablePip.Process);
var cacheableProcess = runnablePip.CacheableProcess ?? pipScope.GetCacheableProcess(runnablePip.Process, runnablePip.Environment);
return(await PipExecutor.PostProcessExecutionAsync(
operationContext : runnablePip.OperationContext,
environment : runnablePip.Environment,
state : pipScope,
cacheableProcess : cacheableProcess,
processExecutionResult : runnablePip.ExecutionResult));
}
}
/// <inheritdoc />
public override async Task <PipResult> ExecuteIpcAsync(RunnablePip runnablePip)
{
using (OnPipExecutionStarted(runnablePip))
{
var environment = runnablePip.Environment;
var ipcPip = (IpcPip)runnablePip.Pip;
var operationContext = runnablePip.OperationContext;
Transition(runnablePip.PipId, WorkerPipState.Executing);
var executionResult = await PipExecutor.ExecuteIpcAsync(operationContext, environment, ipcPip);
runnablePip.SetExecutionResult(executionResult);
return(RunnablePip.CreatePipResultFromExecutionResult(runnablePip.StartTime, executionResult));
}
}
/// <inheritdoc />
public override async Task <RunnableFromCacheResult> CacheLookupAsync(
ProcessRunnablePip runnablePip,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess)
{
ExecutionResult result = await ExecutePipRemotely(runnablePip);
if (result.Result.IndicatesFailure())
{
return(null);
}
return(PipExecutor.TryConvertToRunnableFromCacheResult(
runnablePip.OperationContext,
runnablePip.Environment,
state,
cacheableProcess,
result));
}
private async Task <PipResult> StartServiceOrShutdownServiceAsync(
IPipExecutionEnvironment environment,
PipId pipId,
TaskSourceSlim <bool> serviceLaunchCompletion,
bool isStartup)
{
// Ensure task does not block current thread
await Task.Yield();
var loggingContext = m_executePhaseLoggingContext;
try
{
var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId);
if (isStartup)
{
Logger.Log.ScheduleServicePipStarting(
loggingContext,
serviceProcess.GetDescription(m_context));
Interlocked.Increment(ref m_runningServicesCount);
}
using (var operationContext = m_operationTracker.StartOperation(
isStartup
? PipExecutorCounter.ExecuteServiceDuration
: PipExecutorCounter.ExecuteServiceShutdownDuration,
pipId,
PipType.Process,
loggingContext))
{
var serviceStartTask = PipExecutor.ExecuteServiceStartOrShutdownAsync(
#pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call
// Bug #1155822: There is a race condition where service start/shutdown can
// cause crash in the operation tracker because the parent operation is already completed
// this is not fully understood, but the tracking of details of services operations is not
// important so this disables it
OperationContext.CreateUntracked(loggingContext),
#pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call
environment,
serviceProcess,
processId =>
{
serviceLaunchCompletion.TrySetResult(isStartup);
});
using (
operationContext.StartAsyncOperation(
isStartup
? PipExecutorCounter.ExecuteServiceStartupLaunchDuration
: PipExecutorCounter.ExecuteServiceShutdownLaunchDuration))
{
Analysis.IgnoreResult(
await Task.WhenAny(serviceLaunchCompletion.Task, serviceStartTask),
justification: "Task<Task> doesn't contain any data."
);
}
var result = await serviceStartTask;
return(PipResult.CreateWithPointPerformanceInfo(result.Result));
}
}
finally
{
// Attempt to set the result to false to indicate service startup failure.
// This will not succeed if the service is already started
// and the result is set to true.
if (serviceLaunchCompletion.TrySetResult(false))
{
var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId);
Logger.Log.ScheduleServiceTerminatedBeforeStartupWasSignaled(
loggingContext,
serviceProcess.GetDescription(m_context));
}
}
}
