/// <summary> /// Checks that /// - a sideband file for a given process exists /// - the sideband file is not corrupt (its checksum checks out) /// - the process metadata recorded in the sideband file matches the metadata expected for this process /// </summary> private bool ValidateSidebandFileForProcess(Process process) { var sidebandFile = GetSidebandFile(process); if (!FileUtilities.FileExistsNoFollow(sidebandFile)) { return(failed(SidebandIntegrityCheckFailReason.FileNotFound)); } using (var reader = new SidebandReader(sidebandFile)) { if (!reader.ReadHeader(ignoreChecksum: false)) { return(failed(SidebandIntegrityCheckFailReason.ChecksumMismatch)); } var metadata = reader.ReadMetadata(); var expected = PipExecutor.CreateSidebandMetadata(Scheduler, process); if (!metadata.Equals(expected)) { return(failed(SidebandIntegrityCheckFailReason.MetadataMismatch, $"Expected: {expected}. Actual: {metadata}")); } return(true); } bool failed(SidebandIntegrityCheckFailReason reason, string details = "") { Logger.Log.SidebandIntegrityCheckForProcessFailed(LoggingContext, process.FormattedSemiStableHash, sidebandFile, reason.ToString(), details); return(false); } }
/// <inheritdoc /> public override async Task <ExecutionResult> ExecuteProcessAsync(ProcessRunnablePip processRunnable) { using (OnPipExecutionStarted(processRunnable)) { RunningPipExecutorProcesses.TryAdd(processRunnable.PipId, Unit.Void); var environment = processRunnable.Environment; var process = processRunnable.Process; var operationContext = processRunnable.OperationContext; ContentFingerprint?fingerprint = processRunnable.CacheResult?.Fingerprint; Transition(processRunnable.PipId, WorkerPipState.Executing); ExecutionResult executionResult = await PipExecutor.ExecuteProcessAsync( operationContext, environment, environment.State.GetScope(process), process, fingerprint, processIdListener : UpdateCurrentlyRunningPipsCount, expectedMemoryCounters : GetExpectedMemoryCounters(processRunnable)); processRunnable.SetExecutionResult(executionResult); Unit ignore; RunningPipExecutorProcesses.TryRemove(processRunnable.PipId, out ignore); return(executionResult); } }
/// <summary> /// Checks that /// - a sideband file for a given process exists /// - the sideband file is not corrupt (its checksum checks out) /// - the process metadata recorded in the sideband file matches the metadata expected for this process /// Returns true on success and the sideband state in the out parameter /// </summary> private bool TryGetAndValidateSidebandStateForProcess(Process process, out IReadOnlyCollection <AbsolutePath> paths) { var sidebandFile = GetSidebandFile(process); paths = null; if (!FileUtilities.FileExistsNoFollow(sidebandFile)) { return(failed(SidebandIntegrityCheckFailReason.FileNotFound)); } using (var reader = new SidebandReader(sidebandFile)) { if (!reader.ReadHeader(ignoreChecksum: false)) { return(failed(SidebandIntegrityCheckFailReason.ChecksumMismatch)); } var metadata = reader.ReadMetadata(); var expected = PipExecutor.CreateSidebandMetadata(Scheduler, process); if (!metadata.Equals(expected)) { return(failed(SidebandIntegrityCheckFailReason.MetadataMismatch, $"Expected: {expected}. Actual: {metadata}")); } paths = reader.ReadRecordedPaths().Select(p => AbsolutePath.Create(Context.PathTable, p)).ToHashSet(); return(true); } bool failed(SidebandIntegrityCheckFailReason reason, string details = "") { Logger.Log.SidebandIntegrityCheckForProcessFailed(LoggingContext, process.FormattedSemiStableHash, sidebandFile, reason.ToString(), details); return(false); } }
/// <inheritdoc /> public override async Task <PipResultStatus> MaterializeInputsAsync(RunnablePip runnablePip) { using (OnPipExecutionStarted(runnablePip)) { var result = await PipExecutor.MaterializeInputsAsync(runnablePip.OperationContext, runnablePip.Environment, runnablePip.Pip); return(result); } }
/// <inheritdoc /> public override async Task <RunnableFromCacheResult> CacheLookupAsync( ProcessRunnablePip runnablePip, PipExecutionState.PipScopeState state, CacheableProcess cacheableProcess) { using (OnPipExecutionStarted(runnablePip)) { return(await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess)); } }
/// <inheritdoc /> public override async Task <(RunnableFromCacheResult, PipResultStatus)> CacheLookupAsync( ProcessRunnablePip runnablePip, PipExecutionState.PipScopeState state, CacheableProcess cacheableProcess) { using (OnPipExecutionStarted(runnablePip)) { var cacheResult = await PipExecutor.TryCheckProcessRunnableFromCacheAsync(runnablePip, state, cacheableProcess); return(ValueTuple.Create(cacheResult, cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded)); } }
/// <inheritdoc /> public override async Task <PipResultStatus> MaterializeOutputsAsync(RunnablePip runnablePip) { // Need to create separate operation context since there may be concurrent operations on representing executions on remote workers using (var operationContext = runnablePip.OperationContext.StartAsyncOperation(OperationKind.PassThrough)) using (OnPipExecutionStarted(runnablePip, operationContext)) { var cachingInfo = runnablePip.ExecutionResult?.TwoPhaseCachingInfo; Task cachingInfoAvailableCompletion = Unit.VoidTask; PipResultStatus result = await PipExecutor.MaterializeOutputsAsync(operationContext, runnablePip.Environment, runnablePip.Pip); return(result); } }
/// <inheritdoc /> public override async Task <ExecutionResult> PostProcessAsync(ProcessRunnablePip runnablePip) { using (OnPipExecutionStarted(runnablePip)) { var pipScope = runnablePip.Environment.State.GetScope(runnablePip.Process); var cacheableProcess = runnablePip.CacheableProcess ?? pipScope.GetCacheableProcess(runnablePip.Process, runnablePip.Environment); return(await PipExecutor.PostProcessExecutionAsync( operationContext : runnablePip.OperationContext, environment : runnablePip.Environment, state : pipScope, cacheableProcess : cacheableProcess, processExecutionResult : runnablePip.ExecutionResult)); } }
/// <inheritdoc /> public override async Task <PipResult> ExecuteIpcAsync(RunnablePip runnablePip) { using (OnPipExecutionStarted(runnablePip)) { var environment = runnablePip.Environment; var ipcPip = (IpcPip)runnablePip.Pip; var operationContext = runnablePip.OperationContext; Transition(runnablePip.PipId, WorkerPipState.Executing); var executionResult = await PipExecutor.ExecuteIpcAsync(operationContext, environment, ipcPip); runnablePip.SetExecutionResult(executionResult); return(RunnablePip.CreatePipResultFromExecutionResult(runnablePip.StartTime, executionResult)); } }
/// <inheritdoc /> public override async Task <RunnableFromCacheResult> CacheLookupAsync( ProcessRunnablePip runnablePip, PipExecutionState.PipScopeState state, CacheableProcess cacheableProcess) { ExecutionResult result = await ExecutePipRemotely(runnablePip); if (result.Result.IndicatesFailure()) { return(null); } return(PipExecutor.TryConvertToRunnableFromCacheResult( runnablePip.OperationContext, runnablePip.Environment, state, cacheableProcess, result)); }
private async Task <PipResult> StartServiceOrShutdownServiceAsync( IPipExecutionEnvironment environment, PipId pipId, TaskSourceSlim <bool> serviceLaunchCompletion, bool isStartup) { // Ensure task does not block current thread await Task.Yield(); var loggingContext = m_executePhaseLoggingContext; try { var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId); if (isStartup) { Logger.Log.ScheduleServicePipStarting( loggingContext, serviceProcess.GetDescription(m_context)); Interlocked.Increment(ref m_runningServicesCount); } using (var operationContext = m_operationTracker.StartOperation( isStartup ? PipExecutorCounter.ExecuteServiceDuration : PipExecutorCounter.ExecuteServiceShutdownDuration, pipId, PipType.Process, loggingContext)) { var serviceStartTask = PipExecutor.ExecuteServiceStartOrShutdownAsync( #pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call // Bug #1155822: There is a race condition where service start/shutdown can // cause crash in the operation tracker because the parent operation is already completed // this is not fully understood, but the tracking of details of services operations is not // important so this disables it OperationContext.CreateUntracked(loggingContext), #pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call environment, serviceProcess, processId => { serviceLaunchCompletion.TrySetResult(isStartup); }); using ( operationContext.StartAsyncOperation( isStartup ? PipExecutorCounter.ExecuteServiceStartupLaunchDuration : PipExecutorCounter.ExecuteServiceShutdownLaunchDuration)) { Analysis.IgnoreResult( await Task.WhenAny(serviceLaunchCompletion.Task, serviceStartTask), justification: "Task<Task> doesn't contain any data." ); } var result = await serviceStartTask; return(PipResult.CreateWithPointPerformanceInfo(result.Result)); } } finally { // Attempt to set the result to false to indicate service startup failure. // This will not succeed if the service is already started // and the result is set to true. if (serviceLaunchCompletion.TrySetResult(false)) { var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId); Logger.Log.ScheduleServiceTerminatedBeforeStartupWasSignaled( loggingContext, serviceProcess.GetDescription(m_context)); } } }