private void ReportResult( OperationContext operationContext, Pip pip, ExecutionResult executionResult, PipExecutionStep step) { var pipId = pip.PipId; m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording); if (executionResult.Result == PipResultStatus.Failed) { m_hasFailures = true; } bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion); Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker"); pipCompletion.ExecutionResult = executionResult; if (step == PipExecutionStep.MaterializeOutputs && m_config.FireForgetMaterializeOutput) { // We do not report 'MaterializeOutput' step results back to master. Logger.Log.DistributionWorkerFinishedPipRequest(m_appLoggingContext, pipCompletion.SemiStableHash, step.ToString()); } else { m_buildResults.Add(pipCompletion); } }
private void StartStep(RunnablePip runnablePip, PipExecutionStep step) { var pipId = runnablePip.PipId; var processRunnable = runnablePip as ProcessRunnablePip; Tracing.Logger.Log.DistributionWorkerExecutePipRequest( runnablePip.LoggingContext, runnablePip.Pip.SemiStableHash, runnablePip.Description, runnablePip.Step.AsString()); var completionData = m_pendingPipCompletions[pipId]; completionData.StepExecutionStarted.SetResult(true); switch (step) { case PipExecutionStep.ExecuteProcess: if (runnablePip.PipType == PipType.Process) { SinglePipBuildRequest pipBuildRequest; bool found = m_pendingBuildRequests.TryGetValue(pipId, out pipBuildRequest); Contract.Assert(found, "Could not find corresponding build request for executed pip on worker"); m_pendingBuildRequests[pipId] = null; // Set the cache miss result with fingerprint so ExecuteProcess step can use it var fingerprint = pipBuildRequest.Fingerprint.ToFingerprint(); processRunnable.SetCacheResult(RunnableFromCacheResult.CreateForMiss(new WeakContentFingerprint(fingerprint))); processRunnable.ExpectedRamUsageMb = pipBuildRequest.ExpectedRamUsageMb; } break; } }
internal void Executed(PipExecutionStep step, TimeSpan duration) { lock (m_lock) { StepDurations[(int)step] += duration; } }
/// <summary> /// Efficient toString implementation for PipExecutionStep. /// </summary> public static string AsString(this PipExecutionStep step) { switch (step) { case PipExecutionStep.CacheLookup: return(nameof(PipExecutionStep.CacheLookup)); case PipExecutionStep.Cancel: return(nameof(PipExecutionStep.Cancel)); case PipExecutionStep.CheckIncrementalSkip: return(nameof(PipExecutionStep.CheckIncrementalSkip)); case PipExecutionStep.ChooseWorkerCacheLookup: return(nameof(PipExecutionStep.ChooseWorkerCacheLookup)); case PipExecutionStep.ChooseWorkerCpu: return(nameof(PipExecutionStep.ChooseWorkerCpu)); case PipExecutionStep.Done: return(nameof(PipExecutionStep.Done)); case PipExecutionStep.ExecuteNonProcessPip: return(nameof(PipExecutionStep.ExecuteNonProcessPip)); case PipExecutionStep.ExecuteProcess: return(nameof(PipExecutionStep.ExecuteProcess)); case PipExecutionStep.HandleResult: return(nameof(PipExecutionStep.HandleResult)); case PipExecutionStep.MaterializeInputs: return(nameof(PipExecutionStep.MaterializeInputs)); case PipExecutionStep.MaterializeOutputs: return(nameof(PipExecutionStep.MaterializeOutputs)); case PipExecutionStep.None: return(nameof(PipExecutionStep.None)); case PipExecutionStep.PostProcess: return(nameof(PipExecutionStep.PostProcess)); case PipExecutionStep.RunFromCache: return(nameof(PipExecutionStep.RunFromCache)); case PipExecutionStep.Skip: return(nameof(PipExecutionStep.Skip)); case PipExecutionStep.Start: return(nameof(PipExecutionStep.Start)); case PipExecutionStep.DelayedCacheLookup: return(nameof(PipExecutionStep.DelayedCacheLookup)); default: throw new NotImplementedException("Unknown PipExecutionStep type: " + step); } }
public override void StartStep(RunnablePip runnablePip, PipExecutionStep step) { if (step == PipExecutionStep.PostProcess) { ExecutionResult executionResult; var removed = m_processExecutionResult.TryRemove(runnablePip.PipId, out executionResult); Contract.Assert(removed, "Execution result must be stored from ExecuteProcess step for PostProcess"); runnablePip.SetExecutionResult(executionResult); } m_workerService.StartStep(runnablePip, step); }
internal void Executed(PipExecutionStep step, TimeSpan duration) { lock (m_lock) { StepDurations[(int)step] += duration; } if (step == PipExecutionStep.ExecuteProcess) { IsExecuted = true; } }
public override void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration) { if (step == PipExecutionStep.ExecuteProcess) { // For successful/unsuccessful results of ExecuteProcess, store so that when master calls worker for // PostProcess it can reuse the result rather than sending it unnecessarily // The unsuccessful results are stored as well to preserve the existing behavior where PostProcess is also done for such results. // TODO: Should we skipped PostProcess when Process failed? In such a case then PipExecutor.ReportExecutionResultOutputContent should not be in PostProcess. m_processExecutionResult[runnablePip.PipId] = runnablePip.ExecutionResult; } m_workerService.EndStep(runnablePip, step, duration); }
/// <summary> /// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout /// </summary> public static bool IncludeInTracer(this PipExecutionStep step) { switch (step) { case PipExecutionStep.ExecuteProcess: case PipExecutionStep.MaterializeInputs: case PipExecutionStep.CacheLookup: return(true); default: return(false); } }
/// <summary> /// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout /// </summary> public static bool IncludeInRunningTime(this PipExecutionStep step) { switch (step) { // These steps pertain to distribution and thus should not be considered for running time // which is expected to be comparable whether the pip runs remotely or not case PipExecutionStep.ChooseWorkerCpu: case PipExecutionStep.ChooseWorkerCacheLookup: return(false); default: return(true); } }
/// <summary> /// Transition to another step /// </summary> public void Transition(PipExecutionStep toStep, bool force = false) { if (!force && !Step.CanTransitionTo(toStep)) { Contract.Assert(false, I($"Cannot transition from {Step} to {toStep}")); } Step = toStep; if (toStep == PipExecutionStep.Done) { End(); } }
/// <summary> /// Indicates if the pip execution step is mainly I/O related. /// </summary> public static bool IsIORelated(this PipExecutionStep step) { switch (step) { case PipExecutionStep.Start: // Hashing case PipExecutionStep.CacheLookup: case PipExecutionStep.MaterializeInputs: case PipExecutionStep.MaterializeOutputs: case PipExecutionStep.PostProcess: return(true); default: return(false); } }
internal void RemoteExecuted( uint workerId, PipExecutionStep step, TimeSpan remoteStepDuration, TimeSpan remoteQueueDuration, TimeSpan queueRequestDuration, TimeSpan sendRequestDuration) { lock (m_lock) { Workers.Value[(int)step] = workerId; RemoteStepDurations.Value[(int)step] += remoteStepDuration; RemoteQueueDurations.Value[(int)step] += remoteQueueDuration; QueueRequestDurations.Value[(int)step] += queueRequestDuration; SendRequestDurations.Value[(int)step] += sendRequestDuration; } }
/// <summary> /// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout /// </summary> public static bool IncludeInRunningTime(this PipExecutionStep step, IPipExecutionEnvironment environment) { switch (step) { // These steps pertain to distribution and thus should not be considered for running time // which is expected to be comparable whether the pip runs remotely or not case PipExecutionStep.ChooseWorkerCpu: case PipExecutionStep.ChooseWorkerCacheLookup: return(false); case PipExecutionStep.MaterializeOutputs: // If we materialize outputs in background, then do not include the duration in running time. return(!environment.MaterializeOutputsInBackground); default: return(true); } }
private void ReportResult( Pip pip, ExecutionResult executionResult, PipExecutionStep step) { var pipId = pip.PipId; m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording); if (executionResult.Result == PipResultStatus.Failed) { m_hasFailures = true; } bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion); Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker"); pipCompletion.ExecutionResult = executionResult; // To preserve the path set casing is an option only available for process pips pipCompletion.PreservePathSetCasing = pip.PipType == PipType.Process ? ((Process)pip).PreservePathSetCasing : false; if (step == PipExecutionStep.MaterializeOutputs && m_config.Distribution.FireForgetMaterializeOutput) { // We do not report 'MaterializeOutput' step results back to master. Logger.Log.DistributionWorkerFinishedPipRequest(m_appLoggingContext, pipCompletion.SemiStableHash, step.ToString()); return; } try { m_buildResults.Add(pipCompletion); } catch (InvalidOperationException) { // m_buildResults is already marked as completed due to previously infrastructure errors reported (e.g., materialization errors). // No need to report the other failed results as the build already failed } }
private void ReportResult( OperationContext operationContext, Pip pip, ExecutionResult executionResult, PipExecutionStep step) { var pipId = pip.PipId; m_workerPipStateManager.Transition(pipId, WorkerPipState.Recording); if (executionResult.Result == PipResultStatus.Failed) { m_hasFailures = true; } bool found = m_pendingPipCompletions.TryRemove(pipId, out var pipCompletion); Contract.Assert(found, "Could not find corresponding build completion data for executed pip on worker"); pipCompletion.ExecutionResult = executionResult; m_buildResults.Add(pipCompletion); }
private void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration) { var pipId = runnablePip.PipId; var loggingContext = runnablePip.LoggingContext; var pip = runnablePip.Pip; var description = runnablePip.Description; var executionResult = runnablePip.ExecutionResult; var completionData = m_pendingPipCompletions[pipId]; completionData.SerializedData.ExecuteStepTicks = duration.Ticks; switch (step) { case PipExecutionStep.MaterializeInputs: if (!runnablePip.Result.HasValue || !runnablePip.Result.Value.Status.IndicatesFailure()) { m_workerPipStateManager.Transition(pipId, WorkerPipState.Prepped); } break; case PipExecutionStep.ExecuteProcess: case PipExecutionStep.ExecuteNonProcessPip: executionResult.Seal(); m_workerPipStateManager.Transition(pipId, WorkerPipState.Executed); if (!executionResult.Result.IndicatesFailure()) { foreach (var outputContent in executionResult.OutputContent) { Tracing.Logger.Log.DistributionWorkerPipOutputContent( loggingContext, pip.SemiStableHash, description, outputContent.fileArtifact.Path.ToString(m_environment.Context.PathTable), outputContent.fileInfo.Hash.ToHex()); } } break; case PipExecutionStep.CacheLookup: var runnableProcess = (ProcessRunnablePip)runnablePip; executionResult = new ExecutionResult(); var cacheResult = runnableProcess.CacheResult; executionResult.SetResult( loggingContext, status: cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded); if (cacheResult != null) { executionResult.WeakFingerprint = cacheResult.WeakFingerprint; if (cacheResult.CanRunFromCache) { var cacheHitData = cacheResult.GetCacheHitData(); if (m_environment.State.Cache.IsNewlyAdded(cacheHitData.PathSetHash)) { executionResult.PathSet = cacheHitData.PathSet; } executionResult.PipCacheDescriptorV2Metadata = cacheHitData.Metadata; executionResult.TwoPhaseCachingInfo = new TwoPhaseCachingInfo( weakFingerprint: cacheResult.WeakFingerprint, pathSetHash: cacheHitData.PathSetHash, strongFingerprint: cacheHitData.StrongFingerprint, // NOTE: This should not be used so we set it to default values except the metadata hash (it is used for HistoricMetadataCache). cacheEntry: new CacheEntry(cacheHitData.MetadataHash, "unused", ArrayView <ContentHash> .Empty)); } } executionResult.CacheLookupPerfInfo = runnableProcess.CacheLookupPerfInfo; executionResult.Seal(); break; case PipExecutionStep.PostProcess: // Execution result is already computed during ExecuteProcess. Contract.Assert(executionResult != null); break; } if (executionResult == null) { executionResult = new ExecutionResult(); // If no result is set, the step succeeded executionResult.SetResult(loggingContext, runnablePip.Result?.Status ?? PipResultStatus.Succeeded); executionResult.Seal(); } completionData.StepExecutionCompleted.SetResult(executionResult); }
/// <summary> /// Release pip's resources after worker is done with the task /// </summary> public void ReleaseResources(RunnablePip runnablePip, PipExecutionStep nextStep) { Contract.Assert(runnablePip.AcquiredResourceWorker == this); var stepCompleted = runnablePip.Step; bool isCancelledOrFailed = nextStep == PipExecutionStep.ChooseWorkerCpu || nextStep == PipExecutionStep.HandleResult; var processRunnablePip = runnablePip as ProcessRunnablePip; if (processRunnablePip != null) { switch (stepCompleted) { case PipExecutionStep.CacheLookup: { Interlocked.Decrement(ref m_acquiredCacheLookupSlots); OnWorkerResourcesChanged(WorkerResource.AvailableCacheLookupSlots, increased: true); runnablePip.SetWorker(null); runnablePip.AcquiredResourceWorker = null; break; } case PipExecutionStep.MaterializeInputs: { Interlocked.Decrement(ref m_acquiredMaterializeInputSlots); OnWorkerResourcesChanged(WorkerResource.AvailableMaterializeInputSlots, increased: true); if (isCancelledOrFailed) { releaseExecuteProcessSlots(); releasePostProcessSlots(); } break; } case PipExecutionStep.ExecuteProcess: { releaseExecuteProcessSlots(); if (isCancelledOrFailed) { releasePostProcessSlots(); } break; } case PipExecutionStep.PostProcess: { releasePostProcessSlots(); break; } } } if (runnablePip.PipType == PipType.Ipc) { if (stepCompleted == PipExecutionStep.ExecuteNonProcessPip || isCancelledOrFailed) { Interlocked.Decrement(ref m_acquiredLightSlots); runnablePip.SetWorker(null); runnablePip.AcquiredResourceWorker = null; } } if (AcquiredSlots == 0 && Status == WorkerNodeStatus.Stopping) { DrainCompletion.TrySetResult(true); } void releaseExecuteProcessSlots() { Contract.Assert(processRunnablePip.Resources.HasValue); if (processRunnablePip.Process.IsLight) { Interlocked.Decrement(ref m_acquiredLightSlots); } else { Interlocked.Add(ref m_acquiredProcessSlots, -processRunnablePip.Weight); OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: true); } var resources = processRunnablePip.Resources.Value; m_workerSemaphores.ReleaseResources(resources); } void releasePostProcessSlots() { Interlocked.Decrement(ref m_acquiredPostProcessSlots); runnablePip.SetWorker(null); runnablePip.AcquiredResourceWorker = null; } }
/// <summary> /// Check whether it is valid to transition from one step to another pip execution step. /// </summary> public static bool CanTransitionTo(this PipExecutionStep fromStep, PipExecutionStep toStep) { if (toStep == PipExecutionStep.Cancel) { // You can transition to Cancel step from any step except None, Start, Done return(fromStep != PipExecutionStep.None || fromStep != PipExecutionStep.Start || fromStep != PipExecutionStep.Done); } switch (fromStep) { case PipExecutionStep.None: return(toStep == PipExecutionStep.Start); case PipExecutionStep.Start: return(toStep == PipExecutionStep.Skip || toStep == PipExecutionStep.CheckIncrementalSkip || toStep == PipExecutionStep.ExecuteNonProcessPip || toStep == PipExecutionStep.ChooseWorkerCpu || toStep == PipExecutionStep.HandleResult); case PipExecutionStep.CheckIncrementalSkip: return(toStep == PipExecutionStep.ExecuteNonProcessPip || toStep == PipExecutionStep.ChooseWorkerCacheLookup || toStep == PipExecutionStep.HandleResult); case PipExecutionStep.ChooseWorkerCacheLookup: return(toStep == PipExecutionStep.CacheLookup || toStep == PipExecutionStep.ChooseWorkerCacheLookup); case PipExecutionStep.CacheLookup: return(toStep == PipExecutionStep.RunFromCache || toStep == PipExecutionStep.ChooseWorkerCpu || toStep == PipExecutionStep.HandleResult || toStep == PipExecutionStep.Skip); case PipExecutionStep.ChooseWorkerCpu: return(toStep == PipExecutionStep.MaterializeInputs || toStep == PipExecutionStep.ExecuteNonProcessPip || toStep == PipExecutionStep.ExecuteProcess || toStep == PipExecutionStep.ChooseWorkerCpu); case PipExecutionStep.ExecuteProcess: return(toStep == PipExecutionStep.PostProcess || toStep == PipExecutionStep.ChooseWorkerCpu || /* retry */ toStep == PipExecutionStep.RunFromCache); /* determinism probe - deploy outputs from cache after executing process to enable downstream determinism */ case PipExecutionStep.ExecuteNonProcessPip: case PipExecutionStep.PostProcess: // May need to materialize outputs due to cache convergence case PipExecutionStep.RunFromCache: return(toStep == PipExecutionStep.MaterializeOutputs || /* lazy materialization off */ toStep == PipExecutionStep.HandleResult); /* lazy materialization on */ case PipExecutionStep.MaterializeInputs: return(toStep == PipExecutionStep.ExecuteProcess || toStep == PipExecutionStep.ExecuteNonProcessPip || toStep == PipExecutionStep.HandleResult); /* failure */ case PipExecutionStep.Cancel: case PipExecutionStep.Skip: return(toStep == PipExecutionStep.HandleResult); case PipExecutionStep.MaterializeOutputs: return(toStep == PipExecutionStep.HandleResult || toStep == PipExecutionStep.Done); /* background output materialization */ case PipExecutionStep.HandleResult: return(toStep == PipExecutionStep.Done || toStep == PipExecutionStep.MaterializeOutputs); /* background output materialization */ case PipExecutionStep.Done: return(false); default: throw Contract.AssertFailure("Invalid step:" + fromStep); } }
/// <summary> /// Gets pip execution step performance events by key /// </summary> public IEnumerable <PipExecutionStepPerformanceReportedEvent> GetPipExecutionStepPerformanceEventByKey(uint pipID, PipExecutionStep pipExecutionStep = 0, uint workerID = 0) { Contract.Requires(Accessor != null, "XldbDataStore is not initialized"); var eventKey = new EventKey { EventTypeID = ExecutionEventId.PipExecutionStepPerformanceReported, WorkerID = workerID, PipId = pipID, PipExecutionStepPerformanceKey = pipExecutionStep }; return(GetEventsByKey(eventKey).Cast <PipExecutionStepPerformanceReportedEvent>()); }
/// <summary> /// Notification that the given runnable pip has started a particular step /// </summary> public virtual void StartStep(RunnablePip runnablePip, PipExecutionStep step) { }
/// <summary> /// Notification that the given runnable pip has ended the pip step with the duration taken by that step /// </summary> public virtual void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration) { }
private void EndStep(RunnablePip runnablePip, PipExecutionStep step, TimeSpan duration) { var pipId = runnablePip.PipId; var loggingContext = runnablePip.LoggingContext; var pip = runnablePip.Pip; var description = runnablePip.Description; var executionResult = runnablePip.ExecutionResult; var completionData = m_pendingPipCompletions[pipId]; completionData.SerializedData.ExecuteStepTicks = duration.Ticks; switch (step) { case PipExecutionStep.MaterializeInputs: if (!runnablePip.Result.HasValue || !runnablePip.Result.Value.Status.IndicatesFailure()) { m_workerPipStateManager.Transition(pipId, WorkerPipState.Prepped); } break; case PipExecutionStep.ExecuteProcess: case PipExecutionStep.ExecuteNonProcessPip: executionResult.Seal(); m_workerPipStateManager.Transition(pipId, WorkerPipState.Executed); if (!executionResult.Result.IndicatesFailure()) { foreach (var outputContent in executionResult.OutputContent) { Tracing.Logger.Log.DistributionWorkerPipOutputContent( loggingContext, pip.SemiStableHash, description, outputContent.fileArtifact.Path.ToString(m_environment.Context.PathTable), outputContent.fileInfo.Hash.ToHex()); } } break; case PipExecutionStep.CacheLookup: var runnableProcess = (ProcessRunnablePip)runnablePip; executionResult = new ExecutionResult(); var cacheResult = runnableProcess.CacheResult; executionResult.SetResult( loggingContext, status: cacheResult == null ? PipResultStatus.Failed : PipResultStatus.Succeeded); if (cacheResult != null) { executionResult.PopulateCacheInfoFromCacheResult(cacheResult); } executionResult.CacheLookupPerfInfo = runnableProcess.CacheLookupPerfInfo; executionResult.Seal(); break; case PipExecutionStep.PostProcess: // Execution result is already computed during ExecuteProcess. Contract.Assert(executionResult != null); break; } if (executionResult == null) { executionResult = new ExecutionResult(); // If no result is set, the step succeeded executionResult.SetResult(loggingContext, runnablePip.Result?.Status ?? PipResultStatus.Succeeded); executionResult.Seal(); } completionData.StepExecutionCompleted.SetResult(executionResult); }
/// <inheritdoc /> public IEnumerable <PipExecutionStepPerformanceReportedEvent> GetPipExecutionStepPerformanceEventByKey(uint pipID, PipExecutionStep pipExecutionStep = 0, uint?workerID = null) { Contract.Requires(m_accessor != null, "XldbDataStore is not initialized"); var eventKey = new EventKey { EventTypeID = ExecutionEventId.PipExecutionStepPerformanceReported, WorkerID = workerID ?? s_workerIDDefaultValue, FileRewriteCount = s_fileRewriteCountDefaultValue, PipId = pipID, PipExecutionStepPerformanceKey = pipExecutionStep }; return(GetEventsByKey(eventKey).Cast <PipExecutionStepPerformanceReportedEvent>()); }