private void SendNotifications(CancellationToken cancellationToken) { ExtendedPipCompletionData firstItem; while (!m_pipResultListener.ReadyToSendResultList.IsCompleted && !cancellationToken.IsCancellationRequested) { try { // Sending of notifications is driven by pip results - block until we have a new result to send firstItem = m_pipResultListener.ReadyToSendResultList.Take(cancellationToken); } catch (OperationCanceledException) { // Sending was cancelled break; } catch (InvalidOperationException) { // The results queue was completed // But we still may have events / xlg to send firstItem = null; } // 1. Pip results m_executionResults.Clear(); if (firstItem != null) { m_executionResults.Add(firstItem); while (m_executionResults.Count < m_maxMessagesPerBatch && m_pipResultListener.ReadyToSendResultList.TryTake(out var item)) { m_executionResults.Add(item); } } // 2. Forwarded events m_eventList.Clear(); while (m_outgoingEvents.TryTake(out var item)) { m_eventList.Add(item); } // 3. Pending execution log events using (DistributionService.Counters.StartStopwatch(DistributionCounter.WorkerFlushExecutionLogDuration)) { // Flush execution log to m_pendingExecutionLog m_executionLogTarget.FlushAsync().Wait(); } if (m_executionResults.Count == 0 && m_eventList.Count == 0 && m_flushedExecutionLog.Length == 0) { // Nothing to send. This can potentially happen while exiting. continue; } // Send notification m_notification.WorkerId = ExecutionService.WorkerId; m_notification.CompletedPips = m_executionResults.Select(p => p.SerializedData).ToList(); m_notification.ForwardedEvents = m_eventList; if (m_flushedExecutionLog.Length > 0) { m_notification.ExecutionLogBlobSequenceNumber = m_xlgBlobSequenceNumber++; m_notification.ExecutionLogData = new ArraySegment <byte>(m_flushedExecutionLog.GetBuffer(), 0, (int)m_flushedExecutionLog.Length); } else { m_notification.ExecutionLogBlobSequenceNumber = 0; m_notification.ExecutionLogData = new ArraySegment <byte>(); } using (DistributionService.Counters.StartStopwatch(DistributionCounter.SendNotificationDuration)) { var callResult = m_orchestratorClient.NotifyAsync(m_notification, m_executionResults.Select(a => a.SemiStableHash).ToList(), cancellationToken).GetAwaiter().GetResult(); if (callResult.Succeeded) { foreach (var result in m_executionResults) { ExecutionService.Transition(result.PipId, WorkerPipState.Reported); Tracing.Logger.Log.DistributionWorkerFinishedPipRequest(m_loggingContext, result.SemiStableHash, ((PipExecutionStep)result.SerializedData.Step).ToString()); } m_numBatchesSent++; } else if (!cancellationToken.IsCancellationRequested) { // Fire-forget exit call with failure. // If we fail to send notification to orchestrator and we were not cancelled, the worker should fail. m_executionLogTarget.Deactivate(); DistributionService.ExitAsync(failure: "Notify event failed to send to orchestrator", isUnexpected: true).Forget(); break; } } m_flushedExecutionLog.SetLength(0); } m_finishedSendingPipResults = true; m_outgoingEvents.CompleteAdding(); DistributionService.Counters.AddToCounter(DistributionCounter.BuildResultBatchesSentToOrchestrator, m_numBatchesSent); }