private void SendNotifications(CancellationToken cancellationToken)
{
ExtendedPipCompletionData firstItem;
while (!m_pipResultListener.ReadyToSendResultList.IsCompleted && !cancellationToken.IsCancellationRequested)
{
try
{
// Sending of notifications is driven by pip results - block until we have a new result to send
firstItem = m_pipResultListener.ReadyToSendResultList.Take(cancellationToken);
}
catch (OperationCanceledException)
{
// Sending was cancelled
break;
}
catch (InvalidOperationException)
{
// The results queue was completed
// But we still may have events / xlg to send
firstItem = null;
}
// 1. Pip results
m_executionResults.Clear();
if (firstItem != null)
{
m_executionResults.Add(firstItem);
while (m_executionResults.Count < m_maxMessagesPerBatch && m_pipResultListener.ReadyToSendResultList.TryTake(out var item))
{
m_executionResults.Add(item);
}
}
// 2. Forwarded events
m_eventList.Clear();
while (m_outgoingEvents.TryTake(out var item))
{
m_eventList.Add(item);
}
// 3. Pending execution log events
using (DistributionService.Counters.StartStopwatch(DistributionCounter.WorkerFlushExecutionLogDuration))
{
// Flush execution log to m_pendingExecutionLog
m_executionLogTarget.FlushAsync().Wait();
}
if (m_executionResults.Count == 0 && m_eventList.Count == 0 && m_flushedExecutionLog.Length == 0)
{
// Nothing to send. This can potentially happen while exiting.
continue;
}
// Send notification
m_notification.WorkerId = ExecutionService.WorkerId;
m_notification.CompletedPips = m_executionResults.Select(p => p.SerializedData).ToList();
m_notification.ForwardedEvents = m_eventList;
if (m_flushedExecutionLog.Length > 0)
{
m_notification.ExecutionLogBlobSequenceNumber = m_xlgBlobSequenceNumber++;
m_notification.ExecutionLogData = new ArraySegment <byte>(m_flushedExecutionLog.GetBuffer(), 0, (int)m_flushedExecutionLog.Length);
}
else
{
m_notification.ExecutionLogBlobSequenceNumber = 0;
m_notification.ExecutionLogData = new ArraySegment <byte>();
}
using (DistributionService.Counters.StartStopwatch(DistributionCounter.SendNotificationDuration))
{
var callResult = m_orchestratorClient.NotifyAsync(m_notification,
m_executionResults.Select(a => a.SemiStableHash).ToList(),
cancellationToken).GetAwaiter().GetResult();
if (callResult.Succeeded)
{
foreach (var result in m_executionResults)
{
ExecutionService.Transition(result.PipId, WorkerPipState.Reported);
Tracing.Logger.Log.DistributionWorkerFinishedPipRequest(m_loggingContext, result.SemiStableHash, ((PipExecutionStep)result.SerializedData.Step).ToString());
}
m_numBatchesSent++;
}
else if (!cancellationToken.IsCancellationRequested)
{
// Fire-forget exit call with failure.
// If we fail to send notification to orchestrator and we were not cancelled, the worker should fail.
m_executionLogTarget.Deactivate();
DistributionService.ExitAsync(failure: "Notify event failed to send to orchestrator", isUnexpected: true).Forget();
break;
}
}
m_flushedExecutionLog.SetLength(0);
}
m_finishedSendingPipResults = true;
m_outgoingEvents.CompleteAdding();
DistributionService.Counters.AddToCounter(DistributionCounter.BuildResultBatchesSentToOrchestrator, m_numBatchesSent);
}