public async Task LogExecutionBlobAsync(WorkerNotificationArgs notification)
{
Contract.Requires(notification.ExecutionLogData != null || notification.ExecutionLogData.Count != 0);
m_executionBlobQueue.Add(notification);
// After we put the executionBlob in a queue, we can unblock the caller and give an ACK to the worker.
await Task.Yield();
// Execution log events cannot be logged by multiple threads concurrently since they must be ordered
using (await m_logBlobMutex.AcquireAsync())
{
// We need to dequeue and process the blobs in order.
// Here, we do not necessarily process the blob that is just added to the queue above.
// There might be another thread that adds the next blob to the queue after the current thread,
// and that thread might acquire the lock earlier.
WorkerNotificationArgs executionBlobNotification = null;
Contract.Assert(m_executionBlobQueue.TryTake(out executionBlobNotification), "The executionBlob queue cannot be empty");
int blobSequenceNumber = executionBlobNotification.ExecutionLogBlobSequenceNumber;
ArraySegment <byte> executionLogBlob = executionBlobNotification.ExecutionLogData;
if (m_workerExecutionLogTarget == null)
{
return;
}
try
{
// Workers send execution log blobs one-at-a-time, waiting for a response from the master between each message.
// A sequence number higher than the last logged blob sequence number indicates a worker sent a subsequent blob without waiting for a response.
Contract.Assert(blobSequenceNumber <= m_lastBlobSeqNumber + 1, "Workers should not send a new execution log blob until receiving a response from the master for all previous blobs.");
// Due to network latency and retries, it's possible to receive a message multiple times.
// Ignore any low numbered blobs since they should have already been logged and ack'd at some point before
// the worker could send a higher numbered blob.
if (blobSequenceNumber != m_lastBlobSeqNumber + 1)
{
return;
}
if (m_executionLogBufferStream == null)
{
// Create the stream on demand, because we need to pass the BinaryLogReader stream with the header bytes in order
// to correctly deserialize events
m_executionLogBufferStream = new MemoryStream();
}
// Write the new execution log event content into buffer starting at beginning of buffer stream
m_executionLogBufferStream.SetLength(0);
m_executionLogBufferStream.Write(executionLogBlob.Array, executionLogBlob.Offset, executionLogBlob.Count);
// Reset the buffer stream to beginning and reset reader to ensure it reads events starting from beginning
m_executionLogBufferStream.Position = 0;
if (m_executionLogBinaryReader == null)
{
m_executionLogBinaryReader = new BinaryLogReader(m_executionLogBufferStream, m_masterService.Environment.Context);
m_executionLogReader = new ExecutionLogFileReader(m_executionLogBinaryReader, m_workerExecutionLogTarget);
}
m_executionLogBinaryReader.Reset();
// Read all events into worker execution log target
if (!m_executionLogReader.ReadAllEvents())
{
Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlobAsync), "Failed to read all worker events");
// Disable further processing of execution log since an error was encountered during processing
m_workerExecutionLogTarget = null;
}
else
{
m_lastBlobSeqNumber = blobSequenceNumber;
}
}
catch (Exception ex)
{
Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlobAsync), ex.ToStringDemystified() + Environment.NewLine
+ "Message sequence number: " + blobSequenceNumber
+ " Last sequence number logged: " + m_lastBlobSeqNumber);
// Disable further processing of execution log since an exception was encountered during processing
m_workerExecutionLogTarget = null;
}
}
if (m_executionBlobQueue.IsCompleted)
{
m_executionBlobCompletion.TrySetResult(true);
}
}
public void LogExecutionBlob(ArraySegment <byte> executionLogBlob, int blobSequenceNumber)
{
// Execution log events cannot be logged by multiple threads concurrently since they must be ordered
lock (m_logBlobLock)
{
if (m_workerExecutionLogTarget == null)
{
return;
}
try
{
// Workers send execution log blobs one-at-a-time, waiting for a response from the master between each message.
// A sequence number higher than the last logged blob sequence number indicates a worker sent a subsequent blob without waiting for a response.
Contract.Assert(blobSequenceNumber <= m_lastBlobSeqNumber + 1, "Workers should not send a new execution log blob until receiving a response from the master for all previous blobs.");
// Due to network latency and retries, it's possible to receive a message multiple times.
// Ignore any low numbered blobs since they should have already been logged and ack'd at some point before
// the worker could send a higher numbered blob.
if (blobSequenceNumber != m_lastBlobSeqNumber + 1)
{
return;
}
if (m_executionLogBufferStream == null)
{
// Create the stream on demand, because we need to pass the BinaryLogReader stream with the header bytes in order
// to correctly deserialize events
m_executionLogBufferStream = new MemoryStream();
}
// Write the new execution log event content into buffer starting at beginning of buffer stream
m_executionLogBufferStream.SetLength(0);
m_executionLogBufferStream.Write(executionLogBlob.Array, executionLogBlob.Offset, executionLogBlob.Count);
// Reset the buffer stream to beginning and reset reader to ensure it reads events starting from beginning
m_executionLogBufferStream.Position = 0;
if (m_executionLogBinaryReader == null)
{
m_executionLogBinaryReader = new BinaryLogReader(m_executionLogBufferStream, m_masterService.Environment.Context);
m_executionLogReader = new ExecutionLogFileReader(m_executionLogBinaryReader, m_workerExecutionLogTarget);
}
m_executionLogBinaryReader.Reset();
// Read all events into worker execution log target
if (!m_executionLogReader.ReadAllEvents())
{
Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlob), "Failed to read all worker events");
// Disable further processing of execution log since an error was encountered during processing
m_workerExecutionLogTarget = null;
}
else
{
m_lastBlobSeqNumber = blobSequenceNumber;
}
}
catch (Exception ex)
{
Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlob), ex.ToStringDemystified() + Environment.NewLine
+ "Message sequence number: " + blobSequenceNumber
+ " Last sequence number logged: " + m_lastBlobSeqNumber);
// Disable further processing of execution log since an exception was encountered during processing
m_workerExecutionLogTarget = null;
}
}
}
