public async Task LogExecutionBlobAsync(WorkerNotificationArgs notification) { Contract.Requires(notification.ExecutionLogData != null || notification.ExecutionLogData.Count != 0); m_executionBlobQueue.Add(notification); // After we put the executionBlob in a queue, we can unblock the caller and give an ACK to the worker. await Task.Yield(); // Execution log events cannot be logged by multiple threads concurrently since they must be ordered using (await m_logBlobMutex.AcquireAsync()) { // We need to dequeue and process the blobs in order. // Here, we do not necessarily process the blob that is just added to the queue above. // There might be another thread that adds the next blob to the queue after the current thread, // and that thread might acquire the lock earlier. WorkerNotificationArgs executionBlobNotification = null; Contract.Assert(m_executionBlobQueue.TryTake(out executionBlobNotification), "The executionBlob queue cannot be empty"); int blobSequenceNumber = executionBlobNotification.ExecutionLogBlobSequenceNumber; ArraySegment <byte> executionLogBlob = executionBlobNotification.ExecutionLogData; if (m_workerExecutionLogTarget == null) { return; } try { // Workers send execution log blobs one-at-a-time, waiting for a response from the master between each message. // A sequence number higher than the last logged blob sequence number indicates a worker sent a subsequent blob without waiting for a response. Contract.Assert(blobSequenceNumber <= m_lastBlobSeqNumber + 1, "Workers should not send a new execution log blob until receiving a response from the master for all previous blobs."); // Due to network latency and retries, it's possible to receive a message multiple times. // Ignore any low numbered blobs since they should have already been logged and ack'd at some point before // the worker could send a higher numbered blob. if (blobSequenceNumber != m_lastBlobSeqNumber + 1) { return; } if (m_executionLogBufferStream == null) { // Create the stream on demand, because we need to pass the BinaryLogReader stream with the header bytes in order // to correctly deserialize events m_executionLogBufferStream = new MemoryStream(); } // Write the new execution log event content into buffer starting at beginning of buffer stream m_executionLogBufferStream.SetLength(0); m_executionLogBufferStream.Write(executionLogBlob.Array, executionLogBlob.Offset, executionLogBlob.Count); // Reset the buffer stream to beginning and reset reader to ensure it reads events starting from beginning m_executionLogBufferStream.Position = 0; if (m_executionLogBinaryReader == null) { m_executionLogBinaryReader = new BinaryLogReader(m_executionLogBufferStream, m_masterService.Environment.Context); m_executionLogReader = new ExecutionLogFileReader(m_executionLogBinaryReader, m_workerExecutionLogTarget); } m_executionLogBinaryReader.Reset(); // Read all events into worker execution log target if (!m_executionLogReader.ReadAllEvents()) { Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlobAsync), "Failed to read all worker events"); // Disable further processing of execution log since an error was encountered during processing m_workerExecutionLogTarget = null; } else { m_lastBlobSeqNumber = blobSequenceNumber; } } catch (Exception ex) { Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlobAsync), ex.ToStringDemystified() + Environment.NewLine + "Message sequence number: " + blobSequenceNumber + " Last sequence number logged: " + m_lastBlobSeqNumber); // Disable further processing of execution log since an exception was encountered during processing m_workerExecutionLogTarget = null; } } if (m_executionBlobQueue.IsCompleted) { m_executionBlobCompletion.TrySetResult(true); } }
public void LogExecutionBlob(ArraySegment <byte> executionLogBlob, int blobSequenceNumber) { // Execution log events cannot be logged by multiple threads concurrently since they must be ordered lock (m_logBlobLock) { if (m_workerExecutionLogTarget == null) { return; } try { // Workers send execution log blobs one-at-a-time, waiting for a response from the master between each message. // A sequence number higher than the last logged blob sequence number indicates a worker sent a subsequent blob without waiting for a response. Contract.Assert(blobSequenceNumber <= m_lastBlobSeqNumber + 1, "Workers should not send a new execution log blob until receiving a response from the master for all previous blobs."); // Due to network latency and retries, it's possible to receive a message multiple times. // Ignore any low numbered blobs since they should have already been logged and ack'd at some point before // the worker could send a higher numbered blob. if (blobSequenceNumber != m_lastBlobSeqNumber + 1) { return; } if (m_executionLogBufferStream == null) { // Create the stream on demand, because we need to pass the BinaryLogReader stream with the header bytes in order // to correctly deserialize events m_executionLogBufferStream = new MemoryStream(); } // Write the new execution log event content into buffer starting at beginning of buffer stream m_executionLogBufferStream.SetLength(0); m_executionLogBufferStream.Write(executionLogBlob.Array, executionLogBlob.Offset, executionLogBlob.Count); // Reset the buffer stream to beginning and reset reader to ensure it reads events starting from beginning m_executionLogBufferStream.Position = 0; if (m_executionLogBinaryReader == null) { m_executionLogBinaryReader = new BinaryLogReader(m_executionLogBufferStream, m_masterService.Environment.Context); m_executionLogReader = new ExecutionLogFileReader(m_executionLogBinaryReader, m_workerExecutionLogTarget); } m_executionLogBinaryReader.Reset(); // Read all events into worker execution log target if (!m_executionLogReader.ReadAllEvents()) { Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlob), "Failed to read all worker events"); // Disable further processing of execution log since an error was encountered during processing m_workerExecutionLogTarget = null; } else { m_lastBlobSeqNumber = blobSequenceNumber; } } catch (Exception ex) { Logger.Log.DistributionCallMasterCodeException(m_appLoggingContext, nameof(LogExecutionBlob), ex.ToStringDemystified() + Environment.NewLine + "Message sequence number: " + blobSequenceNumber + " Last sequence number logged: " + m_lastBlobSeqNumber); // Disable further processing of execution log since an exception was encountered during processing m_workerExecutionLogTarget = null; } } }