/// <summary>
/// Function returns an offset to the acquired region that can we safely use for write operation.
/// </summary>
/// <param name="frameLength"></param>
/// <returns>
/// An offset to the acquired region.
/// If reader has been aborted, return uint32_t::max.
/// </returns>
/// <remarks>
/// There is no guarantee that the acquired region is contiguous (it might be overlapping).
/// However it ensures that the next write offset will not be greater than the buffer margin,
/// so the next writer can write an empty FrameHeader.
/// </remarks>
private uint AcquireRegionForWrite(ref int frameLength)
{
// Create AtomicUInt32 proxies once.
//
StdTypesProxy.AtomicUInt32 atomicFreePosition = Sync.FreePosition;
StdTypesProxy.AtomicUInt32 atomicWritePosition = Sync.WritePosition;
StdTypesProxy.AtomicBool atomicTerminateChannel = Sync.TerminateChannel;
TChannelSpinPolicy channelSpinPolicy = default;
while (true)
{
// FreePosition is expected to be less than WritePosition unless WritePosition has overflow.
// To preserve this order, we read FreePosition first. Otherwise, it might advance if we had read WritePosition first.
//
uint freePosition = atomicFreePosition.Load();
uint writePosition = atomicWritePosition.LoadRelaxed();
// Check if there is enough bytes to write frame (full frame or a link).
// Always keep the distance to free offset, at least a size of FrameHeader.
// If WritePosition overflown, then (writePosition - freePosition) is still positive value.
//
if (!(writePosition - freePosition < margin - frameLength))
{
// Not enough free space to acquire region.
//
// Check if the channel is still active.
//
if (atomicTerminateChannel.LoadRelaxed())
{
return(uint.MaxValue);
}
// Advance free position to reclaim memory for writes.
// Retry after that, as another writer might acquire just the released region.
//
AdvanceFreePosition();
continue;
}
// If the end of the requested frame is located in the buffer margin, extend the acquired region.
//
uint frameLengthAdj = 0;
// NextWritePosition is at the frame end.
// NextWriteOffset (calculated from NextWritePosition) must be aligned to sizeof(uint32_t)
// therefore frame length must be also aligned.
//
uint nextWritePosition = (uint)(writePosition + frameLength);
// Ensure that after a full frame there is enough space for the next frame header.
// Otherwise, we will not be able to store next frame, because the frame header will not fit in the buffer.
//
uint nextWriteOffset = nextWritePosition % Size;
if (nextWriteOffset >= margin)
{
// Update frameLength, as we acquired more than requested.
//
frameLengthAdj = Size - nextWriteOffset;
nextWritePosition += frameLengthAdj;
}
uint expectedWritePosition = writePosition;
if (atomicWritePosition.LoadRelaxed() != expectedWritePosition ||
atomicWritePosition.CompareExchange(nextWritePosition, expectedWritePosition) != expectedWritePosition)
{
// Failed to advance write offset, another writer acquired this region.
//
channelSpinPolicy.FailedToAcquireWriteRegion();
continue;
}
frameLength += (int)frameLengthAdj;
// The region should be empty except for free links.
// Frame links are always stored in offset aligned to sizeof int.
//
uint writeOffset = writePosition % Size;
return(writeOffset);
}
}
/// <summary>
/// Wait for the frame become available.
/// </summary>
/// <returns>Returns an offset to the frame buffer.</returns>
/// <remarks>
/// Reader function.
/// If the wait has been aborted, it returns uint32_t::max.
/// </remarks>
internal uint WaitForFrame()
{
uint readPosition;
TChannelSpinPolicy channelSpinPolicy = default;
// Create AtomicUInt32 proxy once.
//
StdTypesProxy.AtomicUInt32 atomicReadPosition = Sync.ReadPosition;
StdTypesProxy.AtomicUInt32 atomicReaderInWaitingStateCount = Sync.ReaderInWaitingStateCount;
StdTypesProxy.AtomicBool atomicTerminateChannel = Sync.TerminateChannel;
MlosProxy.FrameHeader frame = default;
uint shouldWait = 0;
// int spinIndex = 0;
while (true)
{
// Wait for the frame become available.
// Spin on current frame (ReadOffset).
//
readPosition = atomicReadPosition.Load();
uint readOffset = readPosition % Size;
frame = Frame(readOffset);
int frameLength = frame.Length.Load();
if (frameLength > 0)
{
// Writer had updated the length.
// Frame is ready and available for the reader.
// Advance ReadIndex to end of the frame, and allow other reads to process next frame.
//
uint expectedReadPosition = readPosition;
uint nextReadPosition = readPosition + (uint)(frameLength & (~1));
if (atomicReadPosition.LoadRelaxed() != expectedReadPosition ||
atomicReadPosition.CompareExchange(nextReadPosition, expectedReadPosition) != expectedReadPosition)
{
// Other reader advanced ReadPosition therefore it will process the frame.
//
channelSpinPolicy.FailedToAcquireReadRegion();
continue;
}
// Current reader owns the frame. Wait untill the reader completes the write.
//
while ((frameLength & 1) == 1)
{
channelSpinPolicy.WaitForFrameCompletion();
frameLength = frame.Length.Load();
}
break;
}
channelSpinPolicy.WaitForNewFrame();
// if ((++spinIndex & 0xff) == 0)
{
// No frame yet, spin if the channel is stil active.
//
if (atomicTerminateChannel.LoadRelaxed())
{
return(uint.MaxValue);
}
}
// Wait for the synchronization primitive.
//
if (shouldWait != 0)
{
ChannelPolicy.WaitForFrame();
atomicReaderInWaitingStateCount.FetchSub(shouldWait);
shouldWait = 0;
}
else
{
// Before reader enters wait state it will increase ReaderInWaitingState count and then check if are there any messages in the channel.
//
shouldWait = 1;
atomicReaderInWaitingStateCount.FetchAdd(shouldWait);
}
// If (frameLength < 0) there is active cleaning up on this frame by the writer.
// The read offset had already advanced, so retry.
//
}
// Reset InWaitingState counter.
//
atomicReaderInWaitingStateCount.FetchSub(shouldWait);
// Reader acquired read region, frame is ready.
//
return(readPosition % Size);
}
/// <summary>
/// Follows free links until we reach read position.
/// </summary>
/// <remarks>
/// While we follow the links, the method is not cleaning the memory.
/// The memory is cleared by the reader after processing the frame.
/// The whole memory region is clean except locations where negative frame length values are stored
/// to signal that the message has been read and the frame is free-able.
/// Those locations are always aligned to the size of uint32_t. The current reader continues to spin if it reads negative frame length.
/// </remarks>
internal void AdvanceFreePosition()
{
// Create AtomicUInt32 proxies once.
//
StdTypesProxy.AtomicUInt32 atomicFreePosition = Sync.FreePosition;
StdTypesProxy.AtomicUInt32 atomicReadPosition = Sync.ReadPosition;
// Move free position and allow the writer to advance.
//
uint freePosition = atomicFreePosition.Load();
uint readPosition = atomicReadPosition.LoadRelaxed();
if (freePosition == readPosition)
{
// Free position points to the current read position.
//
return;
}
// For diagnostic purposes, following the free links we should get the same distance.
//
uint distance = readPosition - freePosition;
// Follow the free links up to a current read position.
// Cleanup is completed when free position is equal to read position.
// However by the time this cleanup is completed,
// the reader threads might process more frames and advance read position.
//
while (freePosition != readPosition)
{
// Load a frame from the beginning of the free region.
// However, other writer threads might already advance the free position.
// In this case, local free position points to the write region and
// we will fail to advance free offset.
//
uint freeOffset = freePosition % Size;
MlosProxy.FrameHeader frame = Frame(freeOffset);
int frameLength = frame.Length.Load();
if (frameLength >= 0)
{
// Frame is currently processed or has been already cleared.
// Other writer thread advanced free position, there is no point to check using compare_exchange_weak.
// Local free offset is now the write region.
//
return;
}
// Advance free position. The frame length is negative.
//
uint expectedFreePosition = freePosition;
uint nextFreePosition = freePosition - (uint)frameLength;
if (atomicFreePosition.LoadRelaxed() != expectedFreePosition ||
atomicFreePosition.CompareExchange(nextFreePosition, expectedFreePosition) != expectedFreePosition)
{
// Advanced by another writer, local free offset is now the write region.
//
return;
}
freePosition = nextFreePosition;
// Frame length is negative.
//
distance += (uint)frameLength;
}
}
