private static QueueSegment *VolatileRead(ref QueueSegment *segment)
{
var value = segment;
Thread.MemoryBarrier();
return(value);
}
private bool TryDequeueSlow <T>(out T result) where T : unmanaged
{
while (true)
{
QueueSegment *head = _head;
if (head->TryDequeue(out result))
{
return(true);
}
if (head->_nextSegment == null)
{
result = default;
return(false);
}
UDebug.Assert(head->_frozen);
if (head->TryDequeue(out result))
{
return(true);
}
_crossSegmentLock.Lock();
if (head == _head)
{
var next = head->_nextSegment;
QueueSegment.Free(head);
_head = next;
}
_crossSegmentLock.Unlock();
}
}
private bool TryPeek <T>(out T result, bool useResult = false) where T : unmanaged
{
QueueSegment *head = _head;
while (true)
{
QueueSegment *next = VolatileRead(ref head->_nextSegment);
if (head->TryPeek(out result, useResult))
{
return(true);
}
if (next != null)
{
UDebug.Assert(next == head->_nextSegment);
head = next;
}
else if (VolatileRead(ref head->_nextSegment) == null)
{
break;
}
}
result = default;
return(false);
}
private bool TryEnqueueSlow <T>(T item) where T : unmanaged
{
while (true)
{
QueueSegment *tail = _tail;
if (tail->TryEnqueue(item))
{
return(true);
}
// Do not create a new segment if the queue has a fixed size.
if (_fixedSize)
{
return(false);
}
_crossSegmentLock.Lock();
if (tail == _tail)
{
tail->EnsureFrozenForEnqueues();
int nextSize = tail->_preserved ? INITIAL_SEGMENT_LENGTH : Math.Min(tail->Capacity * 2, MAX_SEGMENT_LENGTH);
var newTail = QueueSegment.Allocate(nextSize, _slotStride, _slotOffset);
tail->_nextSegment = newTail;
_tail = newTail;
}
_crossSegmentLock.Unlock();
}
}
private void SnapForObservation(out QueueSegment *headSeg, out int headSeg_Head, out QueueSegment *tailSeg, out int tailSeg_Tail)
{
_crossSegmentLock.Lock();
headSeg = _head;
tailSeg = _tail;
UDebug.Assert(headSeg != null);
UDebug.Assert(tailSeg != null);
UDebug.Assert(tailSeg->_nextSegment == null);
for (QueueSegment *s = headSeg; ; s = s->_nextSegment)
{
s->_preserved = true;
if (s == tailSeg)
{
break;
}
UDebug.Assert(s->_frozen);
}
tailSeg->EnsureFrozenForEnqueues();
headSeg_Head = Volatile.Read(ref headSeg->_headAndTail.Head);
tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail);
_crossSegmentLock.Unlock();
}
/// <summary>
/// Calculates the amount of items over a snapped region of QueueSegments.
/// </summary>
private static long GetCount(QueueSegment *headSeg, int headSeg_Head, QueueSegment *tailSeg, int tailSeg_Tail)
{
UDebug.Assert(headSeg->_preserved);
UDebug.Assert(headSeg->_frozen);
UDebug.Assert(tailSeg->_preserved);
UDebug.Assert(tailSeg->_frozen);
long count = 0;
int headSeg_Tail = (headSeg == tailSeg ? tailSeg_Tail : Volatile.Read(ref headSeg->_headAndTail.Tail)) - headSeg->FreezeOffset;
if (headSeg_Head < headSeg_Tail)
{
headSeg_Head &= headSeg->Mask;
headSeg_Tail &= headSeg->Mask;
count += headSeg_Head < headSeg_Tail ?
headSeg_Tail - headSeg_Head :
headSeg->Capacity - headSeg_Head + headSeg_Tail;
}
if (headSeg != tailSeg)
{
for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment)
{
UDebug.Assert(s->_preserved);
UDebug.Assert(s->_frozen);
count += s->_headAndTail.Tail - s->FreezeOffset;
}
count += tailSeg_Tail - tailSeg->FreezeOffset;
}
return(count);
}
public static int GetCapacity(UnsafeMPMCQueue *queue)
{
UDebug.Assert(queue != null);
UDebug.Assert(queue->_head != null);
SpinWait spinner = default;
while (true)
{
// Capture head and tail
var headSeg = queue->_head;
var tailSeg = queue->_tail;
// Single QueueSegment
if (headSeg == tailSeg)
{
if (headSeg == queue->_head && tailSeg == queue->_tail)
{
return(headSeg->Capacity);
}
}
// Two QueueSegments
else if (headSeg->_nextSegment == tailSeg)
{
if (headSeg == queue->_head && tailSeg == queue->_tail)
{
return(headSeg->Capacity + tailSeg->Capacity);
}
}
// Mutliple QueueSegments (requires locking)
else
{
// Aquire cross-segment lock
queue->_crossSegmentLock.Lock();
if (headSeg == queue->_head && tailSeg == queue->_tail)
{
int capacity = headSeg->Capacity + tailSeg->Capacity;
for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment)
{
UDebug.Assert(s->_frozen, "Internal segment must be frozen as there's a following segment.");
capacity += s->Capacity;
}
// Release cross-segment lock
queue->_crossSegmentLock.Unlock();
return(capacity);
}
// Release cross-segment lock
queue->_crossSegmentLock.Unlock();
}
spinner.SpinOnce();
}
}
/// <summary>
/// Calculates the amount of items in a specific QueueSegment.
/// </summary>
private static int GetCount(QueueSegment *segment, int head, int tail)
{
if (head != tail && head != tail - segment->FreezeOffset)
{
head &= segment->Mask;
tail &= segment->Mask;
return(head < tail ? tail - head : segment->Capacity - head + tail);
}
return(0);
}
internal static void Free(QueueSegment *segment)
{
if (segment == null)
{
return;
}
*segment = default;
Memory.Free(segment);
}
private static T *GetItemWhenAvailable <T>(QueueSegment *segment, int index) where T : unmanaged
{
UDebug.Assert(segment->_preserved);
int expectedSeq = (index + 1) & segment->Mask;
if ((*segment->SequenceNumber(index) & segment->Mask) != expectedSeq)
{
SpinWait spinner = default;
while ((Volatile.Read(ref *segment->SequenceNumber(index)) & segment->Mask) != expectedSeq)
{
spinner.SpinOnce();
}
}
return(segment->Element <T>(index));
}
internal static QueueSegment *Allocate(int capacity, int slotStride, int slotOffset)
{
if (capacity < 1)
{
throw new ArgumentOutOfRangeException(nameof(capacity), string.Format(ThrowHelper.ArgumentOutOfRange_MustBePositive, nameof(capacity)));
}
UDebug.Assert(slotStride > 0);
UDebug.Assert(slotOffset > 0);
capacity = Memory.RoundUpToPowerOf2(capacity);
int alignment = Memory.GetAlignment(slotStride);
var sizeOfQueue = Memory.RoundToAlignment(sizeof(UnsafeMPSCQueue), alignment);
var sizeOfArray = slotStride * capacity;
var ptr = Memory.MallocAndZero(sizeOfQueue + sizeOfArray, alignment);
QueueSegment *queue = (QueueSegment *)ptr;
// Readonly values
queue->_items = (byte *)ptr + sizeOfQueue;
queue->_capacity = capacity;
queue->_slotStride = slotStride;
queue->_slotOffset = slotOffset;
queue->_mask = capacity - 1;
queue->_headAndTail = default;
queue->_nextSegment = default;
queue->_frozen = false;
queue->_preserved = false;
// Set all sequence numbers for this segment
for (int i = 0; i < queue->_capacity; i++)
{
*queue->SequenceNumber(i) = i;
}
return(queue);
}
/// <summary>
/// Tries to dequeue an item from the queue. Returns false if there's no items in the queue.
/// </summary>
public static bool TryDequeue <T>(UnsafeMPMCQueue *queue, out T result) where T : unmanaged
{
UDebug.Assert(queue != null);
UDebug.Assert(queue->_head != null);
UDebug.Assert(typeof(T).TypeHandle.Value == queue->_typeHandle);
QueueSegment *head = queue->_head;
if (head->TryDequeue(out result))
{
return(true);
}
if (head->_nextSegment == null)
{
result = default;
return(false);
}
return(queue->TryDequeueSlow(out result));
}
internal Enumerator(QueueSegment *head, int headHead, QueueSegment *tail, int tailTail)
{
UDebug.Assert(head->_preserved);
UDebug.Assert(head->_frozen);
UDebug.Assert(tail->_preserved);
UDebug.Assert(tail->_frozen);
__state = 0;
__index = 0;
__headTail = 0;
__sTail = 0;
__segIndex = null;
_current = null;
_head = head;
_tail = tail;
_headHead = headHead;
_tailTail = tailTail;
SetupState();
}
public bool MoveNext()
{
// Uses "decompiled" Yield implementation. See: https://csharpindepth.com/articles/IteratorBlockImplementation for more information.
// For loop has been deconstructed to allow re-entry from another state.
switch (__state)
{
#region for (int i = headHead; i < headTail; i++)
case 0:
{
__state = -1;
__index = _headHead - 1;
goto case 1;
}
case 1:
{
__state = -1;
__index++;
if (__index < __headTail)
{
_current = GetItemWhenAvailable <T>(_head, __index);
__state = 1;
return(true);
}
goto case 6;
}
#endregion
#region for (int i = headHead; i < head._slots.Length; i++)
case 2:
{
__state = -1;
__index = _headHead - 1;
goto case 3;
}
case 3:
{
__state = -1;
__index++;
if (__index < _head->Capacity)
{
_current = GetItemWhenAvailable <T>(_head, __index);
__state = 3;
return(true);
}
// Fall over into next state once this state ends.
goto case 4;
}
#endregion
#region for (int i = 0; i < headTail; i++)
case 4:
{
__state = -1;
__index = -1;
goto case 5;
}
case 5:
{
__state = -1;
__index++;
if (__index < __headTail)
{
_current = GetItemWhenAvailable <T>(_head, __index);
__state = 5;
return(true);
}
// Goto state that checks if head and tail are equal
goto case 6;
}
#endregion
case 6: // head != tail
{
__state = -1;
if (_head == _tail)
{
break; // Head and Tail are the same. All items are iterated.
}
// More QueueSegments to iterate.
goto case 7;
}
#region for (ConcurrentQueueSegment<T> s = head._nextSegment!; s != tail; s = s._nextSegment!)
case 7: // for initialise
{
__state = -1;
__segIndex = _head->_nextSegment;
goto case 8;
}
case 8: // for condition + body
{
if (__segIndex != _tail)
{
UDebug.Assert(__segIndex->_preserved);
UDebug.Assert(__segIndex->_frozen);
__sTail = __segIndex->_headAndTail.Tail - __segIndex->FreezeOffset;
goto case 10; // Goto nested for loop
}
goto case 12; // Break out of loop and enter final state
}
case 9: // for increment
{
__segIndex = __segIndex->_nextSegment;
goto case 8; // Next iteration of for
}
#endregion
#region for (int i = 0; i < sTail; i++)
case 10:
{
__state = -1;
__index = -1;
goto case 11;
}
case 11:
{
__state = -1;
__index++;
if (__index < __sTail)
{
_current = GetItemWhenAvailable <T>(__segIndex, __index);
__state = 11;
return(true);
}
// Increment parent for-loop when this one ends
goto case 9;
}
#endregion
#region for (int i = 0; i < tailTail; i++)
case 12:
{
__state = -1;
__index = -1;
_tailTail -= _tail->FreezeOffset;
goto case 13;
}
case 13:
{
__state = -1;
__index++;
if (__index < _tailTail)
{
_current = GetItemWhenAvailable <T>(_tail, __index);
__state = 13;
return(true);
}
break;
}
#endregion
}
return(false);
}
public static int GetCount(UnsafeMPMCQueue *queue)
{
UDebug.Assert(queue != null);
UDebug.Assert(queue->_head != null);
SpinWait spinner = default;
while (true)
{
// Capture head and tail
var headSeg = queue->_head;
var tailSeg = queue->_tail;
int headSeg_Head = Volatile.Read(ref headSeg->_headAndTail.Head);
int headSeg_Tail = Volatile.Read(ref headSeg->_headAndTail.Tail);
// Single QueueSegment
if (headSeg == tailSeg)
{
// Count can be reliably determined if the captured segments still match.
if (headSeg == queue->_head && tailSeg == queue->_tail &&
headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) &&
headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail))
{
return(GetCount(headSeg, headSeg_Head, headSeg_Tail));
}
}
// Two QueueSegments
else if (headSeg->_nextSegment == tailSeg)
{
int tailSeg_Head = Volatile.Read(ref tailSeg->_headAndTail.Head);
int tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail);
if (headSeg == queue->_head &&
tailSeg == queue->_tail &&
headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) &&
headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail) &&
tailSeg_Head == Volatile.Read(ref tailSeg->_headAndTail.Head) &&
tailSeg_Tail == Volatile.Read(ref tailSeg->_headAndTail.Tail))
{
return(GetCount(headSeg, headSeg_Head, headSeg_Tail) + GetCount(tailSeg, tailSeg_Head, tailSeg_Tail));
}
}
// Mutliple QueueSegments (requires locking)
else
{
// Aquire cross-segment lock
queue->_crossSegmentLock.Lock();
if (headSeg == queue->_head && tailSeg == queue->_tail)
{
int tailSeg_Head = Volatile.Read(ref tailSeg->_headAndTail.Head);
int tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail);
if (headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) &&
headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail) &&
tailSeg_Head == Volatile.Read(ref tailSeg->_headAndTail.Head) &&
tailSeg_Tail == Volatile.Read(ref tailSeg->_headAndTail.Tail))
{
int count = GetCount(headSeg, headSeg_Head, headSeg_Tail) + GetCount(tailSeg, tailSeg_Head, tailSeg_Tail);
for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment)
{
UDebug.Assert(s->_frozen, "Internal segment must be frozen as there's a following segment.");
count += s->_headAndTail.Tail - s->FreezeOffset;
}
// Release cross-segment lock
queue->_crossSegmentLock.Unlock();
return(count);
}
}
// Release cross-segment lock
queue->_crossSegmentLock.Unlock();
}
spinner.SpinOnce();
}
}
