private static QueueSegment *VolatileRead(ref QueueSegment *segment) { var value = segment; Thread.MemoryBarrier(); return(value); }
private bool TryDequeueSlow <T>(out T result) where T : unmanaged { while (true) { QueueSegment *head = _head; if (head->TryDequeue(out result)) { return(true); } if (head->_nextSegment == null) { result = default; return(false); } UDebug.Assert(head->_frozen); if (head->TryDequeue(out result)) { return(true); } _crossSegmentLock.Lock(); if (head == _head) { var next = head->_nextSegment; QueueSegment.Free(head); _head = next; } _crossSegmentLock.Unlock(); } }
private bool TryPeek <T>(out T result, bool useResult = false) where T : unmanaged { QueueSegment *head = _head; while (true) { QueueSegment *next = VolatileRead(ref head->_nextSegment); if (head->TryPeek(out result, useResult)) { return(true); } if (next != null) { UDebug.Assert(next == head->_nextSegment); head = next; } else if (VolatileRead(ref head->_nextSegment) == null) { break; } } result = default; return(false); }
private bool TryEnqueueSlow <T>(T item) where T : unmanaged { while (true) { QueueSegment *tail = _tail; if (tail->TryEnqueue(item)) { return(true); } // Do not create a new segment if the queue has a fixed size. if (_fixedSize) { return(false); } _crossSegmentLock.Lock(); if (tail == _tail) { tail->EnsureFrozenForEnqueues(); int nextSize = tail->_preserved ? INITIAL_SEGMENT_LENGTH : Math.Min(tail->Capacity * 2, MAX_SEGMENT_LENGTH); var newTail = QueueSegment.Allocate(nextSize, _slotStride, _slotOffset); tail->_nextSegment = newTail; _tail = newTail; } _crossSegmentLock.Unlock(); } }
private void SnapForObservation(out QueueSegment *headSeg, out int headSeg_Head, out QueueSegment *tailSeg, out int tailSeg_Tail) { _crossSegmentLock.Lock(); headSeg = _head; tailSeg = _tail; UDebug.Assert(headSeg != null); UDebug.Assert(tailSeg != null); UDebug.Assert(tailSeg->_nextSegment == null); for (QueueSegment *s = headSeg; ; s = s->_nextSegment) { s->_preserved = true; if (s == tailSeg) { break; } UDebug.Assert(s->_frozen); } tailSeg->EnsureFrozenForEnqueues(); headSeg_Head = Volatile.Read(ref headSeg->_headAndTail.Head); tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail); _crossSegmentLock.Unlock(); }
/// <summary> /// Calculates the amount of items over a snapped region of QueueSegments. /// </summary> private static long GetCount(QueueSegment *headSeg, int headSeg_Head, QueueSegment *tailSeg, int tailSeg_Tail) { UDebug.Assert(headSeg->_preserved); UDebug.Assert(headSeg->_frozen); UDebug.Assert(tailSeg->_preserved); UDebug.Assert(tailSeg->_frozen); long count = 0; int headSeg_Tail = (headSeg == tailSeg ? tailSeg_Tail : Volatile.Read(ref headSeg->_headAndTail.Tail)) - headSeg->FreezeOffset; if (headSeg_Head < headSeg_Tail) { headSeg_Head &= headSeg->Mask; headSeg_Tail &= headSeg->Mask; count += headSeg_Head < headSeg_Tail ? headSeg_Tail - headSeg_Head : headSeg->Capacity - headSeg_Head + headSeg_Tail; } if (headSeg != tailSeg) { for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment) { UDebug.Assert(s->_preserved); UDebug.Assert(s->_frozen); count += s->_headAndTail.Tail - s->FreezeOffset; } count += tailSeg_Tail - tailSeg->FreezeOffset; } return(count); }
public static int GetCapacity(UnsafeMPMCQueue *queue) { UDebug.Assert(queue != null); UDebug.Assert(queue->_head != null); SpinWait spinner = default; while (true) { // Capture head and tail var headSeg = queue->_head; var tailSeg = queue->_tail; // Single QueueSegment if (headSeg == tailSeg) { if (headSeg == queue->_head && tailSeg == queue->_tail) { return(headSeg->Capacity); } } // Two QueueSegments else if (headSeg->_nextSegment == tailSeg) { if (headSeg == queue->_head && tailSeg == queue->_tail) { return(headSeg->Capacity + tailSeg->Capacity); } } // Mutliple QueueSegments (requires locking) else { // Aquire cross-segment lock queue->_crossSegmentLock.Lock(); if (headSeg == queue->_head && tailSeg == queue->_tail) { int capacity = headSeg->Capacity + tailSeg->Capacity; for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment) { UDebug.Assert(s->_frozen, "Internal segment must be frozen as there's a following segment."); capacity += s->Capacity; } // Release cross-segment lock queue->_crossSegmentLock.Unlock(); return(capacity); } // Release cross-segment lock queue->_crossSegmentLock.Unlock(); } spinner.SpinOnce(); } }
/// <summary> /// Calculates the amount of items in a specific QueueSegment. /// </summary> private static int GetCount(QueueSegment *segment, int head, int tail) { if (head != tail && head != tail - segment->FreezeOffset) { head &= segment->Mask; tail &= segment->Mask; return(head < tail ? tail - head : segment->Capacity - head + tail); } return(0); }
internal static void Free(QueueSegment *segment) { if (segment == null) { return; } *segment = default; Memory.Free(segment); }
private static T *GetItemWhenAvailable <T>(QueueSegment *segment, int index) where T : unmanaged { UDebug.Assert(segment->_preserved); int expectedSeq = (index + 1) & segment->Mask; if ((*segment->SequenceNumber(index) & segment->Mask) != expectedSeq) { SpinWait spinner = default; while ((Volatile.Read(ref *segment->SequenceNumber(index)) & segment->Mask) != expectedSeq) { spinner.SpinOnce(); } } return(segment->Element <T>(index)); }
internal static QueueSegment *Allocate(int capacity, int slotStride, int slotOffset) { if (capacity < 1) { throw new ArgumentOutOfRangeException(nameof(capacity), string.Format(ThrowHelper.ArgumentOutOfRange_MustBePositive, nameof(capacity))); } UDebug.Assert(slotStride > 0); UDebug.Assert(slotOffset > 0); capacity = Memory.RoundUpToPowerOf2(capacity); int alignment = Memory.GetAlignment(slotStride); var sizeOfQueue = Memory.RoundToAlignment(sizeof(UnsafeMPSCQueue), alignment); var sizeOfArray = slotStride * capacity; var ptr = Memory.MallocAndZero(sizeOfQueue + sizeOfArray, alignment); QueueSegment *queue = (QueueSegment *)ptr; // Readonly values queue->_items = (byte *)ptr + sizeOfQueue; queue->_capacity = capacity; queue->_slotStride = slotStride; queue->_slotOffset = slotOffset; queue->_mask = capacity - 1; queue->_headAndTail = default; queue->_nextSegment = default; queue->_frozen = false; queue->_preserved = false; // Set all sequence numbers for this segment for (int i = 0; i < queue->_capacity; i++) { *queue->SequenceNumber(i) = i; } return(queue); }
/// <summary> /// Tries to dequeue an item from the queue. Returns false if there's no items in the queue. /// </summary> public static bool TryDequeue <T>(UnsafeMPMCQueue *queue, out T result) where T : unmanaged { UDebug.Assert(queue != null); UDebug.Assert(queue->_head != null); UDebug.Assert(typeof(T).TypeHandle.Value == queue->_typeHandle); QueueSegment *head = queue->_head; if (head->TryDequeue(out result)) { return(true); } if (head->_nextSegment == null) { result = default; return(false); } return(queue->TryDequeueSlow(out result)); }
internal Enumerator(QueueSegment *head, int headHead, QueueSegment *tail, int tailTail) { UDebug.Assert(head->_preserved); UDebug.Assert(head->_frozen); UDebug.Assert(tail->_preserved); UDebug.Assert(tail->_frozen); __state = 0; __index = 0; __headTail = 0; __sTail = 0; __segIndex = null; _current = null; _head = head; _tail = tail; _headHead = headHead; _tailTail = tailTail; SetupState(); }
public bool MoveNext() { // Uses "decompiled" Yield implementation. See: https://csharpindepth.com/articles/IteratorBlockImplementation for more information. // For loop has been deconstructed to allow re-entry from another state. switch (__state) { #region for (int i = headHead; i < headTail; i++) case 0: { __state = -1; __index = _headHead - 1; goto case 1; } case 1: { __state = -1; __index++; if (__index < __headTail) { _current = GetItemWhenAvailable <T>(_head, __index); __state = 1; return(true); } goto case 6; } #endregion #region for (int i = headHead; i < head._slots.Length; i++) case 2: { __state = -1; __index = _headHead - 1; goto case 3; } case 3: { __state = -1; __index++; if (__index < _head->Capacity) { _current = GetItemWhenAvailable <T>(_head, __index); __state = 3; return(true); } // Fall over into next state once this state ends. goto case 4; } #endregion #region for (int i = 0; i < headTail; i++) case 4: { __state = -1; __index = -1; goto case 5; } case 5: { __state = -1; __index++; if (__index < __headTail) { _current = GetItemWhenAvailable <T>(_head, __index); __state = 5; return(true); } // Goto state that checks if head and tail are equal goto case 6; } #endregion case 6: // head != tail { __state = -1; if (_head == _tail) { break; // Head and Tail are the same. All items are iterated. } // More QueueSegments to iterate. goto case 7; } #region for (ConcurrentQueueSegment<T> s = head._nextSegment!; s != tail; s = s._nextSegment!) case 7: // for initialise { __state = -1; __segIndex = _head->_nextSegment; goto case 8; } case 8: // for condition + body { if (__segIndex != _tail) { UDebug.Assert(__segIndex->_preserved); UDebug.Assert(__segIndex->_frozen); __sTail = __segIndex->_headAndTail.Tail - __segIndex->FreezeOffset; goto case 10; // Goto nested for loop } goto case 12; // Break out of loop and enter final state } case 9: // for increment { __segIndex = __segIndex->_nextSegment; goto case 8; // Next iteration of for } #endregion #region for (int i = 0; i < sTail; i++) case 10: { __state = -1; __index = -1; goto case 11; } case 11: { __state = -1; __index++; if (__index < __sTail) { _current = GetItemWhenAvailable <T>(__segIndex, __index); __state = 11; return(true); } // Increment parent for-loop when this one ends goto case 9; } #endregion #region for (int i = 0; i < tailTail; i++) case 12: { __state = -1; __index = -1; _tailTail -= _tail->FreezeOffset; goto case 13; } case 13: { __state = -1; __index++; if (__index < _tailTail) { _current = GetItemWhenAvailable <T>(_tail, __index); __state = 13; return(true); } break; } #endregion } return(false); }
public static int GetCount(UnsafeMPMCQueue *queue) { UDebug.Assert(queue != null); UDebug.Assert(queue->_head != null); SpinWait spinner = default; while (true) { // Capture head and tail var headSeg = queue->_head; var tailSeg = queue->_tail; int headSeg_Head = Volatile.Read(ref headSeg->_headAndTail.Head); int headSeg_Tail = Volatile.Read(ref headSeg->_headAndTail.Tail); // Single QueueSegment if (headSeg == tailSeg) { // Count can be reliably determined if the captured segments still match. if (headSeg == queue->_head && tailSeg == queue->_tail && headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) && headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail)) { return(GetCount(headSeg, headSeg_Head, headSeg_Tail)); } } // Two QueueSegments else if (headSeg->_nextSegment == tailSeg) { int tailSeg_Head = Volatile.Read(ref tailSeg->_headAndTail.Head); int tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail); if (headSeg == queue->_head && tailSeg == queue->_tail && headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) && headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail) && tailSeg_Head == Volatile.Read(ref tailSeg->_headAndTail.Head) && tailSeg_Tail == Volatile.Read(ref tailSeg->_headAndTail.Tail)) { return(GetCount(headSeg, headSeg_Head, headSeg_Tail) + GetCount(tailSeg, tailSeg_Head, tailSeg_Tail)); } } // Mutliple QueueSegments (requires locking) else { // Aquire cross-segment lock queue->_crossSegmentLock.Lock(); if (headSeg == queue->_head && tailSeg == queue->_tail) { int tailSeg_Head = Volatile.Read(ref tailSeg->_headAndTail.Head); int tailSeg_Tail = Volatile.Read(ref tailSeg->_headAndTail.Tail); if (headSeg_Head == Volatile.Read(ref headSeg->_headAndTail.Head) && headSeg_Tail == Volatile.Read(ref headSeg->_headAndTail.Tail) && tailSeg_Head == Volatile.Read(ref tailSeg->_headAndTail.Head) && tailSeg_Tail == Volatile.Read(ref tailSeg->_headAndTail.Tail)) { int count = GetCount(headSeg, headSeg_Head, headSeg_Tail) + GetCount(tailSeg, tailSeg_Head, tailSeg_Tail); for (QueueSegment *s = headSeg->_nextSegment; s != tailSeg; s = s->_nextSegment) { UDebug.Assert(s->_frozen, "Internal segment must be frozen as there's a following segment."); count += s->_headAndTail.Tail - s->FreezeOffset; } // Release cross-segment lock queue->_crossSegmentLock.Unlock(); return(count); } } // Release cross-segment lock queue->_crossSegmentLock.Unlock(); } spinner.SpinOnce(); } }