/// <summary>
/// Gets the head and tail information of the current contents of the queue.
/// After this call returns, the specified region can be enumerated any number
/// of times and will not change.
/// </summary>
private void SnapForObservation(out ConcurrentQueueSegment <T> head, out int headHead, out ConcurrentQueueSegment <T> tail, out int tailTail)
{
lock (_crossSegmentLock) // _head and _tail may only change while the lock is held.
{
// Snap the head and tail
head = _head;
tail = _tail;
Debug.Assert(head != null);
Debug.Assert(tail != null);
Debug.Assert(tail._nextSegment == null);
// Mark them and all segments in between as preserving, and ensure no additional items
// can be added to the tail.
for (ConcurrentQueueSegment <T> s = head; ; s = s._nextSegment)
{
s._preservedForObservation = true;
if (s == tail)
{
break;
}
Debug.Assert(s._frozenForEnqueues); // any non-tail should already be marked
}
tail.EnsureFrozenForEnqueues(); // we want to prevent the tailTail from moving
// At this point, any dequeues from any segment won't overwrite the value, and
// none of the existing segments can have new items enqueued.
headHead = Volatile.Read(ref head._headAndTail.Head);
tailTail = Volatile.Read(ref tail._headAndTail.Tail);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ConcurrentQueue{T}"/> class that contains elements copied
/// from the specified collection.
/// </summary>
/// <param name="collection">
/// The collection whose elements are copied to the new <see cref="ConcurrentQueue{T}"/>.
/// </param>
/// <exception cref="System.ArgumentNullException">The <paramref name="collection"/> argument is null.</exception>
public ConcurrentQueue(IEnumerable <T> collection)
{
if (collection == null)
{
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.collection);
}
_crossSegmentLock = new object();
// Determine the initial segment size. We'll use the default,
// unless the collection is known to be larger than that, in which
// case we round its length up to a power of 2, as all segments must
// be a power of 2 in length.
int length = InitialSegmentLength;
if (collection is ICollection <T> c)
{
int count = c.Count;
if (count > length)
{
length = Math.Min(ConcurrentQueueSegment <T> .RoundUpToPowerOf2(count), MaxSegmentLength);
}
}
// Initialize the segment and add all of the data to it.
_tail = _head = new ConcurrentQueueSegment <T>(length);
foreach (T item in collection)
{
Enqueue(item);
}
}
/// <summary>
/// Initializes the contents of the queue from an existing collection.
/// </summary>
/// <param name="collection">A collection from which to copy elements.</param>
private void InitializeFromCollection(IEnumerable <T> collection)
{
_crossSegmentLock = new object();
// Determine the initial segment size. We'll use the default,
// unless the collection is known to be larger than than, in which
// case we round its length up to a power of 2, as all segments must
// be a power of 2 in length.
int length = InitialSegmentLength;
var c = collection as ICollection <T>;
if (c != null)
{
int count = c.Count;
if (count > length)
{
length = Math.Min(ConcurrentQueueSegment <T> .RoundUpToPowerOf2(count), MaxSegmentLength);
}
}
// Initialize the segment and add all of the data to it.
_tail = _head = new ConcurrentQueueSegment <T>(length);
foreach (T item in collection)
{
Enqueue(item);
}
}
/// <summary>Computes the number of items in a segment based on a fixed head and tail in that segment.</summary>
private static int GetCount(ConcurrentQueueSegment <T> s, int head, int tail)
{
if (head != tail && head != tail - s.FreezeOffset)
{
head &= s._slotsMask;
tail &= s._slotsMask;
return(head < tail ? tail - head : s._slots.Length - head + tail);
}
return(0);
}
/// <summary>Gets the number of items in snapped region.</summary>
private static long GetCount(ConcurrentQueueSegment <T> head, int headHead, ConcurrentQueueSegment <T> tail, int tailTail)
{
// All of the segments should have been both frozen for enqueues and preserved for observation.
// Validate that here for head and tail; we'll validate it for intermediate segments later.
Debug.Assert(head._preservedForObservation);
Debug.Assert(head._frozenForEnqueues);
Debug.Assert(tail._preservedForObservation);
Debug.Assert(tail._frozenForEnqueues);
long count = 0;
// Head segment. We've already marked it as frozen for enqueues, so its tail position is fixed,
// and we've already marked it as preserved for observation (before we grabbed the head), so we
// can safely enumerate from its head to its tail and access its elements.
int headTail = (head == tail ? tailTail : Volatile.Read(ref head._headAndTail.Tail)) - head.FreezeOffset;
if (headHead < headTail)
{
// Mask the head and tail for the head segment
headHead &= head._slotsMask;
headTail &= head._slotsMask;
// Increase the count by either the one or two regions, based on whether tail
// has wrapped to be less than head.
count += headHead < headTail ?
headTail - headHead :
head._slots.Length - headHead + headTail;
}
// We've enumerated the head. If the tail is different from the head, we need to
// enumerate the remaining segments.
if (head != tail)
{
// Count the contents of each segment between head and tail, not including head and tail.
// Since there were segments before these, for our purposes we consider them to start at
// the 0th element, and since there is at least one segment after each, each was frozen
// by the time we snapped it, so we can iterate until each's frozen tail.
for (ConcurrentQueueSegment <T> s = head._nextSegment; s != tail; s = s._nextSegment)
{
Debug.Assert(s._preservedForObservation);
Debug.Assert(s._frozenForEnqueues);
count += s._headAndTail.Tail - s.FreezeOffset;
}
// Finally, enumerate the tail. As with the intermediate segments, there were segments
// before this in the snapped region, so we can start counting from the beginning. Unlike
// the intermediate segments, we can't just go until the Tail, as that could still be changing;
// instead we need to go until the tail we snapped for observation.
count += tailTail - tail.FreezeOffset;
}
// Return the computed count.
return(count);
}
private IEnumerator<T> Enumerate(ConcurrentQueueSegment<T> head, int headHead, ConcurrentQueueSegment<T> tail, int tailTail)
{
Debug.Assert(head._preservedForObservation);
Debug.Assert(head._frozenForEnqueues);
Debug.Assert(tail._preservedForObservation);
Debug.Assert(tail._frozenForEnqueues);
// Head segment. We've already marked it as not accepting any more enqueues,
// so its tail position is fixed, and we've already marked it as preserved for
// enumeration (before we grabbed its head), so we can safely enumerate from
// its head to its tail.
int headTail = (head == tail ? tailTail : Volatile.Read(ref head._headAndTail.Tail)) - head.FreezeOffset;
if (headHead < headTail)
{
headHead &= head._slotsMask;
headTail &= head._slotsMask;
if (headHead < headTail)
{
for (int i = headHead; i < headTail; i++) yield return GetItemWhenAvailable(head, i);
}
else
{
for (int i = headHead; i < head._slots.Length; i++) yield return GetItemWhenAvailable(head, i);
for (int i = 0; i < headTail; i++) yield return GetItemWhenAvailable(head, i);
}
}
// We've enumerated the head. If the tail is the same, we're done.
if (head != tail)
{
// Each segment between head and tail, not including head and tail. Since there were
// segments before these, for our purposes we consider it to start at the 0th element.
for (ConcurrentQueueSegment<T> s = head._nextSegment!; s != tail; s = s._nextSegment!)
{
Debug.Assert(s._preservedForObservation, "Would have had to been preserved as a segment part of enumeration");
Debug.Assert(s._frozenForEnqueues, "Would have had to be frozen for enqueues as it's intermediate");
int sTail = s._headAndTail.Tail - s.FreezeOffset;
for (int i = 0; i < sTail; i++)
{
yield return GetItemWhenAvailable(s, i);
}
}
// Enumerate the tail. Since there were segments before this, we can just start at
// its beginning, and iterate until the tail we already grabbed.
tailTail -= tail.FreezeOffset;
for (int i = 0; i < tailTail; i++)
{
yield return GetItemWhenAvailable(tail, i);
}
}
}
/// <summary>Attempts to retrieve the value for the first element in the queue.</summary>
/// <param name="result">The value of the first element, if found.</param>
/// <param name="resultUsed">true if the result is needed; otherwise false if only the true/false outcome is needed.</param>
/// <returns>true if an element was found; otherwise, false.</returns>
private bool TryPeek(out T result, bool resultUsed)
{
// Starting with the head segment, look through all of the segments
// for the first one we can find that's not empty.
ConcurrentQueueSegment <T> s = _head;
while (true)
{
// Grab the next segment from this one, before we peek.
// This is to be able to see whether the value has changed
// during the peek operation.
ConcurrentQueueSegment <T> next = Volatile.Read(ref s._nextSegment);
// Peek at the segment. If we find an element, we're done.
if (s.TryPeek(out result, resultUsed))
{
return(true);
}
// The current segment was empty at the moment we checked.
if (next != null)
{
// If prior to the peek there was already a next segment, then
// during the peek no additional items could have been enqueued
// to it and we can just move on to check the next segment.
Debug.Assert(next == s._nextSegment);
s = next;
}
else if (Volatile.Read(ref s._nextSegment) == null)
{
// The next segment is null. Nothing more to peek at.
break;
}
// The next segment was null before we peeked but non-null after.
// That means either when we peeked the first segment had
// already been frozen but the new segment not yet added,
// or that the first segment was empty and between the time
// that we peeked and then checked _nextSegment, so many items
// were enqueued that we filled the first segment and went
// into the next. Since we need to peek in order, we simply
// loop around again to peek on the same segment. The next
// time around on this segment we'll then either successfully
// peek or we'll find that next was non-null before peeking,
// and we'll traverse to that segment.
}
result = default(T);
return(false);
}
/// <summary>
/// Removes all objects from the <see cref="ConcurrentQueue{T}"/>.
/// </summary>
public void Clear()
{
lock (_crossSegmentLock)
{
// Simply substitute a new segment for the existing head/tail,
// as is done in the constructor. Operations currently in flight
// may still read from or write to an existing segment that's
// getting dropped, meaning that in flight operations may not be
// linear with regards to this clear operation. To help mitigate
// in-flight operations enqueuing onto the tail that's about to
// be dropped, we first freeze it; that'll force enqueuers to take
// this lock to synchronize and see the new tail.
_tail.EnsureFrozenForEnqueues();
_tail = _head = new ConcurrentQueueSegment <T>(InitialSegmentLength);
}
}
TryDequeueSlow(out result); // slow path that needs to fix up segments
/// <summary>Tries to dequeue an item, removing empty segments as needed.</summary>
private bool TryDequeueSlow(out T item)
{
while (true)
{
// Get the current head
ConcurrentQueueSegment <T> head = _head;
// Try to take. If we're successful, we're done.
if (head.TryDequeue(out item))
{
return(true);
}
// Check to see whether this segment is the last. If it is, we can consider
// this to be a moment-in-time empty condition (even though between the TryDequeue
// check and this check, another item could have arrived).
if (head._nextSegment == null)
{
item = default(T);
return(false);
}
// At this point we know that head.Next != null, which means
// this segment has been frozen for additional enqueues. But between
// the time that we ran TryDequeue and checked for a next segment,
// another item could have been added. Try to dequeue one more time
// to confirm that the segment is indeed empty.
Debug.Assert(head._frozenForEnqueues);
if (head.TryDequeue(out item))
{
return(true);
}
// This segment is frozen (nothing more can be added) and empty (nothing is in it).
// Update head to point to the next segment in the list, assuming no one's beat us to it.
lock (_crossSegmentLock)
{
if (head == _head)
{
_head = head._nextSegment;
}
}
}
}
TryDequeueSlow(out result); // slow path that needs to fix up segments
/// <summary>Tries to dequeue an item, removing empty segments as needed.</summary>
private bool TryDequeueSlow([MaybeNullWhen(false)] out T item)
{
while (true)
{
// Get the current head
ConcurrentQueueSegment<T> head = _head;
// Try to take. If we're successful, we're done.
if (head.TryDequeue(out item))
{
return true;
}
// Check to see whether this segment is the last. If it is, we can consider
// this to be a moment-in-time empty condition (even though between the TryDequeue
// check and this check, another item could have arrived).
if (head._nextSegment == null)
{
item = default!;
/// <summary>Adds to the end of the queue, adding a new segment if necessary.</summary>
private void EnqueueSlow(T item)
{
while (true)
{
ConcurrentQueueSegment <T> tail = _tail;
// Try to append to the existing tail.
if (tail.TryEnqueue(item))
{
return;
}
// If we were unsuccessful, take the lock so that we can compare and manipulate
// the tail. Assuming another enqueuer hasn't already added a new segment,
// do so, then loop around to try enqueueing again.
lock (_crossSegmentLock)
{
if (tail == _tail)
{
// Make sure no one else can enqueue to this segment.
tail.EnsureFrozenForEnqueues();
// We determine the new segment's length based on the old length.
// In general, we double the size of the segment, to make it less likely
// that we'll need to grow again. However, if the tail segment is marked
// as preserved for observation, something caused us to avoid reusing this
// segment, and if that happens a lot and we grow, we'll end up allocating
// lots of wasted space. As such, in such situations we reset back to the
// initial segment length; if these observations are happening frequently,
// this will help to avoid wasted memory, and if they're not, we'll
// relatively quickly grow again to a larger size.
int nextSize = tail._preservedForObservation ? InitialSegmentLength : Math.Min(tail.Capacity * 2, MaxSegmentLength);
var newTail = new ConcurrentQueueSegment <T>(nextSize);
// Hook up the new tail.
tail._nextSegment = newTail;
_tail = newTail;
}
}
}
}
/// <summary>Gets the item stored in the <paramref name="i"/>th entry in <paramref name="segment"/>.</summary>
private T GetItemWhenAvailable(ConcurrentQueueSegment <T> segment, int i)
{
Debug.Assert(segment._preservedForObservation);
// Get the expected value for the sequence number
int expectedSequenceNumberAndMask = (i + 1) & segment._slotsMask;
// If the expected sequence number is not yet written, we're still waiting for
// an enqueuer to finish storing it. Spin until it's there.
if ((segment._slots[i].SequenceNumber & segment._slotsMask) != expectedSequenceNumberAndMask)
{
var spinner = new SpinWait();
while ((Volatile.Read(ref segment._slots[i].SequenceNumber) & segment._slotsMask) != expectedSequenceNumberAndMask)
{
spinner.SpinOnce();
}
}
// Return the value from the slot.
return(segment._slots[i].Item);
}
private volatile ConcurrentQueueSegment <T> _head; // SOS's ThreadPool command depends on this name
/// <summary>
/// Initializes a new instance of the <see cref="ConcurrentQueue{T}"/> class.
/// </summary>
public ConcurrentQueue()
{
_crossSegmentLock = new object();
_tail = _head = new ConcurrentQueueSegment <T>(InitialSegmentLength);
}
