/// <summary> /// Gets the head and tail information of the current contents of the queue. /// After this call returns, the specified region can be enumerated any number /// of times and will not change. /// </summary> private void SnapForObservation(out ConcurrentQueueSegment <T> head, out int headHead, out ConcurrentQueueSegment <T> tail, out int tailTail) { lock (_crossSegmentLock) // _head and _tail may only change while the lock is held. { // Snap the head and tail head = _head; tail = _tail; Debug.Assert(head != null); Debug.Assert(tail != null); Debug.Assert(tail._nextSegment == null); // Mark them and all segments in between as preserving, and ensure no additional items // can be added to the tail. for (ConcurrentQueueSegment <T> s = head; ; s = s._nextSegment) { s._preservedForObservation = true; if (s == tail) { break; } Debug.Assert(s._frozenForEnqueues); // any non-tail should already be marked } tail.EnsureFrozenForEnqueues(); // we want to prevent the tailTail from moving // At this point, any dequeues from any segment won't overwrite the value, and // none of the existing segments can have new items enqueued. headHead = Volatile.Read(ref head._headAndTail.Head); tailTail = Volatile.Read(ref tail._headAndTail.Tail); } }
/// <summary> /// Initializes a new instance of the <see cref="ConcurrentQueue{T}"/> class that contains elements copied /// from the specified collection. /// </summary> /// <param name="collection"> /// The collection whose elements are copied to the new <see cref="ConcurrentQueue{T}"/>. /// </param> /// <exception cref="System.ArgumentNullException">The <paramref name="collection"/> argument is null.</exception> public ConcurrentQueue(IEnumerable <T> collection) { if (collection == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.collection); } _crossSegmentLock = new object(); // Determine the initial segment size. We'll use the default, // unless the collection is known to be larger than that, in which // case we round its length up to a power of 2, as all segments must // be a power of 2 in length. int length = InitialSegmentLength; if (collection is ICollection <T> c) { int count = c.Count; if (count > length) { length = Math.Min(ConcurrentQueueSegment <T> .RoundUpToPowerOf2(count), MaxSegmentLength); } } // Initialize the segment and add all of the data to it. _tail = _head = new ConcurrentQueueSegment <T>(length); foreach (T item in collection) { Enqueue(item); } }
/// <summary> /// Initializes the contents of the queue from an existing collection. /// </summary> /// <param name="collection">A collection from which to copy elements.</param> private void InitializeFromCollection(IEnumerable <T> collection) { _crossSegmentLock = new object(); // Determine the initial segment size. We'll use the default, // unless the collection is known to be larger than than, in which // case we round its length up to a power of 2, as all segments must // be a power of 2 in length. int length = InitialSegmentLength; var c = collection as ICollection <T>; if (c != null) { int count = c.Count; if (count > length) { length = Math.Min(ConcurrentQueueSegment <T> .RoundUpToPowerOf2(count), MaxSegmentLength); } } // Initialize the segment and add all of the data to it. _tail = _head = new ConcurrentQueueSegment <T>(length); foreach (T item in collection) { Enqueue(item); } }
/// <summary>Computes the number of items in a segment based on a fixed head and tail in that segment.</summary> private static int GetCount(ConcurrentQueueSegment <T> s, int head, int tail) { if (head != tail && head != tail - s.FreezeOffset) { head &= s._slotsMask; tail &= s._slotsMask; return(head < tail ? tail - head : s._slots.Length - head + tail); } return(0); }
/// <summary>Gets the number of items in snapped region.</summary> private static long GetCount(ConcurrentQueueSegment <T> head, int headHead, ConcurrentQueueSegment <T> tail, int tailTail) { // All of the segments should have been both frozen for enqueues and preserved for observation. // Validate that here for head and tail; we'll validate it for intermediate segments later. Debug.Assert(head._preservedForObservation); Debug.Assert(head._frozenForEnqueues); Debug.Assert(tail._preservedForObservation); Debug.Assert(tail._frozenForEnqueues); long count = 0; // Head segment. We've already marked it as frozen for enqueues, so its tail position is fixed, // and we've already marked it as preserved for observation (before we grabbed the head), so we // can safely enumerate from its head to its tail and access its elements. int headTail = (head == tail ? tailTail : Volatile.Read(ref head._headAndTail.Tail)) - head.FreezeOffset; if (headHead < headTail) { // Mask the head and tail for the head segment headHead &= head._slotsMask; headTail &= head._slotsMask; // Increase the count by either the one or two regions, based on whether tail // has wrapped to be less than head. count += headHead < headTail ? headTail - headHead : head._slots.Length - headHead + headTail; } // We've enumerated the head. If the tail is different from the head, we need to // enumerate the remaining segments. if (head != tail) { // Count the contents of each segment between head and tail, not including head and tail. // Since there were segments before these, for our purposes we consider them to start at // the 0th element, and since there is at least one segment after each, each was frozen // by the time we snapped it, so we can iterate until each's frozen tail. for (ConcurrentQueueSegment <T> s = head._nextSegment; s != tail; s = s._nextSegment) { Debug.Assert(s._preservedForObservation); Debug.Assert(s._frozenForEnqueues); count += s._headAndTail.Tail - s.FreezeOffset; } // Finally, enumerate the tail. As with the intermediate segments, there were segments // before this in the snapped region, so we can start counting from the beginning. Unlike // the intermediate segments, we can't just go until the Tail, as that could still be changing; // instead we need to go until the tail we snapped for observation. count += tailTail - tail.FreezeOffset; } // Return the computed count. return(count); }
private IEnumerator<T> Enumerate(ConcurrentQueueSegment<T> head, int headHead, ConcurrentQueueSegment<T> tail, int tailTail) { Debug.Assert(head._preservedForObservation); Debug.Assert(head._frozenForEnqueues); Debug.Assert(tail._preservedForObservation); Debug.Assert(tail._frozenForEnqueues); // Head segment. We've already marked it as not accepting any more enqueues, // so its tail position is fixed, and we've already marked it as preserved for // enumeration (before we grabbed its head), so we can safely enumerate from // its head to its tail. int headTail = (head == tail ? tailTail : Volatile.Read(ref head._headAndTail.Tail)) - head.FreezeOffset; if (headHead < headTail) { headHead &= head._slotsMask; headTail &= head._slotsMask; if (headHead < headTail) { for (int i = headHead; i < headTail; i++) yield return GetItemWhenAvailable(head, i); } else { for (int i = headHead; i < head._slots.Length; i++) yield return GetItemWhenAvailable(head, i); for (int i = 0; i < headTail; i++) yield return GetItemWhenAvailable(head, i); } } // We've enumerated the head. If the tail is the same, we're done. if (head != tail) { // Each segment between head and tail, not including head and tail. Since there were // segments before these, for our purposes we consider it to start at the 0th element. for (ConcurrentQueueSegment<T> s = head._nextSegment!; s != tail; s = s._nextSegment!) { Debug.Assert(s._preservedForObservation, "Would have had to been preserved as a segment part of enumeration"); Debug.Assert(s._frozenForEnqueues, "Would have had to be frozen for enqueues as it's intermediate"); int sTail = s._headAndTail.Tail - s.FreezeOffset; for (int i = 0; i < sTail; i++) { yield return GetItemWhenAvailable(s, i); } } // Enumerate the tail. Since there were segments before this, we can just start at // its beginning, and iterate until the tail we already grabbed. tailTail -= tail.FreezeOffset; for (int i = 0; i < tailTail; i++) { yield return GetItemWhenAvailable(tail, i); } } }
/// <summary>Attempts to retrieve the value for the first element in the queue.</summary> /// <param name="result">The value of the first element, if found.</param> /// <param name="resultUsed">true if the result is needed; otherwise false if only the true/false outcome is needed.</param> /// <returns>true if an element was found; otherwise, false.</returns> private bool TryPeek(out T result, bool resultUsed) { // Starting with the head segment, look through all of the segments // for the first one we can find that's not empty. ConcurrentQueueSegment <T> s = _head; while (true) { // Grab the next segment from this one, before we peek. // This is to be able to see whether the value has changed // during the peek operation. ConcurrentQueueSegment <T> next = Volatile.Read(ref s._nextSegment); // Peek at the segment. If we find an element, we're done. if (s.TryPeek(out result, resultUsed)) { return(true); } // The current segment was empty at the moment we checked. if (next != null) { // If prior to the peek there was already a next segment, then // during the peek no additional items could have been enqueued // to it and we can just move on to check the next segment. Debug.Assert(next == s._nextSegment); s = next; } else if (Volatile.Read(ref s._nextSegment) == null) { // The next segment is null. Nothing more to peek at. break; } // The next segment was null before we peeked but non-null after. // That means either when we peeked the first segment had // already been frozen but the new segment not yet added, // or that the first segment was empty and between the time // that we peeked and then checked _nextSegment, so many items // were enqueued that we filled the first segment and went // into the next. Since we need to peek in order, we simply // loop around again to peek on the same segment. The next // time around on this segment we'll then either successfully // peek or we'll find that next was non-null before peeking, // and we'll traverse to that segment. } result = default(T); return(false); }
/// <summary> /// Removes all objects from the <see cref="ConcurrentQueue{T}"/>. /// </summary> public void Clear() { lock (_crossSegmentLock) { // Simply substitute a new segment for the existing head/tail, // as is done in the constructor. Operations currently in flight // may still read from or write to an existing segment that's // getting dropped, meaning that in flight operations may not be // linear with regards to this clear operation. To help mitigate // in-flight operations enqueuing onto the tail that's about to // be dropped, we first freeze it; that'll force enqueuers to take // this lock to synchronize and see the new tail. _tail.EnsureFrozenForEnqueues(); _tail = _head = new ConcurrentQueueSegment <T>(InitialSegmentLength); } }
TryDequeueSlow(out result); // slow path that needs to fix up segments /// <summary>Tries to dequeue an item, removing empty segments as needed.</summary> private bool TryDequeueSlow(out T item) { while (true) { // Get the current head ConcurrentQueueSegment <T> head = _head; // Try to take. If we're successful, we're done. if (head.TryDequeue(out item)) { return(true); } // Check to see whether this segment is the last. If it is, we can consider // this to be a moment-in-time empty condition (even though between the TryDequeue // check and this check, another item could have arrived). if (head._nextSegment == null) { item = default(T); return(false); } // At this point we know that head.Next != null, which means // this segment has been frozen for additional enqueues. But between // the time that we ran TryDequeue and checked for a next segment, // another item could have been added. Try to dequeue one more time // to confirm that the segment is indeed empty. Debug.Assert(head._frozenForEnqueues); if (head.TryDequeue(out item)) { return(true); } // This segment is frozen (nothing more can be added) and empty (nothing is in it). // Update head to point to the next segment in the list, assuming no one's beat us to it. lock (_crossSegmentLock) { if (head == _head) { _head = head._nextSegment; } } } }
TryDequeueSlow(out result); // slow path that needs to fix up segments /// <summary>Tries to dequeue an item, removing empty segments as needed.</summary> private bool TryDequeueSlow([MaybeNullWhen(false)] out T item) { while (true) { // Get the current head ConcurrentQueueSegment<T> head = _head; // Try to take. If we're successful, we're done. if (head.TryDequeue(out item)) { return true; } // Check to see whether this segment is the last. If it is, we can consider // this to be a moment-in-time empty condition (even though between the TryDequeue // check and this check, another item could have arrived). if (head._nextSegment == null) { item = default!;
/// <summary>Adds to the end of the queue, adding a new segment if necessary.</summary> private void EnqueueSlow(T item) { while (true) { ConcurrentQueueSegment <T> tail = _tail; // Try to append to the existing tail. if (tail.TryEnqueue(item)) { return; } // If we were unsuccessful, take the lock so that we can compare and manipulate // the tail. Assuming another enqueuer hasn't already added a new segment, // do so, then loop around to try enqueueing again. lock (_crossSegmentLock) { if (tail == _tail) { // Make sure no one else can enqueue to this segment. tail.EnsureFrozenForEnqueues(); // We determine the new segment's length based on the old length. // In general, we double the size of the segment, to make it less likely // that we'll need to grow again. However, if the tail segment is marked // as preserved for observation, something caused us to avoid reusing this // segment, and if that happens a lot and we grow, we'll end up allocating // lots of wasted space. As such, in such situations we reset back to the // initial segment length; if these observations are happening frequently, // this will help to avoid wasted memory, and if they're not, we'll // relatively quickly grow again to a larger size. int nextSize = tail._preservedForObservation ? InitialSegmentLength : Math.Min(tail.Capacity * 2, MaxSegmentLength); var newTail = new ConcurrentQueueSegment <T>(nextSize); // Hook up the new tail. tail._nextSegment = newTail; _tail = newTail; } } } }
/// <summary>Gets the item stored in the <paramref name="i"/>th entry in <paramref name="segment"/>.</summary> private T GetItemWhenAvailable(ConcurrentQueueSegment <T> segment, int i) { Debug.Assert(segment._preservedForObservation); // Get the expected value for the sequence number int expectedSequenceNumberAndMask = (i + 1) & segment._slotsMask; // If the expected sequence number is not yet written, we're still waiting for // an enqueuer to finish storing it. Spin until it's there. if ((segment._slots[i].SequenceNumber & segment._slotsMask) != expectedSequenceNumberAndMask) { var spinner = new SpinWait(); while ((Volatile.Read(ref segment._slots[i].SequenceNumber) & segment._slotsMask) != expectedSequenceNumberAndMask) { spinner.SpinOnce(); } } // Return the value from the slot. return(segment._slots[i].Item); }
private volatile ConcurrentQueueSegment <T> _head; // SOS's ThreadPool command depends on this name /// <summary> /// Initializes a new instance of the <see cref="ConcurrentQueue{T}"/> class. /// </summary> public ConcurrentQueue() { _crossSegmentLock = new object(); _tail = _head = new ConcurrentQueueSegment <T>(InitialSegmentLength); }