/// <summary> /// Helper method of GetEnumerator to separate out yield return statement, and prevent lazy evaluation. /// </summary> private IEnumerator <T> GetEnumerator(Segment head, Segment tail, int headLow, int tailHigh) { try { SpinWait spin = new SpinWait(); if (head == tail) { for (int i = headLow; i <= tailHigh; i++) { // If the position is reserved by an Enqueue operation, but the value is not written into, // spin until the value is available. spin.Reset(); while (!head._state[i]._value) { spin.SpinOnce(); } yield return(head._array[i]); } } else { //iterate on head segment for (int i = headLow; i < SEGMENT_SIZE; i++) { // If the position is reserved by an Enqueue operation, but the value is not written into, // spin until the value is available. spin.Reset(); while (!head._state[i]._value) { spin.SpinOnce(); } yield return(head._array[i]); } //iterate on middle segments Segment curr = head.Next; while (curr != tail) { for (int i = 0; i < SEGMENT_SIZE; i++) { // If the position is reserved by an Enqueue operation, but the value is not written into, // spin until the value is available. spin.Reset(); while (!curr._state[i]._value) { spin.SpinOnce(); } yield return(curr._array[i]); } curr = curr.Next; } //iterate on tail segment for (int i = 0; i <= tailHigh; i++) { // If the position is reserved by an Enqueue operation, but the value is not written into, // spin until the value is available. spin.Reset(); while (!tail._state[i]._value) { spin.SpinOnce(); } yield return(tail._array[i]); } } } finally { // This Decrement must happen after the enumeration is over. Interlocked.Decrement(ref _numSnapshotTakers); } }
bool TryTake(out T item, int milliseconds, CancellationToken cancellationToken, bool throwComplete) { if (milliseconds < -1) { throw new ArgumentOutOfRangeException("milliseconds"); } item = default(T); SpinWait sw = new SpinWait(); long start = milliseconds == -1 ? 0 : watch.ElapsedMilliseconds; do { cancellationToken.ThrowIfCancellationRequested(); int cachedRemoveId = removeId; int cachedAddId = addId; // Empty case if (cachedRemoveId == cachedAddId) { if (milliseconds == 0) { return(false); } if (IsCompleted) { if (throwComplete) { ThrowCompleteException(); } else { return(false); } } if (sw.Count <= spinCount) { sw.SpinOnce(); } else { mreAdd.Reset(); if (cachedRemoveId != removeId || cachedAddId != addId) { mreAdd.Set(); continue; } mreAdd.Wait(ComputeTimeout(milliseconds, start), cancellationToken); } continue; } if (Interlocked.CompareExchange(ref removeId, cachedRemoveId + 1, cachedRemoveId) != cachedRemoveId) { continue; } while (!underlyingColl.TryTake(out item)) { ; } mreRemove.Set(); return(true); } while (milliseconds == -1 || (watch.ElapsedMilliseconds - start) < milliseconds); return(false); }
/// <summary> /// Slow path helper for TryPop. This method assumes an initial attempt to pop an element /// has already occurred and failed, so it begins spinning right away. /// </summary> /// <param name="count">The number of items to pop.</param> /// <param name="poppedHead"> /// When this method returns, if the pop succeeded, contains the removed object. If no object was /// available to be removed, the value is unspecified. This parameter is passed uninitialized. /// </param> /// <returns>The number of objects successfully popped from the top of /// the <see cref="ConcurrentStack{T}"/>.</returns> private int TryPopCore(int count, [NotNullWhen(true)] out Node?poppedHead) { SpinWait spin = new SpinWait(); // Try to CAS the head with its current next. We stop when we succeed or // when we notice that the stack is empty, whichever comes first. Node? head; Node next; int backoff = 1; Random?r = null; while (true) { head = _head; // Is the stack empty? if (head == null) { if (count == 1 && CDSCollectionETWBCLProvider.Log.IsEnabled()) { CDSCollectionETWBCLProvider.Log.ConcurrentStack_FastPopFailed(spin.Count); } poppedHead = null; return(0); } next = head; int nodesCount = 1; for (; nodesCount < count && next._next != null; nodesCount++) { next = next._next; } // Try to swap the new head. If we succeed, break out of the loop. if (Interlocked.CompareExchange(ref _head, next._next, head) == head) { if (count == 1 && CDSCollectionETWBCLProvider.Log.IsEnabled()) { CDSCollectionETWBCLProvider.Log.ConcurrentStack_FastPopFailed(spin.Count); } // Return the popped Node. poppedHead = head; return(nodesCount); } // We failed to CAS the new head. Spin briefly and retry. for (int i = 0; i < backoff; i++) { spin.SpinOnce(sleep1Threshold: -1); } if (spin.NextSpinWillYield) { if (r == null) { r = new Random(); } backoff = r.Next(1, BACKOFF_MAX_YIELDS); } else { backoff *= 2; } } }
public bool TryAdd(T item, int millisecondsTimeout, CancellationToken cancellationToken) { if (millisecondsTimeout < -1) { throw new ArgumentOutOfRangeException("millisecondsTimeout"); } long start = millisecondsTimeout == -1 ? 0 : watch.ElapsedMilliseconds; SpinWait sw = new SpinWait(); do { cancellationToken.ThrowIfCancellationRequested(); int cachedAddId = addId; int cachedRemoveId = removeId; int itemsIn = cachedAddId - cachedRemoveId; // Check our transaction id against completed stored one if (isComplete.Value && cachedAddId >= completeId) { ThrowCompleteException(); } // If needed, we check and wait that the collection isn't full if (upperBound != -1 && itemsIn >= upperBound) { if (millisecondsTimeout == 0) { return(false); } if (sw.Count <= spinCount) { sw.SpinOnce(); } else { mreRemove.Reset(); if (cachedRemoveId != removeId || cachedAddId != addId) { mreRemove.Set(); continue; } mreRemove.Wait(ComputeTimeout(millisecondsTimeout, start), cancellationToken); } continue; } // Validate the steps we have been doing until now if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) != cachedAddId) { continue; } // We have a slot reserved in the underlying collection, try to take it if (!underlyingColl.TryAdd(item)) { throw new InvalidOperationException("The underlying collection didn't accept the item."); } // Wake up process that may have been sleeping mreAdd.Set(); return(true); } while (millisecondsTimeout == -1 || (watch.ElapsedMilliseconds - start) < millisecondsTimeout); return(false); }
/// <summary>Tries to dequeue an element from the queue.</summary> public bool TryDequeue(out T item) { Slot[] slots = _slots; // Loop in case of contention... var spinner = new SpinWait(); while (true) { // Get the head at which to try to dequeue. int currentHead = Volatile.Read(ref _headAndTail.Head); int slotsIndex = currentHead & _slotsMask; // Read the sequence number for the head position. int sequenceNumber = Volatile.Read(ref slots[slotsIndex].SequenceNumber); // We can dequeue from this slot if it's been filled by an enqueuer, which // would have left the sequence number at pos+1. int diff = sequenceNumber - (currentHead + 1); if (diff == 0) { // We may be racing with other dequeuers. Try to reserve the slot by incrementing // the head. Once we've done that, no one else will be able to read from this slot, // and no enqueuer will be able to read from this slot until we've written the new // sequence number. WARNING: The next few lines are not reliable on a runtime that // supports thread aborts. If a thread abort were to sneak in after the CompareExchange // but before the Volatile.Write, enqueuers trying to enqueue into this slot would // spin indefinitely. If this implementation is ever used on such a platform, this // if block should be wrapped in a finally / prepared region. if (Interlocked.CompareExchange(ref _headAndTail.Head, currentHead + 1, currentHead) == currentHead) { // Successfully reserved the slot. Note that after the above CompareExchange, other threads // trying to dequeue from this slot will end up spinning until we do the subsequent Write. item = slots[slotsIndex].Item; if (!Volatile.Read(ref _preservedForObservation)) { // If we're preserving, though, we don't zero out the slot, as we need it for // enumerations, peeking, ToArray, etc. And we don't update the sequence number, // so that an enqueuer will see it as full and be forced to move to a new segment. slots[slotsIndex].Item = default(T); Volatile.Write(ref slots[slotsIndex].SequenceNumber, currentHead + slots.Length); } return(true); } } else if (diff < 0) { // The sequence number was less than what we needed, which means this slot doesn't // yet contain a value we can dequeue, i.e. the segment is empty. Technically it's // possible that multiple enqueuers could have written concurrently, with those // getting later slots actually finishing first, so there could be elements after // this one that are available, but we need to dequeue in order. So before declaring // failure and that the segment is empty, we check the tail to see if we're actually // empty or if we're just waiting for items in flight or after this one to become available. bool frozen = _frozenForEnqueues; int currentTail = Volatile.Read(ref _headAndTail.Tail); if (currentTail - currentHead <= 0 || (frozen && (currentTail - FreezeOffset - currentHead <= 0))) { item = default(T); return(false); } // It's possible it could have become frozen after we checked _frozenForEnqueues // and before reading the tail. That's ok: in that rare race condition, we just // loop around again. } // Lost a race. Spin a bit, then try again. spinner.SpinOnce(sleep1Threshold: -1); } }
public void Add(T item, CancellationToken token) { SpinWait sw = new SpinWait(); long cachedAddId; while (true) { token.ThrowIfCancellationRequested(); cachedAddId = addId; long cachedRemoveId = removeId; if (upperBound != -1) { if (cachedAddId - cachedRemoveId > upperBound) { if (sw.Count <= spinCount) { sw.SpinOnce(); } else { if (mreRemove.IsSet) { continue; } if (cachedRemoveId != removeId) { continue; } mreRemove.Wait(token); mreRemove.Reset(); } continue; } } // Check our transaction id against completed stored one if (isComplete.Value && cachedAddId >= completeId) { ThrowCompleteException(); } if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) == cachedAddId) { break; } } if (isComplete.Value && cachedAddId >= completeId) { ThrowCompleteException(); } while (!underlyingColl.TryAdd(item)) { ; } if (!mreAdd.IsSet) { mreAdd.Set(); } }
public T Take(CancellationToken token) { SpinWait sw = new SpinWait(); while (true) { token.ThrowIfCancellationRequested(); long cachedRemoveId = removeId; long cachedAddId = addId; // Empty case if (cachedRemoveId == cachedAddId) { if (IsCompleted) { ThrowCompleteException(); } if (sw.Count <= spinCount) { sw.SpinOnce(); } else { if (cachedAddId != addId) { continue; } if (IsCompleted) { ThrowCompleteException(); } mreAdd.Wait(token); mreAdd.Reset(); } continue; } if (Interlocked.CompareExchange(ref removeId, cachedRemoveId + 1, cachedRemoveId) == cachedRemoveId) { break; } } T item; while (!underlyingColl.TryTake(out item)) { ; } if (!mreRemove.IsSet) { mreRemove.Set(); } return(item); }
public bool TryAdd(T item, int millisecondsTimeout, CancellationToken cancellationToken) { if (millisecondsTimeout < -1) { throw new ArgumentOutOfRangeException("millisecondsTimeout"); } long start = millisecondsTimeout == -1 ? 0 : watch.ElapsedMilliseconds; SpinWait sw = new SpinWait(); do { cancellationToken.ThrowIfCancellationRequested(); int cachedAddId = addId; int cachedRemoveId = removeId; int itemsIn = cachedAddId - cachedRemoveId; if (isComplete.Value && cachedAddId >= completeId) { ThrowCompleteException(); } if (upperBound != -1 && itemsIn >= upperBound) { if (millisecondsTimeout == 0) { return(false); } if (sw.Count <= spinCount) { sw.SpinOnce(); } else { mreRemove.Reset(); if (cachedRemoveId != removeId || cachedAddId != addId) { mreRemove.Set(); continue; } mreRemove.Wait(ComputeTimeout(millisecondsTimeout, start), cancellationToken); } continue; } if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) != cachedAddId) { continue; } if (!underlyingColl.TryAdd(item)) { throw new InvalidOperationException("The underlying collection didn't accept the item."); } mreAdd.Set(); return(true); } while (millisecondsTimeout == -1 || (watch.ElapsedMilliseconds - start) < millisecondsTimeout); return(false); }