/// <summary>
/// Helper method of GetEnumerator to separate out yield return statement, and prevent lazy evaluation.
/// </summary>
private IEnumerator <T> GetEnumerator(Segment head, Segment tail, int headLow, int tailHigh)
{
try
{
SpinWait spin = new SpinWait();
if (head == tail)
{
for (int i = headLow; i <= tailHigh; i++)
{
// If the position is reserved by an Enqueue operation, but the value is not written into,
// spin until the value is available.
spin.Reset();
while (!head._state[i]._value)
{
spin.SpinOnce();
}
yield return(head._array[i]);
}
}
else
{
//iterate on head segment
for (int i = headLow; i < SEGMENT_SIZE; i++)
{
// If the position is reserved by an Enqueue operation, but the value is not written into,
// spin until the value is available.
spin.Reset();
while (!head._state[i]._value)
{
spin.SpinOnce();
}
yield return(head._array[i]);
}
//iterate on middle segments
Segment curr = head.Next;
while (curr != tail)
{
for (int i = 0; i < SEGMENT_SIZE; i++)
{
// If the position is reserved by an Enqueue operation, but the value is not written into,
// spin until the value is available.
spin.Reset();
while (!curr._state[i]._value)
{
spin.SpinOnce();
}
yield return(curr._array[i]);
}
curr = curr.Next;
}
//iterate on tail segment
for (int i = 0; i <= tailHigh; i++)
{
// If the position is reserved by an Enqueue operation, but the value is not written into,
// spin until the value is available.
spin.Reset();
while (!tail._state[i]._value)
{
spin.SpinOnce();
}
yield return(tail._array[i]);
}
}
}
finally
{
// This Decrement must happen after the enumeration is over.
Interlocked.Decrement(ref _numSnapshotTakers);
}
}
bool TryTake(out T item, int milliseconds, CancellationToken cancellationToken, bool throwComplete)
{
if (milliseconds < -1)
{
throw new ArgumentOutOfRangeException("milliseconds");
}
item = default(T);
SpinWait sw = new SpinWait();
long start = milliseconds == -1 ? 0 : watch.ElapsedMilliseconds;
do
{
cancellationToken.ThrowIfCancellationRequested();
int cachedRemoveId = removeId;
int cachedAddId = addId;
// Empty case
if (cachedRemoveId == cachedAddId)
{
if (milliseconds == 0)
{
return(false);
}
if (IsCompleted)
{
if (throwComplete)
{
ThrowCompleteException();
}
else
{
return(false);
}
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
mreAdd.Reset();
if (cachedRemoveId != removeId || cachedAddId != addId)
{
mreAdd.Set();
continue;
}
mreAdd.Wait(ComputeTimeout(milliseconds, start), cancellationToken);
}
continue;
}
if (Interlocked.CompareExchange(ref removeId, cachedRemoveId + 1, cachedRemoveId) != cachedRemoveId)
{
continue;
}
while (!underlyingColl.TryTake(out item))
{
;
}
mreRemove.Set();
return(true);
} while (milliseconds == -1 || (watch.ElapsedMilliseconds - start) < milliseconds);
return(false);
}
/// <summary>
/// Slow path helper for TryPop. This method assumes an initial attempt to pop an element
/// has already occurred and failed, so it begins spinning right away.
/// </summary>
/// <param name="count">The number of items to pop.</param>
/// <param name="poppedHead">
/// When this method returns, if the pop succeeded, contains the removed object. If no object was
/// available to be removed, the value is unspecified. This parameter is passed uninitialized.
/// </param>
/// <returns>The number of objects successfully popped from the top of
/// the <see cref="ConcurrentStack{T}"/>.</returns>
private int TryPopCore(int count, [NotNullWhen(true)] out Node?poppedHead)
{
SpinWait spin = new SpinWait();
// Try to CAS the head with its current next. We stop when we succeed or
// when we notice that the stack is empty, whichever comes first.
Node? head;
Node next;
int backoff = 1;
Random?r = null;
while (true)
{
head = _head;
// Is the stack empty?
if (head == null)
{
if (count == 1 && CDSCollectionETWBCLProvider.Log.IsEnabled())
{
CDSCollectionETWBCLProvider.Log.ConcurrentStack_FastPopFailed(spin.Count);
}
poppedHead = null;
return(0);
}
next = head;
int nodesCount = 1;
for (; nodesCount < count && next._next != null; nodesCount++)
{
next = next._next;
}
// Try to swap the new head. If we succeed, break out of the loop.
if (Interlocked.CompareExchange(ref _head, next._next, head) == head)
{
if (count == 1 && CDSCollectionETWBCLProvider.Log.IsEnabled())
{
CDSCollectionETWBCLProvider.Log.ConcurrentStack_FastPopFailed(spin.Count);
}
// Return the popped Node.
poppedHead = head;
return(nodesCount);
}
// We failed to CAS the new head. Spin briefly and retry.
for (int i = 0; i < backoff; i++)
{
spin.SpinOnce(sleep1Threshold: -1);
}
if (spin.NextSpinWillYield)
{
if (r == null)
{
r = new Random();
}
backoff = r.Next(1, BACKOFF_MAX_YIELDS);
}
else
{
backoff *= 2;
}
}
}
public bool TryAdd(T item, int millisecondsTimeout, CancellationToken cancellationToken)
{
if (millisecondsTimeout < -1)
{
throw new ArgumentOutOfRangeException("millisecondsTimeout");
}
long start = millisecondsTimeout == -1 ? 0 : watch.ElapsedMilliseconds;
SpinWait sw = new SpinWait();
do
{
cancellationToken.ThrowIfCancellationRequested();
int cachedAddId = addId;
int cachedRemoveId = removeId;
int itemsIn = cachedAddId - cachedRemoveId;
// Check our transaction id against completed stored one
if (isComplete.Value && cachedAddId >= completeId)
{
ThrowCompleteException();
}
// If needed, we check and wait that the collection isn't full
if (upperBound != -1 && itemsIn >= upperBound)
{
if (millisecondsTimeout == 0)
{
return(false);
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
mreRemove.Reset();
if (cachedRemoveId != removeId || cachedAddId != addId)
{
mreRemove.Set();
continue;
}
mreRemove.Wait(ComputeTimeout(millisecondsTimeout, start), cancellationToken);
}
continue;
}
// Validate the steps we have been doing until now
if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) != cachedAddId)
{
continue;
}
// We have a slot reserved in the underlying collection, try to take it
if (!underlyingColl.TryAdd(item))
{
throw new InvalidOperationException("The underlying collection didn't accept the item.");
}
// Wake up process that may have been sleeping
mreAdd.Set();
return(true);
} while (millisecondsTimeout == -1 || (watch.ElapsedMilliseconds - start) < millisecondsTimeout);
return(false);
}
/// <summary>Tries to dequeue an element from the queue.</summary>
public bool TryDequeue(out T item)
{
Slot[] slots = _slots;
// Loop in case of contention...
var spinner = new SpinWait();
while (true)
{
// Get the head at which to try to dequeue.
int currentHead = Volatile.Read(ref _headAndTail.Head);
int slotsIndex = currentHead & _slotsMask;
// Read the sequence number for the head position.
int sequenceNumber = Volatile.Read(ref slots[slotsIndex].SequenceNumber);
// We can dequeue from this slot if it's been filled by an enqueuer, which
// would have left the sequence number at pos+1.
int diff = sequenceNumber - (currentHead + 1);
if (diff == 0)
{
// We may be racing with other dequeuers. Try to reserve the slot by incrementing
// the head. Once we've done that, no one else will be able to read from this slot,
// and no enqueuer will be able to read from this slot until we've written the new
// sequence number. WARNING: The next few lines are not reliable on a runtime that
// supports thread aborts. If a thread abort were to sneak in after the CompareExchange
// but before the Volatile.Write, enqueuers trying to enqueue into this slot would
// spin indefinitely. If this implementation is ever used on such a platform, this
// if block should be wrapped in a finally / prepared region.
if (Interlocked.CompareExchange(ref _headAndTail.Head, currentHead + 1, currentHead) == currentHead)
{
// Successfully reserved the slot. Note that after the above CompareExchange, other threads
// trying to dequeue from this slot will end up spinning until we do the subsequent Write.
item = slots[slotsIndex].Item;
if (!Volatile.Read(ref _preservedForObservation))
{
// If we're preserving, though, we don't zero out the slot, as we need it for
// enumerations, peeking, ToArray, etc. And we don't update the sequence number,
// so that an enqueuer will see it as full and be forced to move to a new segment.
slots[slotsIndex].Item = default(T);
Volatile.Write(ref slots[slotsIndex].SequenceNumber, currentHead + slots.Length);
}
return(true);
}
}
else if (diff < 0)
{
// The sequence number was less than what we needed, which means this slot doesn't
// yet contain a value we can dequeue, i.e. the segment is empty. Technically it's
// possible that multiple enqueuers could have written concurrently, with those
// getting later slots actually finishing first, so there could be elements after
// this one that are available, but we need to dequeue in order. So before declaring
// failure and that the segment is empty, we check the tail to see if we're actually
// empty or if we're just waiting for items in flight or after this one to become available.
bool frozen = _frozenForEnqueues;
int currentTail = Volatile.Read(ref _headAndTail.Tail);
if (currentTail - currentHead <= 0 || (frozen && (currentTail - FreezeOffset - currentHead <= 0)))
{
item = default(T);
return(false);
}
// It's possible it could have become frozen after we checked _frozenForEnqueues
// and before reading the tail. That's ok: in that rare race condition, we just
// loop around again.
}
// Lost a race. Spin a bit, then try again.
spinner.SpinOnce(sleep1Threshold: -1);
}
}
public void Add(T item, CancellationToken token)
{
SpinWait sw = new SpinWait();
long cachedAddId;
while (true)
{
token.ThrowIfCancellationRequested();
cachedAddId = addId;
long cachedRemoveId = removeId;
if (upperBound != -1)
{
if (cachedAddId - cachedRemoveId > upperBound)
{
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
if (mreRemove.IsSet)
{
continue;
}
if (cachedRemoveId != removeId)
{
continue;
}
mreRemove.Wait(token);
mreRemove.Reset();
}
continue;
}
}
// Check our transaction id against completed stored one
if (isComplete.Value && cachedAddId >= completeId)
{
ThrowCompleteException();
}
if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) == cachedAddId)
{
break;
}
}
if (isComplete.Value && cachedAddId >= completeId)
{
ThrowCompleteException();
}
while (!underlyingColl.TryAdd(item))
{
;
}
if (!mreAdd.IsSet)
{
mreAdd.Set();
}
}
public T Take(CancellationToken token)
{
SpinWait sw = new SpinWait();
while (true)
{
token.ThrowIfCancellationRequested();
long cachedRemoveId = removeId;
long cachedAddId = addId;
// Empty case
if (cachedRemoveId == cachedAddId)
{
if (IsCompleted)
{
ThrowCompleteException();
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
if (cachedAddId != addId)
{
continue;
}
if (IsCompleted)
{
ThrowCompleteException();
}
mreAdd.Wait(token);
mreAdd.Reset();
}
continue;
}
if (Interlocked.CompareExchange(ref removeId, cachedRemoveId + 1, cachedRemoveId) == cachedRemoveId)
{
break;
}
}
T item;
while (!underlyingColl.TryTake(out item))
{
;
}
if (!mreRemove.IsSet)
{
mreRemove.Set();
}
return(item);
}
public bool TryAdd(T item, int millisecondsTimeout, CancellationToken cancellationToken)
{
if (millisecondsTimeout < -1)
{
throw new ArgumentOutOfRangeException("millisecondsTimeout");
}
long start = millisecondsTimeout == -1 ? 0 : watch.ElapsedMilliseconds;
SpinWait sw = new SpinWait();
do
{
cancellationToken.ThrowIfCancellationRequested();
int cachedAddId = addId;
int cachedRemoveId = removeId;
int itemsIn = cachedAddId - cachedRemoveId;
if (isComplete.Value && cachedAddId >= completeId)
{
ThrowCompleteException();
}
if (upperBound != -1 && itemsIn >= upperBound)
{
if (millisecondsTimeout == 0)
{
return(false);
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
mreRemove.Reset();
if (cachedRemoveId != removeId || cachedAddId != addId)
{
mreRemove.Set();
continue;
}
mreRemove.Wait(ComputeTimeout(millisecondsTimeout, start), cancellationToken);
}
continue;
}
if (Interlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) != cachedAddId)
{
continue;
}
if (!underlyingColl.TryAdd(item))
{
throw new InvalidOperationException("The underlying collection didn't accept the item.");
}
mreAdd.Set();
return(true);
} while (millisecondsTimeout == -1 || (watch.ElapsedMilliseconds - start) < millisecondsTimeout);
return(false);
}
