public void Enqueue(T item)
{
Node node = new Node();
node.Value = item;
Node oldTail = null;
Node oldNext = null;
bool update = false;
while (!update)
{
oldTail = Tail;
oldNext = oldTail.Next;
// Did tail was already updated ?
if (tail == oldTail)
{
if (oldNext == null)
{
// The place is for us
update = CustomInterlocked.CompareExchange(ref Tail.next, node, null) == null;
}
else
{
// another Thread already used the place so give him a hand by putting tail where it should be
CustomInterlocked.CompareExchange(ref tail, oldNext, oldTail);
}
}
}
// At this point we added correctly our node, now we have to update tail. If it fails then it will be done by another thread
CustomInterlocked.CompareExchange(ref tail, node, oldTail);
CustomInterlocked.Increment(ref count);
}
public void PushRange(T[] items, int startIndex, int count)
{
RangeArgumentsCheck(items, startIndex, count);
Node insert = null;
Node first = null;
for (int i = startIndex; i < count; i++)
{
Node temp = new Node();
temp.Value = items[i];
temp.Next = insert;
insert = temp;
if (first == null)
{
first = temp;
}
}
do
{
first.Next = Head;
} while (CustomInterlocked.CompareExchange(ref head, insert, first.Next) != first.Next);
CustomInterlocked.Add(ref this.count, count);
}
public void EnterWriteLock()
{
SpinWait sw = new SpinWait();
do
{
int state = rwlock;
if (state < RwWrite)
{
if (CustomInterlocked.CompareExchange(ref rwlock, RwWrite, state) == state)
{
return;
}
state = rwlock;
}
// We register our interest in taking the Write lock (if upgradeable it's already done)
while ((state & RwWait) == 0 && CustomInterlocked.CompareExchange(ref rwlock, state | RwWait, state) != state)
{
state = rwlock;
}
// Before falling to sleep
while (rwlock > RwWait)
{
sw.SpinOnce();
}
} while (true);
}
public bool CompareAndExchange(bool expected, bool newVal)
{
int newTemp = newVal ? Set : UnSet;
int expectedTemp = expected ? Set : UnSet;
return(CustomInterlocked.CompareExchange(ref flag, newTemp, expectedTemp) == expectedTemp);
}
bool ListPop(out T data)
{
Node rightNode = null, rightNodeNext = null, leftNode = head;
data = default(T);
do
{
rightNode = head.Next;
if (rightNode == tail)
{
return(false);
}
data = rightNode.Data;
rightNodeNext = rightNode.Next;
if (!rightNodeNext.Marked)
{
if (CustomInterlocked.CompareExchange(ref rightNode.next, new Node(rightNodeNext), rightNodeNext) == rightNodeNext)
{
break;
}
}
} while (true);
if (CustomInterlocked.CompareExchange(ref leftNode.next, rightNodeNext, rightNode) != rightNodeNext)
{
ListSearch(rightNode.Key, ref leftNode);
}
return(true);
}
public PopResult PopBottom(out T obj)
{
obj = default(T);
int b = CustomInterlocked.Decrement(ref bottom);
var a = array;
int t = top;
int size = b - t;
if (size < 0)
{
// Set bottom to t
CustomInterlocked.Add(ref bottom, t - b);
return(PopResult.Empty);
}
obj = a.segment[b % a.size];
if (size > 0)
{
return(PopResult.Succeed);
}
CustomInterlocked.Add(ref bottom, t + 1 - b);
if (CustomInterlocked.CompareExchange(ref top, t + 1, t) != t)
{
return(PopResult.Empty);
}
return(PopResult.Succeed);
}
bool InsertInternal(uint key, TKey subKey, T data, Func <T> dataCreator, out Node current)
{
Node node = new Node().Init(ComputeRegularKey(key), subKey, data);
uint b = key % (uint)size;
Node bucket;
if ((bucket = GetBucket(b)) == null)
{
bucket = InitializeBucket(b);
}
if (!ListInsert(node, bucket, out current, dataCreator))
{
return(false);
}
int csize = size;
if (CustomInterlocked.Increment(ref count) / csize > MaxLoad && (csize & 0x40000000) == 0)
{
CustomInterlocked.CompareExchange(ref size, 2 * csize, csize);
}
current = node;
return(true);
}
Node SetBucket(uint index, Node node)
{
try {
slim.EnterReadLock();
CheckSegment(index, true);
CustomInterlocked.CompareExchange(ref buckets[index], node, null);
return((Node)buckets[index]);
} finally {
slim.ExitReadLock();
}
}
public void Push(T item)
{
Node temp = new Node();
temp.Value = item;
do
{
temp.Next = Head;
} while (CustomInterlocked.CompareExchange(ref head, temp, temp.Next) != temp.Next);
CustomInterlocked.Increment(ref count);
}
public void Add(TCompletion continuation)
{
if (single == null && CustomInterlocked.CompareExchange(ref single, continuation, null) == null)
{
return;
}
if (completed == null)
{
CustomInterlocked.CompareExchange(ref completed, new ConcurrentOrderedList <TCompletion> (), null);
}
Completed.TryAdd(continuation);
}
void AddHint(int index)
{
// We only take thread index that can be stored in 5 bits (i.e. thread ids 1-15)
if (index > 0xF)
{
return;
}
var hs = hints;
// If cas failed then we don't retry
CustomInterlocked.CompareExchange(ref hints, (int)(((uint)hs) << 4 | (uint)index), (int)hs);
}
Node ListSearch(ulong key, TKey subKey, ref Node left, Node h)
{
Node leftNodeNext = null, rightNode = null;
do
{
Node t = h;
Node tNext = t.Next;
do
{
if (!tNext.Marked)
{
left = t;
leftNodeNext = tNext;
}
t = tNext.Marked ? tNext.Next : tNext;
if (t == tail)
{
break;
}
tNext = t.Next;
} while (tNext.Marked || t.Key < key || (tNext.Key == key && !comparer.Equals(subKey, t.SubKey)));
rightNode = t;
if (leftNodeNext == rightNode)
{
if (rightNode != tail && rightNode.Next.Marked)
{
continue;
}
else
{
return(rightNode);
}
}
if (CustomInterlocked.CompareExchange(ref left.next, rightNode, leftNodeNext) == leftNodeNext)
{
if (rightNode != tail && rightNode.Next.Marked)
{
continue;
}
else
{
return(rightNode);
}
}
} while (true);
}
Node ListSearch(int key, ref Node left)
{
Node leftNodeNext = null, rightNode = null;
do
{
Node t = head;
Node tNext = t.Next;
do
{
if (!tNext.Marked)
{
left = t;
leftNodeNext = tNext;
}
t = tNext.Marked ? tNext.Next : tNext;
if (t == tail)
{
break;
}
tNext = t.Next;
} while (tNext.Marked || t.Key < key);
rightNode = t;
if (leftNodeNext == rightNode)
{
if (rightNode != tail && rightNode.Next.Marked)
{
continue;
}
else
{
return(rightNode);
}
}
if (CustomInterlocked.CompareExchange(ref left.next, rightNode, leftNodeNext) == leftNodeNext)
{
if (rightNode != tail && rightNode.Next.Marked)
{
continue;
}
else
{
return(rightNode);
}
}
} while (true);
}
public bool Remove(TCompletion continuation)
{
TCompletion temp = Single;
if (temp != null && temp == continuation && CustomInterlocked.CompareExchange(ref single, null, continuation) == continuation)
{
return(true);
}
if (completed != null)
{
return(Completed.TryRemove(continuation));
}
return(false);
}
bool TryGetHint(out int index)
{
/* Funny little thing to know, since hints is signed (because CAS has no uint overload),
* a shift-right operation is an arithmetic shift which might set high-order right bits
* to 1 instead of 0 if the number turns negative.
*/
var hs = hints;
index = 0;
if (CustomInterlocked.CompareExchange(ref hints, (int)(((uint)hs) >> 4), hs) == hs)
{
index = (int)(hs & 0xF);
}
return(index > 0);
}
bool ListInsert(Node newNode)
{
int key = newNode.Key;
Node rightNode = null, leftNode = null;
do
{
rightNode = ListSearch(key, ref leftNode);
if (rightNode != tail && rightNode.Key == key)
{
return(false);
}
newNode.next = rightNode;
if (CustomInterlocked.CompareExchange(ref leftNode.next, newNode, rightNode) == rightNode)
{
return(true);
}
} while (true);
}
public bool TryDequeue(out T result)
{
result = default(T);
Node oldNext = null;
bool advanced = false;
while (!advanced)
{
Node oldHead = Head;
Node oldTail = Tail;
oldNext = oldHead.Next;
if (oldHead == head)
{
// Empty case ?
if (oldHead == oldTail)
{
// This should be false then
if (oldNext != null)
{
// If not then the linked list is mal formed, update tail
CustomInterlocked.CompareExchange(ref tail, oldNext, oldTail);
continue;
}
result = default(T);
return(false);
}
else
{
result = oldNext.Value;
advanced = CustomInterlocked.CompareExchange(ref head, oldNext, oldHead) == oldHead;
}
}
}
oldNext.Value = default(T);
CustomInterlocked.Decrement(ref count);
return(true);
}
public bool TryPop(out T result)
{
Node temp;
do
{
temp = Head;
// The stak is empty
if (temp == null)
{
result = default(T);
return(false);
}
} while (CustomInterlocked.CompareExchange(ref head, temp.Next, temp) != temp);
CustomInterlocked.Decrement(ref count);
result = temp.Value;
return(true);
}
bool ListDelete(Node startPoint, ulong key, TKey subKey, out T data)
{
Node rightNode = null, rightNodeNext = null, leftNode = null;
data = default(T);
Node markedNode = null;
do
{
rightNode = ListSearch(key, subKey, ref leftNode, startPoint);
if (rightNode == tail || rightNode.Key != key || !comparer.Equals(subKey, rightNode.SubKey))
{
return(false);
}
data = rightNode.Data;
rightNodeNext = rightNode.Next;
if (!rightNodeNext.Marked)
{
if (markedNode == null)
{
markedNode = new Node();
}
markedNode.Init(rightNodeNext);
if (CustomInterlocked.CompareExchange(ref rightNode.next, markedNode, rightNodeNext) == rightNodeNext)
{
break;
}
}
} while (true);
if (CustomInterlocked.CompareExchange(ref leftNode.next, rightNodeNext, rightNode) != rightNode)
{
ListSearch(rightNode.Key, subKey, ref leftNode, startPoint);
}
return(true);
}
public PopResult PopTop(out T obj)
{
obj = default(T);
int t = top;
int b = bottom;
if (b - t <= 0)
{
return(PopResult.Empty);
}
if (CustomInterlocked.CompareExchange(ref top, t + 1, t) != t)
{
return(PopResult.Abort);
}
var a = array;
obj = a.segment[t % a.size];
return(PopResult.Succeed);
}
public int TryPopRange(T[] items, int startIndex, int count)
{
RangeArgumentsCheck(items, startIndex, count);
Node temp;
Node end;
do
{
temp = Head;
if (temp == null)
{
return(0);
}
end = temp;
for (int j = 0; j < count; j++)
{
end = end.Next;
if (end == null)
{
break;
}
}
} while (CustomInterlocked.CompareExchange(ref head, end, temp) != temp);
int i;
for (i = startIndex; i < startIndex + count && temp != null; i++)
{
items[i] = temp.Value;
end = temp;
temp = temp.Next;
}
CustomInterlocked.Add(ref this.count, -(i - startIndex));
return(i - startIndex);
}
bool ListInsert(Node newNode, Node startPoint, out Node current, Func <T> dataCreator)
{
ulong key = newNode.Key;
Node rightNode = null, leftNode = null;
do
{
rightNode = current = ListSearch(key, newNode.SubKey, ref leftNode, startPoint);
if (rightNode != tail && rightNode.Key == key && comparer.Equals(newNode.SubKey, rightNode.SubKey))
{
return(false);
}
newNode.next = rightNode;
if (dataCreator != null)
{
newNode.Data = dataCreator();
}
if (CustomInterlocked.CompareExchange(ref leftNode.next, newNode, rightNode) == rightNode)
{
return(true);
}
} while (true);
}
bool TryTake(out T item, int milliseconds, CancellationToken cancellationToken, bool throwComplete)
{
if (milliseconds < -1)
{
throw new ArgumentOutOfRangeException("milliseconds");
}
item = default(T);
SpinWait sw = new SpinWait();
long start = milliseconds == -1 ? 0 : watch.ElapsedMilliseconds;
do
{
cancellationToken.ThrowIfCancellationRequested();
int cachedRemoveId = removeId;
int cachedAddId = addId;
// Empty case
if (cachedRemoveId == cachedAddId)
{
if (milliseconds == 0)
{
return(false);
}
if (IsCompleted)
{
if (throwComplete)
{
ThrowCompleteException();
}
else
{
return(false);
}
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
mreAdd.Reset();
if (cachedRemoveId != removeId || cachedAddId != addId)
{
mreAdd.Set();
continue;
}
mreAdd.Wait(ComputeTimeout(milliseconds, start), cancellationToken);
}
continue;
}
if (CustomInterlocked.CompareExchange(ref removeId, cachedRemoveId + 1, cachedRemoveId) != cachedRemoveId)
{
continue;
}
while (!underlyingColl.TryTake(out item))
{
;
}
mreRemove.Set();
return(true);
} while (milliseconds == -1 || (watch.ElapsedMilliseconds - start) < milliseconds);
return(false);
}
public bool TryAdd(T item, int millisecondsTimeout, CancellationToken cancellationToken)
{
if (millisecondsTimeout < -1)
{
throw new ArgumentOutOfRangeException("millisecondsTimeout");
}
long start = millisecondsTimeout == -1 ? 0 : watch.ElapsedMilliseconds;
SpinWait sw = new SpinWait();
do
{
cancellationToken.ThrowIfCancellationRequested();
int cachedAddId = addId;
int cachedRemoveId = removeId;
int itemsIn = cachedAddId - cachedRemoveId;
// Check our transaction id against completed stored one
if (isComplete.Value && cachedAddId >= completeId)
{
ThrowCompleteException();
}
// If needed, we check and wait that the collection isn't full
if (upperBound != -1 && itemsIn >= upperBound)
{
if (millisecondsTimeout == 0)
{
return(false);
}
if (sw.Count <= spinCount)
{
sw.SpinOnce();
}
else
{
mreRemove.Reset();
if (cachedRemoveId != removeId || cachedAddId != addId)
{
mreRemove.Set();
continue;
}
mreRemove.Wait(ComputeTimeout(millisecondsTimeout, start), cancellationToken);
}
continue;
}
// Validate the steps we have been doing until now
if (CustomInterlocked.CompareExchange(ref addId, cachedAddId + 1, cachedAddId) != cachedAddId)
{
continue;
}
// We have a slot reserved in the underlying collection, try to take it
if (!underlyingColl.TryAdd(item))
{
throw new InvalidOperationException("The underlying collection didn't accept the item.");
}
// Wake up process that may have been sleeping
mreAdd.Set();
return(true);
} while (millisecondsTimeout == -1 || (watch.ElapsedMilliseconds - start) < millisecondsTimeout);
return(false);
}
