internal static void Push(Work work)
{
SpinlockTTAS.Enter(ref Lock);
work.Next = WorkStack;
WorkStack = work;
var waiters = Waiters;
if (0 <= waiters)
{
Events[waiters].Set();
}
SpinlockTTAS.Exit(ref Lock);
}
internal static void SynchronizedPushTimed(ref Worker wr, WorkTimed x)
{
SpinlockTTAS.Enter(ref Scheduler.Lock);
var h = TopTimed;
if (null == h)
{
TopTimed = x;
x.Up = x;
x.Next = x;
var waiter = Waiters;
if (0 <= Waiters)
{
Events[waiter].Set();
}
}
else if (x.Ticks - h.Ticks < 0)
{
x.Next = h.Up;
h.Up = x;
x.Up = x;
TopTimed = x;
var waiter = Waiters;
if (0 <= Waiters)
{
Events[waiter].Set();
}
}
else if (x.Ticks - h.Up.Ticks > 0)
{
x.Up = h.Up;
h.Up = x;
x.Next = x;
}
else
{
var y = h.Up;
Work z = h.Up;
while (x.Ticks - y.Up.Ticks < 0)
{
var t = y.Next;
y.Next = z;
z = t;
}
x.Up = y.Up;
x.Next = z;
y.Up = x;
h.Up = x;
}
SpinlockTTAS.Exit(ref Scheduler.Lock);
}
internal static void Enter(ref SpinlockTTAS self)
{
Spin:
// First spin using shared reads. This improves performance a bit.
var st = self.state;
if (st < Open)
{
goto Spin;
}
if (Interlocked.Exchange(ref self.state, ~st) < Open)
{
goto Spin;
}
}
internal static void Push(ref Worker wr, Work work)
{
var older = wr.WorkStack;
wr.WorkStack = work;
if (null != older)
{
goto CheckScheduler;
}
goto Exit;
CheckScheduler:
if (0 <= Scheduler.Waiters) // null == Scheduler.WorkStack)
{
goto MaybePushToScheduler;
}
Contention:
work.Next = older;
Exit:
return;
MaybePushToScheduler:
if (SpinlockTTAS.Open != SpinlockTTAS.GetState(ref Scheduler.Lock))
{
goto Contention;
}
if (SpinlockTTAS.Open != SpinlockTTAS.TryEnter(ref Scheduler.Lock))
{
goto Contention;
}
if (null == Scheduler.WorkStack)
{
goto PushToScheduler;
}
SpinlockTTAS.Exit(ref Scheduler.Lock);
work.Next = older;
return;
PushToScheduler:
Scheduler.WorkStack = older;
int waiters = Scheduler.Waiters;
if (0 <= waiters)
{
Scheduler.Events[waiters].Set();
}
SpinlockTTAS.Exit(ref Scheduler.Lock);
}
private static void ReallyInit()
{
SpinlockTTAS.Enter(ref Scheduler.Lock);
if (Volatile.Read(ref Events) == null)
{
var numWorkers = Environment.ProcessorCount;
Waiters = -1;
Events = new WorkerEvent[numWorkers];
var threads = new Thread[numWorkers];
for (int i = 0; i < numWorkers; ++i)
{
Events[i] = new WorkerEvent(i);
var index = i;
var thread = new Thread(() => Worker.Run(index));
threads[i] = thread;
thread.IsBackground = true;
}
for (int i = 0; i < numWorkers; ++i)
{
threads[i].Start();
}
SpinlockTTAS.Exit(ref Scheduler.Lock);
}
}
internal static void Run(int me)
{
var wr = new Worker();
wr.Init();
//Func<Worker> box = () => wr;
var mine = Scheduler.Events[me];
Work work = null;
while (true)
{
try {
goto Begin;
WorkerLoop:
wr.WorkStack = work.Next;
Do:
wr.Handler = work;
work.DoWork(ref wr);
work = wr.WorkStack;
Begin:
if (null != work)
{
goto WorkerLoop;
}
EnterScheduler:
SpinlockTTAS.Enter(ref Scheduler.Lock);
TryScheduler:
work = Scheduler.WorkStack;
if (null == work)
{
goto TryTimed;
}
{
var next = work.Next;
Scheduler.WorkStack = next;
if (null == next)
{
goto NoWake;
}
var waiter = Scheduler.Waiters;
if (0 <= waiter)
{
Scheduler.Events[waiter].Set();
}
}
NoWake:
SpinlockTTAS.Exit(ref Scheduler.Lock);
goto Do;
TryTimed:
var waitTicks = Timeout.Infinite;
{
var tt = Scheduler.TopTimed;
if (null == tt)
{
goto JustWait;
}
waitTicks = tt.Ticks - Environment.TickCount;
if (waitTicks > 0)
{
goto JustWait;
}
Scheduler.DropTimed();
SpinlockTTAS.Exit(ref Scheduler.Lock);
var pk = tt.Pick;
if (null == pk)
{
goto GotIt;
}
TryPick:
var st = Pick.TryPick(pk);
if (st > 0)
{
goto EnterScheduler;
}
if (st < 0)
{
goto TryPick;
}
Pick.SetNacks(ref wr, tt.Me, pk);
GotIt:
work = tt;
goto Do;
}
JustWait:
var n = Scheduler.Waiters;
if (n < 0)
{
mine.Next = -1;
Scheduler.Waiters = me;
}
else
{
WorkerEvent other = Scheduler.Events[n];
mine.Next = other.Next;
other.Next = me;
}
mine.Reset();
SpinlockTTAS.Exit(ref Scheduler.Lock);
mine.Wait(waitTicks);
SpinlockTTAS.Enter(ref Scheduler.Lock);
if (Scheduler.Waiters == me)
{
Scheduler.Waiters = mine.Next;
}
else
{
WorkerEvent ev = Scheduler.Events[Scheduler.Waiters];
if (ev.Next != me)
{
do
{
ev = Scheduler.Events[ev.Next];
} while (ev.Next != me);
}
ev.Next = mine.Next;
}
goto TryScheduler;
} catch (Exception e) {
work = new FailWork(wr.WorkStack, e, wr.Handler);
}
}
}
internal static int TryEnter(ref SpinlockTTAS self)
{
return(Interlocked.Exchange(ref self.state, Locked));
}
internal static int GetState(ref SpinlockTTAS self)
{
return(self.state);
}
internal static void Exit(ref SpinlockTTAS self)
{
Debug.Assert(self.state == Locked); // Must be locked!
self.state = Open;
}