internal void Enter(ref Worker wr, Work work) {
work.Next = null;
var owner = Interlocked.Exchange(ref Tail, work);
if (owner != null) {
owner.Next = work;
} else {
Release:
work.DoWork(ref wr);
var prev = work;
work = Volatile.Read(ref prev.Next);
if (null != work)
goto Release;
if (prev == Interlocked.CompareExchange(ref Tail, null, prev))
return;
Spin:
work = Volatile.Read(ref prev.Next);
if (null != work)
goto Release;
goto Spin;
}
}
internal static void Push(Scheduler sr, Work work, Work last, int n)
{
Enter(sr);
last.Next = sr.WorkStack;
sr.WorkStack = work;
sr.NumWorkStack += n;
UnsafeSignal(sr);
Exit(sr);
}
internal static void Do(Work work, ref Worker wr) {
#if TRAMPOLINE
unsafe {
if (Unsafe.GetStackPtr() < wr.StackLimit) {
work.Next = wr.WorkStack;
wr.WorkStack = work;
} else {
work.DoWork(ref wr);
}
}
#else
work.DoWork(ref wr);
#endif
}
internal void ExitAndEnter(ref Worker wr, Work owner) {
var work = owner.Next;
if (null != work) {
owner.Next = null;
var last = Interlocked.Exchange(ref Tail, owner);
Release:
work.DoWork(ref wr);
var prev = work;
Spin:
work = Volatile.Read(ref prev.Next);
if (null != work)
goto Release;
if (prev != last)
goto Spin;
}
}
internal static void PushAllAndDec(Scheduler sr, Work work) {
if (null == work) {
Dec(sr);
} else {
var n = 1;
var last = work;
FindLast:
var next = last.Next;
if (null == next)
goto FoundLast;
n += 1;
last = next;
goto FindLast;
FoundLast:
PushAndDec(sr, work, last, n);
}
}
internal static void PushAndDec(Scheduler sr, Work work, Work last, int n) {
Enter(sr);
last.Next = sr.WorkStack;
sr.WorkStack = work;
sr.NumWorkStack += n;
sr.NumActive -= 1;
Debug.Assert(0 <= sr.NumActive);
UnsafeSignal(sr);
Exit(sr);
}
internal static void PushAll(Scheduler sr, Work work) {
if (null == work)
return;
var n = 1;
var last = work;
FindLast:
var next = last.Next;
if (null == next)
goto FoundLast;
n += 1;
last = next;
goto FindLast;
FoundLast:
Push(sr, work, last, n);
}
internal static void PushNew(Scheduler sr, Work work) {
Push(sr, work, work, 1);
}
internal FailWork(Work next, Exception e, Handler hr)
{
this.Next = next;
this.e = e;
this.hr = hr;
}
internal Work(Work next)
{
this.Next = next;
}
internal FailWork(Work next, Exception e, Handler hr)
{
this.Next = next;
this.e = e;
this.hr = hr;
}
internal Work(Work next)
{
this.Next = next;
}
internal static void PushNew(ref Worker wr, Work work)
{
PushNew(ref wr, work, work);
}
internal static void Push(ref Worker wr, Work work, Work last)
{
last.Next = null;
PushNew(ref wr, work, last);
}
internal static void Push(ref Worker wr, Work work)
{
work.Next = null;
PushNew(ref wr, work, work);
}
internal static void Run(Scheduler sr, int me)
{
IsWorkerThread = true;
var wr = new Worker();
wr.Init(sr, 4000);
wr.RandomHi = (ulong)DateTime.UtcNow.Ticks;
var iK = new IdleCont();
var wdm = (1L << 32) / sr.Events.Length;
wr.Event = sr.Events[me];
while (null != sr)
{
try {
Restart:
Work work = wr.WorkStack;
if (null == work)
{
goto EnterScheduler;
}
WorkerLoop:
wr.Handler = work;
{
var next = work.Next;
if (null != next && null == sr.WorkStack)
{
Scheduler.PushAll(sr, next);
next = null;
}
wr.WorkStack = next;
}
work.DoWork(ref wr);
work = wr.WorkStack;
if (null != work)
{
goto WorkerLoop;
}
wr.Handler = null;
EnterScheduler:
work = sr.WorkStack;
if (null == work)
{
goto TryIdle;
}
Scheduler.Enter(sr);
EnteredScheduler:
work = sr.WorkStack;
if (null == work)
{
goto ExitAndTryIdle;
}
SchedulerGotSome : {
var last = work;
int numWorkStack = sr.NumWorkStack;
int n = (int)((numWorkStack - 1L) * wdm >> 32) + 1;
sr.NumWorkStack = numWorkStack - n;
n -= 1;
while (n > 0)
{
last = last.Next;
n -= 1;
}
var next = last.Next;
last.Next = null;
sr.WorkStack = next;
if (null != next)
{
Scheduler.UnsafeSignal(sr);
}
Scheduler.Exit(sr);
goto WorkerLoop;
}
ExitAndTryIdle:
Scheduler.Exit(sr);
TryIdle:
iK.Value = Timeout.Infinite;
var iJ = sr.IdleHandler;
if (null != iJ)
{
wr.Handler = iK;
iJ.DoJob(ref wr, iK);
}
if (0 == iK.Value)
{
goto Restart;
}
Scheduler.Enter(sr);
work = sr.WorkStack;
if (null != work)
{
goto SchedulerGotSome;
}
Scheduler.UnsafeWait(sr, iK.Value, wr.Event);
goto EnteredScheduler;
} catch (KillException) {
Scheduler.Kill(sr);
Scheduler.Dec(sr);
sr = null;
} catch (Exception e) {
wr.WorkStack = new FailWork(wr.WorkStack, e, wr.Handler);
}
}
}
