public void SelectThreads()
{
ThreadReselectionRequested = false;
for (int core = 0; core < CpuCoresCount; core++)
{
KThread thread = SchedulingData.ScheduledThreads(core).FirstOrDefault();
CoreContexts[core].SelectThread(thread);
}
for (int core = 0; core < CpuCoresCount; core++)
{
// If the core is not idle (there's already a thread running on it),
// then we don't need to attempt load balancing.
if (SchedulingData.ScheduledThreads(core).Any())
{
continue;
}
int[] srcCoresHighestPrioThreads = new int[CpuCoresCount];
int srcCoresHighestPrioThreadsCount = 0;
KThread dst = null;
// Select candidate threads that could run on this core.
// Give preference to threads that are not yet selected.
foreach (KThread thread in SchedulingData.SuggestedThreads(core))
{
if (thread.CurrentCore < 0 || thread != CoreContexts[thread.CurrentCore].SelectedThread)
{
dst = thread;
break;
}
srcCoresHighestPrioThreads[srcCoresHighestPrioThreadsCount++] = thread.CurrentCore;
}
// Not yet selected candidate found.
if (dst != null)
{
// Priorities < 2 are used for the kernel message dispatching
// threads, we should skip load balancing entirely.
if (dst.DynamicPriority >= 2)
{
SchedulingData.TransferToCore(dst.DynamicPriority, core, dst);
CoreContexts[core].SelectThread(dst);
}
continue;
}
// All candidates are already selected, choose the best one
// (the first one that doesn't make the source core idle if moved).
for (int index = 0; index < srcCoresHighestPrioThreadsCount; index++)
{
int srcCore = srcCoresHighestPrioThreads[index];
KThread src = SchedulingData.ScheduledThreads(srcCore).ElementAtOrDefault(1);
if (src != null)
{
// Run the second thread on the queue on the source core,
// move the first one to the current core.
KThread origSelectedCoreSrc = CoreContexts[srcCore].SelectedThread;
CoreContexts[srcCore].SelectThread(src);
SchedulingData.TransferToCore(origSelectedCoreSrc.DynamicPriority, core, origSelectedCoreSrc);
CoreContexts[core].SelectThread(origSelectedCoreSrc);
}
}
}
}
private void PreemptThread(int prio, int core)
{
IEnumerable <KThread> scheduledThreads = SchedulingData.ScheduledThreads(core);
KThread selectedThread = scheduledThreads.FirstOrDefault(x => x.DynamicPriority == prio);
// Yield priority queue.
if (selectedThread != null)
{
SchedulingData.Reschedule(prio, core, selectedThread);
}
IEnumerable <KThread> SuitableCandidates()
{
foreach (KThread thread in SchedulingData.SuggestedThreads(core))
{
int srcCore = thread.CurrentCore;
if (srcCore >= 0)
{
KThread highestPrioSrcCore = SchedulingData.ScheduledThreads(srcCore).FirstOrDefault();
if (highestPrioSrcCore != null && highestPrioSrcCore.DynamicPriority < 2)
{
break;
}
if (highestPrioSrcCore == thread)
{
continue;
}
}
// If the candidate was scheduled after the current thread, then it's not worth it.
if (selectedThread == null || selectedThread.LastScheduledTime >= thread.LastScheduledTime)
{
yield return(thread);
}
}
}
// Select candidate threads that could run on this core.
// Only take into account threads that are not yet selected.
KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority == prio);
if (dst != null)
{
SchedulingData.TransferToCore(prio, core, dst);
selectedThread = dst;
}
// If the priority of the currently selected thread is lower than preemption priority,
// then allow threads with lower priorities to be selected aswell.
if (selectedThread != null && selectedThread.DynamicPriority > prio)
{
Func <KThread, bool> predicate = x => x.DynamicPriority >= selectedThread.DynamicPriority;
dst = SuitableCandidates().FirstOrDefault(predicate);
if (dst != null)
{
SchedulingData.TransferToCore(dst.DynamicPriority, core, dst);
}
}
ThreadReselectionRequested = true;
}
public KernelResult WaitProcessWideKeyAtomic(
ulong mutexAddress,
ulong condVarAddress,
int threadHandle,
long timeout)
{
_system.CriticalSection.Enter();
KThread currentThread = _system.Scheduler.GetCurrentThread();
currentThread.SignaledObj = null;
currentThread.ObjSyncResult = KernelResult.TimedOut;
if (currentThread.ShallBeTerminated ||
currentThread.SchedFlags == ThreadSchedState.TerminationPending)
{
_system.CriticalSection.Leave();
return(KernelResult.ThreadTerminating);
}
(KernelResult result, _) = MutexUnlock(currentThread, mutexAddress);
if (result != KernelResult.Success)
{
_system.CriticalSection.Leave();
return(result);
}
currentThread.MutexAddress = mutexAddress;
currentThread.ThreadHandleForUserMutex = threadHandle;
currentThread.CondVarAddress = condVarAddress;
CondVarThreads.Add(currentThread);
if (timeout != 0)
{
currentThread.Reschedule(ThreadSchedState.Paused);
if (timeout > 0)
{
_system.TimeManager.ScheduleFutureInvocation(currentThread, timeout);
}
}
_system.CriticalSection.Leave();
if (timeout > 0)
{
_system.TimeManager.UnscheduleFutureInvocation(currentThread);
}
_system.CriticalSection.Enter();
if (currentThread.MutexOwner != null)
{
currentThread.MutexOwner.RemoveMutexWaiter(currentThread);
}
CondVarThreads.Remove(currentThread);
_system.CriticalSection.Leave();
return((KernelResult)currentThread.ObjSyncResult);
}
public KernelResult WaitProcessWideKeyAtomic(ulong mutexAddress, ulong condVarAddress, int threadHandle, long timeout)
{
_context.CriticalSection.Enter();
KThread currentThread = KernelStatic.GetCurrentThread();
currentThread.SignaledObj = null;
currentThread.ObjSyncResult = KernelResult.TimedOut;
if (currentThread.ShallBeTerminated ||
currentThread.SchedFlags == ThreadSchedState.TerminationPending)
{
_context.CriticalSection.Leave();
return(KernelResult.ThreadTerminating);
}
(int mutexValue, _) = MutexUnlock(currentThread, mutexAddress);
KernelTransfer.KernelToUser(condVarAddress, 1);
if (!KernelTransfer.KernelToUser(mutexAddress, mutexValue))
{
_context.CriticalSection.Leave();
return(KernelResult.InvalidMemState);
}
currentThread.MutexAddress = mutexAddress;
currentThread.ThreadHandleForUserMutex = threadHandle;
currentThread.CondVarAddress = condVarAddress;
_condVarThreads.Add(currentThread);
if (timeout != 0)
{
currentThread.Reschedule(ThreadSchedState.Paused);
if (timeout > 0)
{
_context.TimeManager.ScheduleFutureInvocation(currentThread, timeout);
}
}
_context.CriticalSection.Leave();
if (timeout > 0)
{
_context.TimeManager.UnscheduleFutureInvocation(currentThread);
}
_context.CriticalSection.Enter();
if (currentThread.MutexOwner != null)
{
currentThread.MutexOwner.RemoveMutexWaiter(currentThread);
}
_condVarThreads.Remove(currentThread);
_context.CriticalSection.Leave();
return(currentThread.ObjSyncResult);
}
private KThread TryAcquireMutex(KThread requester)
{
ulong address = requester.MutexAddress;
KProcess currentProcess = _system.Scheduler.GetCurrentProcess();
int mutexValue, newMutexValue;
do
{
if (!KernelTransfer.UserToKernelInt32(_system, address, out mutexValue))
{
//Invalid address.
requester.SignaledObj = null;
requester.ObjSyncResult = KernelResult.InvalidMemState;
return(null);
}
if (mutexValue != 0)
{
//Update value to indicate there is a mutex waiter now.
newMutexValue = mutexValue | HasListenersMask;
}
else
{
//No thread owning the mutex, assign to requesting thread.
newMutexValue = requester.ThreadHandleForUserMutex;
}
}while (!currentProcess.CpuMemory.AtomicCompareExchangeInt32((long)address, mutexValue, newMutexValue));
if (mutexValue == 0)
{
//We now own the mutex.
requester.SignaledObj = null;
requester.ObjSyncResult = KernelResult.Success;
requester.ReleaseAndResume();
return(null);
}
mutexValue &= ~HasListenersMask;
KThread mutexOwner = currentProcess.HandleTable.GetObject <KThread>(mutexValue);
if (mutexOwner != null)
{
//Mutex already belongs to another thread, wait for it.
mutexOwner.AddMutexWaiter(requester);
}
else
{
//Invalid mutex owner.
requester.SignaledObj = null;
requester.ObjSyncResult = KernelResult.InvalidHandle;
requester.ReleaseAndResume();
}
return(mutexOwner);
}
public KernelResult WaitForAddressIfLessThan(
ulong address,
int value,
bool shouldDecrement,
long timeout)
{
KThread currentThread = _system.Scheduler.GetCurrentThread();
_system.CriticalSection.Enter();
if (currentThread.ShallBeTerminated ||
currentThread.SchedFlags == ThreadSchedState.TerminationPending)
{
_system.CriticalSection.Leave();
return(KernelResult.ThreadTerminating);
}
currentThread.SignaledObj = null;
currentThread.ObjSyncResult = KernelResult.TimedOut;
KProcess currentProcess = _system.Scheduler.GetCurrentProcess();
if (!KernelTransfer.UserToKernelInt32(_system, address, out int currentValue))
{
_system.CriticalSection.Leave();
return(KernelResult.InvalidMemState);
}
if (shouldDecrement)
{
currentValue = currentProcess.CpuMemory.AtomicDecrementInt32((long)address) + 1;
}
if (currentValue < value)
{
if (timeout == 0)
{
_system.CriticalSection.Leave();
return(KernelResult.TimedOut);
}
currentThread.MutexAddress = address;
currentThread.WaitingInArbitration = true;
InsertSortedByPriority(ArbiterThreads, currentThread);
currentThread.Reschedule(ThreadSchedState.Paused);
if (timeout > 0)
{
_system.TimeManager.ScheduleFutureInvocation(currentThread, timeout);
}
_system.CriticalSection.Leave();
if (timeout > 0)
{
_system.TimeManager.UnscheduleFutureInvocation(currentThread);
}
_system.CriticalSection.Enter();
if (currentThread.WaitingInArbitration)
{
ArbiterThreads.Remove(currentThread);
currentThread.WaitingInArbitration = false;
}
_system.CriticalSection.Leave();
return((KernelResult)currentThread.ObjSyncResult);
}
_system.CriticalSection.Leave();
return(KernelResult.InvalidState);
}
public void ContextSwitch()
{
lock (CoreContexts)
{
if (MultiCoreScheduling)
{
int selectedCount = 0;
for (int core = 0; core < CpuCoresCount; core++)
{
KCoreContext coreContext = CoreContexts[core];
if (coreContext.ContextSwitchNeeded && (coreContext.CurrentThread?.IsCurrentHostThread() ?? false))
{
coreContext.ContextSwitch();
}
if (coreContext.CurrentThread?.IsCurrentHostThread() ?? false)
{
selectedCount++;
}
}
if (selectedCount == 0)
{
CoreManager.Reset(Thread.CurrentThread);
}
else if (selectedCount == 1)
{
CoreManager.Set(Thread.CurrentThread);
}
else
{
throw new InvalidOperationException("Thread scheduled in more than one core!");
}
}
else
{
KThread currentThread = CoreContexts[_currentCore].CurrentThread;
bool hasThreadExecuting = currentThread != null;
if (hasThreadExecuting)
{
// If this is not the thread that is currently executing, we need
// to request an interrupt to allow safely starting another thread.
if (!currentThread.IsCurrentHostThread())
{
currentThread.Context.RequestInterrupt();
return;
}
CoreManager.Reset(currentThread.HostThread);
}
// Advance current core and try picking a thread,
// keep advancing if it is null.
for (int core = 0; core < 4; core++)
{
_currentCore = (_currentCore + 1) % CpuCoresCount;
KCoreContext coreContext = CoreContexts[_currentCore];
coreContext.UpdateCurrentThread();
if (coreContext.CurrentThread != null)
{
CoreManager.Set(coreContext.CurrentThread.HostThread);
coreContext.CurrentThread.Execute();
break;
}
}
// If nothing was running before, then we are on a "external"
// HLE thread, we don't need to wait.
if (!hasThreadExecuting)
{
return;
}
}
}
CoreManager.Wait(Thread.CurrentThread);
}
public KernelResult WaitFor(KSynchronizationObject[] syncObjs, long timeout, out int handleIndex)
{
handleIndex = 0;
KernelResult result = KernelResult.TimedOut;
_context.CriticalSection.Enter();
// Check if objects are already signaled before waiting.
for (int index = 0; index < syncObjs.Length; index++)
{
if (!syncObjs[index].IsSignaled())
{
continue;
}
handleIndex = index;
_context.CriticalSection.Leave();
return(KernelResult.Success);
}
if (timeout == 0)
{
_context.CriticalSection.Leave();
return(result);
}
KThread currentThread = _context.Scheduler.GetCurrentThread();
if (currentThread.ShallBeTerminated ||
currentThread.SchedFlags == ThreadSchedState.TerminationPending)
{
result = KernelResult.ThreadTerminating;
}
else if (currentThread.SyncCancelled)
{
currentThread.SyncCancelled = false;
result = KernelResult.Cancelled;
}
else
{
LinkedListNode <KThread>[] syncNodes = new LinkedListNode <KThread> [syncObjs.Length];
for (int index = 0; index < syncObjs.Length; index++)
{
syncNodes[index] = syncObjs[index].AddWaitingThread(currentThread);
}
currentThread.WaitingSync = true;
currentThread.SignaledObj = null;
currentThread.ObjSyncResult = result;
currentThread.Reschedule(ThreadSchedState.Paused);
if (timeout > 0)
{
_context.TimeManager.ScheduleFutureInvocation(currentThread, timeout);
}
_context.CriticalSection.Leave();
currentThread.WaitingSync = false;
if (timeout > 0)
{
_context.TimeManager.UnscheduleFutureInvocation(currentThread);
}
_context.CriticalSection.Enter();
result = currentThread.ObjSyncResult;
handleIndex = -1;
for (int index = 0; index < syncObjs.Length; index++)
{
syncObjs[index].RemoveWaitingThread(syncNodes[index]);
if (syncObjs[index] == currentThread.SignaledObj)
{
handleIndex = index;
}
}
}
_context.CriticalSection.Leave();
return(result);
}
public void RemoveThread(KThread thread)
{
CoreManager.RemoveThread(thread.HostThread);
}
public void ExitThread(KThread thread)
{
thread.Context.Running = false;
CoreManager.Exit(thread.HostThread);
}
public void YieldWithLoadBalancing()
{
System.CriticalSection.Enter();
if (SchedFlags != ThreadSchedState.Running)
{
System.CriticalSection.Leave();
System.Scheduler.ContextSwitch();
return;
}
int prio = DynamicPriority;
int core = CurrentCore;
KThread nextThreadOnCurrentQueue = null;
if (DynamicPriority < KScheduler.PrioritiesCount)
{
// Move current thread to the end of the queue.
_schedulingData.Reschedule(prio, core, this);
Func <KThread, bool> predicate = x => x.DynamicPriority == prio;
nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
}
IEnumerable <KThread> SuitableCandidates()
{
foreach (KThread thread in _schedulingData.SuggestedThreads(core))
{
int srcCore = thread.CurrentCore;
if (srcCore >= 0)
{
KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
{
continue;
}
}
// If the candidate was scheduled after the current thread, then it's not worth it,
// unless the priority is higher than the current one.
if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
{
yield return(thread);
}
}
}
KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
if (dst != null)
{
_schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
_scheduler.ThreadReselectionRequested = true;
}
if (this != nextThreadOnCurrentQueue)
{
_scheduler.ThreadReselectionRequested = true;
}
System.CriticalSection.Leave();
System.Scheduler.ContextSwitch();
}
public KernelResult Start()
{
if (!System.KernelInitialized)
{
System.CriticalSection.Enter();
if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
{
_forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
CombineForcePauseFlags();
}
System.CriticalSection.Leave();
}
KernelResult result = KernelResult.ThreadTerminating;
System.CriticalSection.Enter();
if (!ShallBeTerminated)
{
KThread currentThread = System.Scheduler.GetCurrentThread();
while (SchedFlags != ThreadSchedState.TerminationPending &&
currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
!currentThread.ShallBeTerminated)
{
if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
{
result = KernelResult.InvalidState;
break;
}
if (currentThread._forcePauseFlags == ThreadSchedState.None)
{
if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
{
CombineForcePauseFlags();
}
SetNewSchedFlags(ThreadSchedState.Running);
result = KernelResult.Success;
break;
}
else
{
currentThread.CombineForcePauseFlags();
System.CriticalSection.Leave();
System.CriticalSection.Enter();
if (currentThread.ShallBeTerminated)
{
break;
}
}
}
}
System.CriticalSection.Leave();
return(result);
}
private static void RotateScheduledQueue(KernelContext context, int core, int prio)
{
IEnumerable <KThread> scheduledThreads = context.PriorityQueue.ScheduledThreads(core);
KThread selectedThread = scheduledThreads.FirstOrDefault(x => x.DynamicPriority == prio);
KThread nextThread = null;
// Yield priority queue.
if (selectedThread != null)
{
nextThread = context.PriorityQueue.Reschedule(prio, core, selectedThread);
}
IEnumerable <KThread> SuitableCandidates()
{
foreach (KThread suggested in context.PriorityQueue.SuggestedThreads(core))
{
int suggestedCore = suggested.ActiveCore;
if (suggestedCore >= 0)
{
KThread selectedSuggestedCore = context.PriorityQueue.ScheduledThreads(suggestedCore).FirstOrDefault();
if (selectedSuggestedCore == suggested || (selectedSuggestedCore != null && selectedSuggestedCore.DynamicPriority < 2))
{
continue;
}
}
// If the candidate was scheduled after the current thread, then it's not worth it.
if (nextThread == selectedThread ||
nextThread == null ||
nextThread.LastScheduledTime >= suggested.LastScheduledTime)
{
yield return(suggested);
}
}
}
// Select candidate threads that could run on this core.
// Only take into account threads that are not yet selected.
KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority == prio);
if (dst != null)
{
context.PriorityQueue.TransferToCore(prio, core, dst);
}
// If the priority of the currently selected thread is lower or same as the preemption priority,
// then try to migrate a thread with lower priority.
KThread bestCandidate = context.PriorityQueue.ScheduledThreads(core).FirstOrDefault();
if (bestCandidate != null && bestCandidate.DynamicPriority >= prio)
{
dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority < bestCandidate.DynamicPriority);
if (dst != null)
{
context.PriorityQueue.TransferToCore(dst.DynamicPriority, core, dst);
}
}
context.ThreadReselectionRequested = true;
}
