internal WorkerPool(OrleansTaskScheduler sched, IHostEnvironmentStatistics hostStatistics, ExecutorService executorService, ILoggerFactory loggerFactory, int maxActiveThreads, bool enableWorkerThreadInjection)
{
scheduler = sched;
this.hostStatistics = hostStatistics;
MaxActiveThreads = maxActiveThreads;
EnableWorkerThreadInjection = enableWorkerThreadInjection;
MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
this.executorService = executorService;
this.loggerFactory = loggerFactory;
if (EnableWorkerThreadInjection)
{
threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
}
pool = new HashSet <WorkerPoolThread>();
createThreadCount = 0;
lockable = new object();
for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
{
var t = new WorkerPoolThread(this, scheduler, executorService, loggerFactory, hostStatistics, createThreadCount);
pool.Add(t);
}
createThreadCount++;
systemThread = new WorkerPoolThread(this, scheduler, executorService, loggerFactory, hostStatistics, createThreadCount, true);
running = false;
runningThreadCount = 0;
longTurnTimer = null;
this.timerLogger = loggerFactory.CreateLogger <SafeTimer>();
}
internal WorkerPool(OrleansTaskScheduler sched, ICorePerformanceMetrics performanceMetrics, int maxActiveThreads, bool enableWorkerThreadInjection)
{
scheduler = sched;
MaxActiveThreads = maxActiveThreads;
EnableWorkerThreadInjection = enableWorkerThreadInjection;
MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
this.performanceMetrics = performanceMetrics;
if (EnableWorkerThreadInjection)
{
threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
}
pool = new HashSet <WorkerPoolThread>();
createThreadCount = 0;
lockable = new object();
for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
{
var t = new WorkerPoolThread(this, scheduler, performanceMetrics, createThreadCount);
pool.Add(t);
}
createThreadCount++;
systemThread = new WorkerPoolThread(this, scheduler, performanceMetrics, createThreadCount, true);
running = false;
runningThreadCount = 0;
longTurnTimer = null;
}
internal void CreateNewThread()
{
lock (lockable)
{
var t = new WorkerPoolThread(this, scheduler);
pool.Add(t);
t.Start();
}
}
internal void CreateNewThread()
{
lock (lockable)
{
createThreadCount++;
var t = new WorkerPoolThread(this, scheduler, this.performanceMetrics, createThreadCount);
pool.Add(t);
t.Start();
}
}
internal void CreateNewThread()
{
lock (lockable)
{
createThreadCount++;
var t = new WorkerPoolThread(this, scheduler, this.executorService, this.loggerFactory, this.hostStatistics, createThreadCount);
pool.Add(t);
t.Start();
}
}
internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool injectMoreWorkerThreads)
{
scheduler = sched;
MaxActiveThreads = maxActiveThreads;
InjectMoreWorkerThreads = injectMoreWorkerThreads;
MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
pool = new HashSet <WorkerPoolThread>();
lockable = new object();
for (int i = 0; i < MaxActiveThreads; i++)
{
var t = new WorkerPoolThread(this, scheduler);
pool.Add(t);
}
systemThread = new WorkerPoolThread(this, scheduler, true);
running = false;
runningThreadCount = 0;
longTurnTimer = null;
}
internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool injectMoreWorkerThreads)
{
scheduler = sched;
MaxActiveThreads = maxActiveThreads;
InjectMoreWorkerThreads = injectMoreWorkerThreads;
MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
pool = new HashSet<WorkerPoolThread>();
lockable = new object();
for (int i = 0; i < MaxActiveThreads; i++)
{
var t = new WorkerPoolThread(this, scheduler);
pool.Add(t);
}
systemThread = new WorkerPoolThread(this, scheduler, true);
running = false;
runningThreadCount = 0;
longTurnTimer = null;
}
internal void RecordLeavingThread(WorkerPoolThread t)
{
bool restart = false;
lock (lockable)
{
pool.Remove(t);
if (running && (pool.Count < MaxActiveThreads + 2))
{
restart = true;
}
}
if (!restart)
{
return;
}
var tnew = new WorkerPoolThread(this, scheduler);
tnew.Start();
}
internal void RecordLeavingThread(WorkerPoolThread t)
{
bool restart = false;
lock (lockable)
{
pool.Remove(t);
if (running && (pool.Count < MaxActiveThreads + 2))
{
restart = true;
}
if (!restart)
{
return;
}
createThreadCount++;
var tnew = new WorkerPoolThread(this, scheduler, this.performanceMetrics, createThreadCount);
tnew.Start();
}
}
internal void RecordLeavingThread(WorkerPoolThread t)
{
bool restart = false;
lock (lockable)
{
pool.Remove(t);
if (running && (pool.Count < MaxActiveThreads + 2))
{
restart = true;
}
if (!restart)
{
return;
}
createThreadCount++;
var tnew = new WorkerPoolThread(this, scheduler, this.executorService, this.loggerFactory, this.hostStatistics, createThreadCount);
tnew.Start();
}
}
internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool enableWorkerThreadInjection)
{
scheduler = sched;
MaxActiveThreads = maxActiveThreads;
EnableWorkerThreadInjection = enableWorkerThreadInjection;
MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
if (EnableWorkerThreadInjection)
{
threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
}
pool = new HashSet<WorkerPoolThread>();
createThreadCount = 0;
lockable = new object();
for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
{
var t = new WorkerPoolThread(this, scheduler, createThreadCount);
pool.Add(t);
}
createThreadCount++;
systemThread = new WorkerPoolThread(this, scheduler, createThreadCount, true);
running = false;
runningThreadCount = 0;
longTurnTimer = null;
}
protected override void Run()
{
try
{
// We can't set these in the constructor because that doesn't run on our thread
current = this;
RuntimeContext.InitializeThread(scheduler);
int noWorkCount = 0;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStartExecution();
}
#endif
// Until we're cancelled...
while (!Cts.IsCancellationRequested)
{
// Wait for a CPU
if (!IsSystem)
{
TakeCpu();
}
try
{
#if DEBUG
if (Log.IsVerbose3)
{
Log.Verbose3("Worker thread {0} - Waiting for {1} work item", this.ManagedThreadId, IsSystem ? "System" : "Any");
}
#endif
// Get some work to do
IWorkItem todo;
todo = IsSystem ? scheduler.RunQueue.GetSystem(Cts.Token, maxWorkQueueWait) :
scheduler.RunQueue.Get(Cts.Token, maxWorkQueueWait);
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStartProcessing();
}
#endif
if (todo != null)
{
if (!IsSystem)
{
pool.RecordRunningThread();
}
// Capture the queue wait time for this task
TimeSpan waitTime = todo.TimeSinceQueued;
if (waitTime > scheduler.DelayWarningThreshold && !Debugger.IsAttached)
{
SchedulerStatisticsGroup.NumLongQueueWaitTimes.Increment();
Log.Warn(ErrorCode.SchedulerWorkerPoolThreadQueueWaitTime, "Queue wait time of {0} for Item {1}", waitTime, todo);
}
#if DEBUG
if (Log.IsVerbose3)
{
Log.Verbose3("Queue wait time for {0} work item is {1}", todo.ItemType, waitTime);
}
#endif
// Do the work
try
{
RuntimeContext.SetExecutionContext(todo.SchedulingContext, scheduler);
if (todo.ItemType != WorkItemType.WorkItemGroup)
{
// for WorkItemGroup we will track CurrentWorkItem inside WorkItemGroup.
CurrentWorkItem = todo;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectTurnsStats)
{
SchedulerStatisticsGroup.OnThreadStartsTurnExecution(WorkerThreadStatisticsNumber, todo.SchedulingContext);
}
#endif
}
todo.Execute();
}
catch (ThreadAbortException ex)
{
// The current turn was aborted (indicated by the exception state being set to true).
// In this case, we just reset the abort so that life continues. No need to do anything else.
if ((ex.ExceptionState != null) && ex.ExceptionState.Equals(true))
{
Thread.ResetAbort();
}
else
{
Log.Error(ErrorCode.Runtime_Error_100029, "Caught thread abort exception, allowing it to propagate outwards", ex);
}
}
catch (Exception ex)
{
var errorStr = String.Format("Worker thread caught an exception thrown from task {0}.", todo);
Log.Error(ErrorCode.Runtime_Error_100030, errorStr, ex);
}
finally
{
#if TRACK_DETAILED_STATS
if (todo.ItemType != WorkItemType.WorkItemGroup)
{
if (StatisticsCollector.CollectTurnsStats)
{
//SchedulerStatisticsGroup.OnTurnExecutionEnd(CurrentStateTime.Elapsed);
SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(CurrentStateStarted));
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.IncrementNumberOfProcessed();
}
CurrentWorkItem = null;
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStopProcessing();
}
#endif
if (!IsSystem)
{
pool.RecordIdlingThread();
}
RuntimeContext.ResetExecutionContext();
noWorkCount = 0;
}
}
else // todo was null -- no work to do
{
if (Cts.IsCancellationRequested)
{
// Cancelled -- we're done
// Note that the finally block will release the CPU, since it will get invoked
// even for a break or a return
break;
}
noWorkCount++;
}
}
catch (ThreadAbortException tae)
{
// Can be reported from RunQueue.Get when Silo is being shutdown, so downgrade to verbose log
if (Log.IsVerbose)
{
Log.Verbose("Received thread abort exception -- exiting. {0}", tae);
}
Thread.ResetAbort();
break;
}
catch (Exception ex)
{
Log.Error(ErrorCode.Runtime_Error_100031, "Exception bubbled up to worker thread", ex);
break;
}
finally
{
CurrentWorkItem = null; // Also sets CurrentTask to null
// Release the CPU
if (!IsSystem)
{
PutCpu();
}
}
// If we've gone a minute without any work to do, let's give up
if (!IsSystem && (maxWorkQueueWait.Multiply(noWorkCount) > TimeSpan.FromMinutes(1)) && pool.CanExit())
{
#if DEBUG
if (Log.IsVerbose)
{
Log.Verbose("Scheduler thread leaving because there's not enough work to do");
}
#endif
break;
}
}
}
catch (Exception exc)
{
Log.Error(ErrorCode.SchedulerWorkerThreadExc, "WorkerPoolThread caugth exception:", exc);
}
finally
{
if (!IsSystem)
{
pool.RecordLeavingThread(this);
}
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStopExecution();
}
#endif
CurrentWorkItem = null;
}
}
protected override void Run()
{
try
{
// We can't set these in the constructor because that doesn't run on our thread
current = this;
RuntimeContext.InitializeThread(scheduler);
int noWorkCount = 0;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStartExecution();
}
#endif
// Until we're cancelled...
while (!Cts.IsCancellationRequested)
{
// Wait for a CPU
if (!IsSystem)
TakeCpu();
try
{
#if DEBUG
if (Log.IsVerbose3) Log.Verbose3("Worker thread {0} - Waiting for {1} work item", this.ManagedThreadId, IsSystem ? "System" : "Any");
#endif
// Get some work to do
IWorkItem todo;
todo = IsSystem ? scheduler.RunQueue.GetSystem(Cts.Token, maxWorkQueueWait) :
scheduler.RunQueue.Get(Cts.Token, maxWorkQueueWait);
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStartProcessing();
}
#endif
if (todo != null)
{
if (!IsSystem)
pool.RecordRunningThread();
// Capture the queue wait time for this task
TimeSpan waitTime = todo.TimeSinceQueued;
if (waitTime > scheduler.DelayWarningThreshold && !Debugger.IsAttached)
{
SchedulerStatisticsGroup.NumLongQueueWaitTimes.Increment();
Log.Warn(ErrorCode.SchedulerWorkerPoolThreadQueueWaitTime, "Queue wait time of {0} for Item {1}", waitTime, todo);
}
#if DEBUG
if (Log.IsVerbose3) Log.Verbose3("Queue wait time for {0} work item is {1}", todo.ItemType, waitTime);
#endif
// Do the work
try
{
RuntimeContext.SetExecutionContext(todo.SchedulingContext, scheduler);
CurrentWorkItem = todo;
#if TRACK_DETAILED_STATS
if (todo.ItemType != WorkItemType.WorkItemGroup)
{
if (StatisticsCollector.CollectTurnsStats)
{
SchedulerStatisticsGroup.OnThreadStartsTurnExecution(WorkerThreadStatisticsNumber, todo.SchedulingContext);
}
}
#endif
todo.Execute();
}
catch (ThreadAbortException ex)
{
// The current turn was aborted (indicated by the exception state being set to true).
// In this case, we just reset the abort so that life continues. No need to do anything else.
if ((ex.ExceptionState != null) && ex.ExceptionState.Equals(true))
Thread.ResetAbort();
else
Log.Error(ErrorCode.Runtime_Error_100029, "Caught thread abort exception, allowing it to propagate outwards", ex);
}
catch (Exception ex)
{
var errorStr = String.Format("Worker thread caught an exception thrown from task {0}.", todo);
Log.Error(ErrorCode.Runtime_Error_100030, errorStr, ex);
}
finally
{
#if TRACK_DETAILED_STATS
if (todo.ItemType != WorkItemType.WorkItemGroup)
{
if (StatisticsCollector.CollectTurnsStats)
{
//SchedulerStatisticsGroup.OnTurnExecutionEnd(CurrentStateTime.Elapsed);
SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(CurrentStateStarted));
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.IncrementNumberOfProcessed();
}
CurrentWorkItem = null;
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStopProcessing();
}
#endif
if (!IsSystem)
pool.RecordIdlingThread();
RuntimeContext.ResetExecutionContext();
noWorkCount = 0;
}
}
else // todo was null -- no work to do
{
if (Cts.IsCancellationRequested)
{
// Cancelled -- we're done
// Note that the finally block will release the CPU, since it will get invoked
// even for a break or a return
break;
}
noWorkCount++;
}
}
catch (ThreadAbortException tae)
{
// Can be reported from RunQueue.Get when Silo is being shutdown, so downgrade to verbose log
if (Log.IsVerbose) Log.Verbose("Received thread abort exception -- exiting. {0}", tae);
Thread.ResetAbort();
break;
}
catch (Exception ex)
{
Log.Error(ErrorCode.Runtime_Error_100031, "Exception bubbled up to worker thread", ex);
break;
}
finally
{
CurrentWorkItem = null; // Also sets CurrentTask to null
// Release the CPU
if (!IsSystem)
PutCpu();
}
// If we've gone a minute without any work to do, let's give up
if (!IsSystem && (maxWorkQueueWait.Multiply(noWorkCount) > TimeSpan.FromMinutes(1)) && pool.CanExit())
{
#if DEBUG
if (Log.IsVerbose) Log.Verbose("Scheduler thread leaving because there's not enough work to do");
#endif
break;
}
}
}
catch (Exception exc)
{
Log.Error(ErrorCode.SchedulerWorkerThreadExc, "WorkerPoolThread caugth exception:", exc);
}
finally
{
if (!IsSystem)
pool.RecordLeavingThread(this);
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
threadTracking.OnStopExecution();
}
#endif
CurrentWorkItem = null;
}
}
internal void CreateNewThread()
{
lock (lockable)
{
createThreadCount++;
var t = new WorkerPoolThread(this, scheduler, createThreadCount);
pool.Add(t);
t.Start();
}
}
internal void RecordLeavingThread(WorkerPoolThread t)
{
bool restart = false;
lock (lockable)
{
pool.Remove(t);
if (running && (pool.Count < MaxActiveThreads + 2))
restart = true;
if (!restart) return;
createThreadCount++;
var tnew = new WorkerPoolThread(this, scheduler, createThreadCount);
tnew.Start();
}
}