Exemplo n.º 1
0
 internal WorkerPool(OrleansTaskScheduler sched, IHostEnvironmentStatistics hostStatistics, ExecutorService executorService, ILoggerFactory loggerFactory, int maxActiveThreads, bool enableWorkerThreadInjection)
 {
     scheduler                   = sched;
     this.hostStatistics         = hostStatistics;
     MaxActiveThreads            = maxActiveThreads;
     EnableWorkerThreadInjection = enableWorkerThreadInjection;
     MaxWorkQueueWait            = TimeSpan.FromMilliseconds(50);
     this.executorService        = executorService;
     this.loggerFactory          = loggerFactory;
     if (EnableWorkerThreadInjection)
     {
         threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
     }
     pool = new HashSet <WorkerPoolThread>();
     createThreadCount = 0;
     lockable          = new object();
     for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
     {
         var t = new WorkerPoolThread(this, scheduler, executorService, loggerFactory, hostStatistics, createThreadCount);
         pool.Add(t);
     }
     createThreadCount++;
     systemThread       = new WorkerPoolThread(this, scheduler, executorService, loggerFactory, hostStatistics, createThreadCount, true);
     running            = false;
     runningThreadCount = 0;
     longTurnTimer      = null;
     this.timerLogger   = loggerFactory.CreateLogger <SafeTimer>();
 }
Exemplo n.º 2
0
 internal WorkerPool(OrleansTaskScheduler sched, ICorePerformanceMetrics performanceMetrics, int maxActiveThreads, bool enableWorkerThreadInjection)
 {
     scheduler                   = sched;
     MaxActiveThreads            = maxActiveThreads;
     EnableWorkerThreadInjection = enableWorkerThreadInjection;
     MaxWorkQueueWait            = TimeSpan.FromMilliseconds(50);
     this.performanceMetrics     = performanceMetrics;
     if (EnableWorkerThreadInjection)
     {
         threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
     }
     pool = new HashSet <WorkerPoolThread>();
     createThreadCount = 0;
     lockable          = new object();
     for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
     {
         var t = new WorkerPoolThread(this, scheduler, performanceMetrics, createThreadCount);
         pool.Add(t);
     }
     createThreadCount++;
     systemThread       = new WorkerPoolThread(this, scheduler, performanceMetrics, createThreadCount, true);
     running            = false;
     runningThreadCount = 0;
     longTurnTimer      = null;
 }
Exemplo n.º 3
0
 internal void CreateNewThread()
 {
     lock (lockable)
     {
         var t = new WorkerPoolThread(this, scheduler);
         pool.Add(t);
         t.Start();
     }
 }
Exemplo n.º 4
0
 internal void CreateNewThread()
 {
     lock (lockable)
     {
         createThreadCount++;
         var t = new WorkerPoolThread(this, scheduler, this.performanceMetrics, createThreadCount);
         pool.Add(t);
         t.Start();
     }
 }
Exemplo n.º 5
0
 internal void CreateNewThread()
 {
     lock (lockable)
     {
         createThreadCount++;
         var t = new WorkerPoolThread(this, scheduler, this.executorService, this.loggerFactory, this.hostStatistics, createThreadCount);
         pool.Add(t);
         t.Start();
     }
 }
Exemplo n.º 6
0
 internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool injectMoreWorkerThreads)
 {
     scheduler               = sched;
     MaxActiveThreads        = maxActiveThreads;
     InjectMoreWorkerThreads = injectMoreWorkerThreads;
     MaxWorkQueueWait        = TimeSpan.FromMilliseconds(50);
     threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
     pool     = new HashSet <WorkerPoolThread>();
     lockable = new object();
     for (int i = 0; i < MaxActiveThreads; i++)
     {
         var t = new WorkerPoolThread(this, scheduler);
         pool.Add(t);
     }
     systemThread       = new WorkerPoolThread(this, scheduler, true);
     running            = false;
     runningThreadCount = 0;
     longTurnTimer      = null;
 }
Exemplo n.º 7
0
 internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool injectMoreWorkerThreads)
 {
     scheduler = sched;
     MaxActiveThreads = maxActiveThreads;
     InjectMoreWorkerThreads = injectMoreWorkerThreads;
     MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
     threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
     pool = new HashSet<WorkerPoolThread>();
     lockable = new object();
     for (int i = 0; i < MaxActiveThreads; i++)
     {
         var t = new WorkerPoolThread(this, scheduler);
         pool.Add(t);
     }
     systemThread = new WorkerPoolThread(this, scheduler, true);
     running = false;
     runningThreadCount = 0;
     longTurnTimer = null;
 }
Exemplo n.º 8
0
        internal void RecordLeavingThread(WorkerPoolThread t)
        {
            bool restart = false;

            lock (lockable)
            {
                pool.Remove(t);
                if (running && (pool.Count < MaxActiveThreads + 2))
                {
                    restart = true;
                }
            }
            if (!restart)
            {
                return;
            }

            var tnew = new WorkerPoolThread(this, scheduler);

            tnew.Start();
        }
Exemplo n.º 9
0
        internal void RecordLeavingThread(WorkerPoolThread t)
        {
            bool restart = false;

            lock (lockable)
            {
                pool.Remove(t);
                if (running && (pool.Count < MaxActiveThreads + 2))
                {
                    restart = true;
                }

                if (!restart)
                {
                    return;
                }

                createThreadCount++;
                var tnew = new WorkerPoolThread(this, scheduler, this.performanceMetrics, createThreadCount);
                tnew.Start();
            }
        }
Exemplo n.º 10
0
        internal void RecordLeavingThread(WorkerPoolThread t)
        {
            bool restart = false;

            lock (lockable)
            {
                pool.Remove(t);
                if (running && (pool.Count < MaxActiveThreads + 2))
                {
                    restart = true;
                }

                if (!restart)
                {
                    return;
                }

                createThreadCount++;
                var tnew = new WorkerPoolThread(this, scheduler, this.executorService, this.loggerFactory, this.hostStatistics, createThreadCount);
                tnew.Start();
            }
        }
Exemplo n.º 11
0
 internal WorkerPool(OrleansTaskScheduler sched, int maxActiveThreads, bool enableWorkerThreadInjection)
 {
     scheduler = sched;
     MaxActiveThreads = maxActiveThreads;
     EnableWorkerThreadInjection = enableWorkerThreadInjection;
     MaxWorkQueueWait = TimeSpan.FromMilliseconds(50);
     if (EnableWorkerThreadInjection)
     {
         threadLimitingSemaphore = new Semaphore(maxActiveThreads, maxActiveThreads);
     }
     pool = new HashSet<WorkerPoolThread>();
     createThreadCount = 0;
     lockable = new object();
     for (createThreadCount = 0; createThreadCount < MaxActiveThreads; createThreadCount++)
     {
         var t = new WorkerPoolThread(this, scheduler, createThreadCount);
         pool.Add(t);
     }
     createThreadCount++;
     systemThread = new WorkerPoolThread(this, scheduler, createThreadCount, true);
     running = false;
     runningThreadCount = 0;
     longTurnTimer = null;
 }
Exemplo n.º 12
0
        protected override void Run()
        {
            try
            {
                // We can't set these in the constructor because that doesn't run on our thread
                current = this;
                RuntimeContext.InitializeThread(scheduler);

                int noWorkCount = 0;

#if TRACK_DETAILED_STATS
                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                {
                    threadTracking.OnStartExecution();
                }
#endif

                // Until we're cancelled...
                while (!Cts.IsCancellationRequested)
                {
                    // Wait for a CPU
                    if (!IsSystem)
                    {
                        TakeCpu();
                    }

                    try
                    {
#if DEBUG
                        if (Log.IsVerbose3)
                        {
                            Log.Verbose3("Worker thread {0} - Waiting for {1} work item", this.ManagedThreadId, IsSystem ? "System" : "Any");
                        }
#endif
                        // Get some work to do
                        IWorkItem todo;

                        todo = IsSystem ? scheduler.RunQueue.GetSystem(Cts.Token, maxWorkQueueWait) :
                               scheduler.RunQueue.Get(Cts.Token, maxWorkQueueWait);

#if TRACK_DETAILED_STATS
                        if (StatisticsCollector.CollectThreadTimeTrackingStats)
                        {
                            threadTracking.OnStartProcessing();
                        }
#endif
                        if (todo != null)
                        {
                            if (!IsSystem)
                            {
                                pool.RecordRunningThread();
                            }


                            // Capture the queue wait time for this task
                            TimeSpan waitTime = todo.TimeSinceQueued;
                            if (waitTime > scheduler.DelayWarningThreshold && !Debugger.IsAttached)
                            {
                                SchedulerStatisticsGroup.NumLongQueueWaitTimes.Increment();
                                Log.Warn(ErrorCode.SchedulerWorkerPoolThreadQueueWaitTime, "Queue wait time of {0} for Item {1}", waitTime, todo);
                            }
#if DEBUG
                            if (Log.IsVerbose3)
                            {
                                Log.Verbose3("Queue wait time for {0} work item is {1}", todo.ItemType, waitTime);
                            }
#endif
                            // Do the work
                            try
                            {
                                RuntimeContext.SetExecutionContext(todo.SchedulingContext, scheduler);
                                if (todo.ItemType != WorkItemType.WorkItemGroup)
                                {
                                    // for WorkItemGroup we will track CurrentWorkItem inside WorkItemGroup.
                                    CurrentWorkItem = todo;
#if TRACK_DETAILED_STATS
                                    if (StatisticsCollector.CollectTurnsStats)
                                    {
                                        SchedulerStatisticsGroup.OnThreadStartsTurnExecution(WorkerThreadStatisticsNumber, todo.SchedulingContext);
                                    }
#endif
                                }
                                todo.Execute();
                            }
                            catch (ThreadAbortException ex)
                            {
                                // The current turn was aborted (indicated by the exception state being set to true).
                                // In this case, we just reset the abort so that life continues. No need to do anything else.
                                if ((ex.ExceptionState != null) && ex.ExceptionState.Equals(true))
                                {
                                    Thread.ResetAbort();
                                }
                                else
                                {
                                    Log.Error(ErrorCode.Runtime_Error_100029, "Caught thread abort exception, allowing it to propagate outwards", ex);
                                }
                            }
                            catch (Exception ex)
                            {
                                var errorStr = String.Format("Worker thread caught an exception thrown from task {0}.", todo);
                                Log.Error(ErrorCode.Runtime_Error_100030, errorStr, ex);
                            }
                            finally
                            {
#if TRACK_DETAILED_STATS
                                if (todo.ItemType != WorkItemType.WorkItemGroup)
                                {
                                    if (StatisticsCollector.CollectTurnsStats)
                                    {
                                        //SchedulerStatisticsGroup.OnTurnExecutionEnd(CurrentStateTime.Elapsed);
                                        SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(CurrentStateStarted));
                                    }
                                    if (StatisticsCollector.CollectThreadTimeTrackingStats)
                                    {
                                        threadTracking.IncrementNumberOfProcessed();
                                    }
                                    CurrentWorkItem = null;
                                }
                                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                                {
                                    threadTracking.OnStopProcessing();
                                }
#endif
                                if (!IsSystem)
                                {
                                    pool.RecordIdlingThread();
                                }

                                RuntimeContext.ResetExecutionContext();
                                noWorkCount = 0;
                            }
                        }
                        else // todo was null -- no work to do
                        {
                            if (Cts.IsCancellationRequested)
                            {
                                // Cancelled -- we're done
                                // Note that the finally block will release the CPU, since it will get invoked
                                // even for a break or a return
                                break;
                            }
                            noWorkCount++;
                        }
                    }
                    catch (ThreadAbortException tae)
                    {
                        // Can be reported from RunQueue.Get when Silo is being shutdown, so downgrade to verbose log
                        if (Log.IsVerbose)
                        {
                            Log.Verbose("Received thread abort exception -- exiting. {0}", tae);
                        }
                        Thread.ResetAbort();
                        break;
                    }
                    catch (Exception ex)
                    {
                        Log.Error(ErrorCode.Runtime_Error_100031, "Exception bubbled up to worker thread", ex);
                        break;
                    }
                    finally
                    {
                        CurrentWorkItem = null; // Also sets CurrentTask to null

                        // Release the CPU
                        if (!IsSystem)
                        {
                            PutCpu();
                        }
                    }

                    // If we've gone a minute without any work to do, let's give up
                    if (!IsSystem && (maxWorkQueueWait.Multiply(noWorkCount) > TimeSpan.FromMinutes(1)) && pool.CanExit())
                    {
#if DEBUG
                        if (Log.IsVerbose)
                        {
                            Log.Verbose("Scheduler thread leaving because there's not enough work to do");
                        }
#endif
                        break;
                    }
                }
            }
            catch (Exception exc)
            {
                Log.Error(ErrorCode.SchedulerWorkerThreadExc, "WorkerPoolThread caugth exception:", exc);
            }
            finally
            {
                if (!IsSystem)
                {
                    pool.RecordLeavingThread(this);
                }

#if TRACK_DETAILED_STATS
                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                {
                    threadTracking.OnStopExecution();
                }
#endif
                CurrentWorkItem = null;
            }
        }
Exemplo n.º 13
0
        protected override void Run()
        {
            try
            {
                // We can't set these in the constructor because that doesn't run on our thread
                current = this;
                RuntimeContext.InitializeThread(scheduler);
                
                int noWorkCount = 0;
                
#if TRACK_DETAILED_STATS
                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                {
                    threadTracking.OnStartExecution();
                }
#endif

                // Until we're cancelled...
                while (!Cts.IsCancellationRequested)
                {
                    // Wait for a CPU
                    if (!IsSystem)
                        TakeCpu();
                    
                    try
                    {
#if DEBUG
                        if (Log.IsVerbose3) Log.Verbose3("Worker thread {0} - Waiting for {1} work item", this.ManagedThreadId, IsSystem ? "System" : "Any");
#endif
                        // Get some work to do
                        IWorkItem todo;

                        todo = IsSystem ? scheduler.RunQueue.GetSystem(Cts.Token, maxWorkQueueWait) : 
                            scheduler.RunQueue.Get(Cts.Token, maxWorkQueueWait);

#if TRACK_DETAILED_STATS
                        if (StatisticsCollector.CollectThreadTimeTrackingStats)
                        {
                            threadTracking.OnStartProcessing();
                        }
#endif
                        if (todo != null)
                        {
                            if (!IsSystem)
                                pool.RecordRunningThread();
                            

                            // Capture the queue wait time for this task
                            TimeSpan waitTime = todo.TimeSinceQueued;
                            if (waitTime > scheduler.DelayWarningThreshold && !Debugger.IsAttached)
                            {
                                SchedulerStatisticsGroup.NumLongQueueWaitTimes.Increment();
                                Log.Warn(ErrorCode.SchedulerWorkerPoolThreadQueueWaitTime, "Queue wait time of {0} for Item {1}", waitTime, todo);
                            }
#if DEBUG
                            if (Log.IsVerbose3) Log.Verbose3("Queue wait time for {0} work item is {1}", todo.ItemType, waitTime);
#endif
                            // Do the work
                            try
                            {
                                RuntimeContext.SetExecutionContext(todo.SchedulingContext, scheduler);
                                CurrentWorkItem = todo;
#if TRACK_DETAILED_STATS
                                if (todo.ItemType != WorkItemType.WorkItemGroup)
                                {
                                    if (StatisticsCollector.CollectTurnsStats)
                                    {
                                        SchedulerStatisticsGroup.OnThreadStartsTurnExecution(WorkerThreadStatisticsNumber, todo.SchedulingContext);
                                    }
                                }
#endif
                                todo.Execute();
                            }
                            catch (ThreadAbortException ex)
                            {
                                // The current turn was aborted (indicated by the exception state being set to true).
                                // In this case, we just reset the abort so that life continues. No need to do anything else.
                                if ((ex.ExceptionState != null) && ex.ExceptionState.Equals(true))
                                    Thread.ResetAbort();
                                else
                                    Log.Error(ErrorCode.Runtime_Error_100029, "Caught thread abort exception, allowing it to propagate outwards", ex);
                            }
                            catch (Exception ex)
                            {
                                var errorStr = String.Format("Worker thread caught an exception thrown from task {0}.", todo);
                                Log.Error(ErrorCode.Runtime_Error_100030, errorStr, ex);
                            }
                            finally
                            {
#if TRACK_DETAILED_STATS
                                if (todo.ItemType != WorkItemType.WorkItemGroup)
                                {
                                    if (StatisticsCollector.CollectTurnsStats)
                                    {
                                        //SchedulerStatisticsGroup.OnTurnExecutionEnd(CurrentStateTime.Elapsed);
                                        SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(CurrentStateStarted));
                                    }
                                    if (StatisticsCollector.CollectThreadTimeTrackingStats)
                                    {
                                        threadTracking.IncrementNumberOfProcessed();
                                    }
                                    CurrentWorkItem = null;
                                }
                                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                                {
                                    threadTracking.OnStopProcessing();
                                }
#endif
                                if (!IsSystem)
                                    pool.RecordIdlingThread();
                                
                                RuntimeContext.ResetExecutionContext();
                                noWorkCount = 0;
                            }
                        }
                        else // todo was null -- no work to do
                        {
                            if (Cts.IsCancellationRequested)
                            {
                                // Cancelled -- we're done
                                // Note that the finally block will release the CPU, since it will get invoked
                                // even for a break or a return
                                break;
                            }
                            noWorkCount++;
                        }
                    }
                    catch (ThreadAbortException tae)
                    {
                        // Can be reported from RunQueue.Get when Silo is being shutdown, so downgrade to verbose log
                        if (Log.IsVerbose) Log.Verbose("Received thread abort exception -- exiting. {0}", tae);
                        Thread.ResetAbort();
                        break;
                    }
                    catch (Exception ex)
                    {
                        Log.Error(ErrorCode.Runtime_Error_100031, "Exception bubbled up to worker thread", ex);
                        break;
                    }
                    finally
                    {
                        CurrentWorkItem = null; // Also sets CurrentTask to null

                        // Release the CPU
                        if (!IsSystem)
                            PutCpu();
                    }

                    // If we've gone a minute without any work to do, let's give up
                    if (!IsSystem && (maxWorkQueueWait.Multiply(noWorkCount) > TimeSpan.FromMinutes(1)) && pool.CanExit())
                    {
#if DEBUG
                        if (Log.IsVerbose) Log.Verbose("Scheduler thread leaving because there's not enough work to do");
#endif
                        break;
                    }
                }
            }
            catch (Exception exc)
            {
                Log.Error(ErrorCode.SchedulerWorkerThreadExc, "WorkerPoolThread caugth exception:", exc);
            }
            finally
            {
                if (!IsSystem)
                    pool.RecordLeavingThread(this);
                
#if TRACK_DETAILED_STATS
                if (StatisticsCollector.CollectThreadTimeTrackingStats)
                {
                    threadTracking.OnStopExecution();
                }
#endif
                CurrentWorkItem = null;
            }
        }
Exemplo n.º 14
0
 internal void CreateNewThread()
 {
     lock (lockable)
     {
         createThreadCount++;
         var t = new WorkerPoolThread(this, scheduler, createThreadCount);
         pool.Add(t);
         t.Start();
     }
 }
Exemplo n.º 15
0
        internal void RecordLeavingThread(WorkerPoolThread t)
        {
            bool restart = false;
            lock (lockable)
            {
                pool.Remove(t);
                if (running && (pool.Count < MaxActiveThreads + 2))
                    restart = true;

                if (!restart) return;

                createThreadCount++;
                var tnew = new WorkerPoolThread(this, scheduler, createThreadCount);
                tnew.Start();
            }
        }