internal static void InitSchedulerLogging()
{
TraceLogger.UnInitialize();
//TraceLogger.LogConsumers.Add(new LogWriterToConsole());
if (!Logger.TelemetryConsumers.OfType <ConsoleTelemetryConsumer>().Any())
{
Logger.TelemetryConsumers.Add(new ConsoleTelemetryConsumer());
}
var traceLevels = new[]
{
Tuple.Create("Scheduler", Severity.Verbose3),
Tuple.Create("Scheduler.WorkerPoolThread", Severity.Verbose2),
};
TraceLogger.SetTraceLevelOverrides(new List <Tuple <string, Severity> >(traceLevels));
var orleansConfig = new ClusterConfiguration();
orleansConfig.StandardLoad();
NodeConfiguration config = orleansConfig.GetOrAddConfigurationForNode("Primary");
StatisticsCollector.Initialize(config);
SchedulerStatisticsGroup.Init();
}
public ThreadPoolExecutor(
ThreadPoolExecutorOptions options,
SchedulerStatisticsGroup schedulerStatistics,
StageAnalysisStatisticsGroup schedulerStageStatistics,
IOptions <StatisticsOptions> statisticsOptions)
{
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.schedulerStatistics = schedulerStatistics;
this.schedulerStageStatistics = schedulerStageStatistics;
this.statisticsOptions = statisticsOptions;
workQueue = new ThreadPoolWorkQueue();
statistic = new ThreadPoolTrackingStatistic(options.Name, options.LoggerFactory, statisticsOptions, schedulerStageStatistics);
executingWorkTracker = new ExecutingWorkItemsTracker(this);
log = options.LoggerFactory.CreateLogger <ThreadPoolExecutor>();
options.CancellationTokenSource.Token.Register(Complete);
for (var threadIndex = 0; threadIndex < options.DegreeOfParallelism; threadIndex++)
{
RunWorker(threadIndex);
}
}
public IWorkItem Get(CancellationToken ct, TimeSpan timeout)
{
try
{
IWorkItem todo;
#if PRIORITIZE_SYSTEM_TASKS
// TryTakeFromAny is a static method with no state held from one call to another, so each request is independent,
// and it doesn’t attempt to randomize where it next takes from, and does not provide any level of fairness across collections.
// It has a “fast path” that just iterates over the collections from 0 to N to see if any of the collections already have data,
// and if it finds one, it takes from that collection without considering the others, so it will bias towards the earlier collections.
// If none of the collections has data, then it will fall through to the “slow path” of waiting on a collection of wait handles,
// one for each collection, at which point it’s subject to the fairness provided by the OS with regards to waiting on events.
if (BlockingCollection <IWorkItem> .TryTakeFromAny(queueArray, out todo, timeout) >= 0)
#else
if (mainQueue.TryTake(out todo, timeout))
#endif
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemDequeue();
}
#endif
return(todo);
}
return(null);
}
catch (InvalidOperationException)
{
return(null);
}
}
public OrleansTaskScheduler(
IOptions <SchedulingOptions> options,
ILoggerFactory loggerFactory,
SchedulerStatisticsGroup schedulerStatistics,
IOptions <StatisticsOptions> statisticsOptions)
{
this.schedulerStatistics = schedulerStatistics;
this.statisticsOptions = statisticsOptions;
this.logger = loggerFactory.CreateLogger <OrleansTaskScheduler>();
this.workItemGroupLogger = loggerFactory.CreateLogger <WorkItemGroup>();
this.activationTaskSchedulerLogger = loggerFactory.CreateLogger <ActivationTaskScheduler>();
this.cancellationTokenSource = new CancellationTokenSource();
this.SchedulingOptions = options.Value;
applicationTurnsStopped = false;
TurnWarningLengthThreshold = options.Value.TurnWarningLengthThreshold;
this.MaxPendingItemsSoftLimit = options.Value.MaxPendingWorkItemsSoftLimit;
this.StoppedWorkItemGroupWarningInterval = options.Value.StoppedActivationWarningInterval;
workgroupDirectory = new ConcurrentDictionary <IGrainContext, WorkItemGroup>();
IntValueStatistic.FindOrCreate(StatisticNames.SCHEDULER_WORKITEMGROUP_COUNT, () => WorkItemGroupCount);
if (!schedulerStatistics.CollectShedulerQueuesStats)
{
return;
}
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageRunQueueLengthLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageEnqueuedLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageArrivalRateLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumRunQueueLengthLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumEnqueuedLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumArrivalRateLevelTwo);
}
public StatisticsTracker(ThreadPoolTrackingStatistic statistic, TimeSpan delayWarningThreshold, ILogger log, SchedulerStatisticsGroup schedulerStatistics)
{
this.statistic = statistic;
this.delayWarningThreshold = delayWarningThreshold;
this.log = log;
this.schedulerStatistics = schedulerStatistics;
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
IOptions <LoadSheddingOptions> loadSheddingOptions,
IOptions <StorageOptions> azureStorageOptions,
ILocalSiloDetails siloDetails,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
IHostEnvironmentStatistics hostEnvironmentStatistics,
IAppEnvironmentStatistics appEnvironmentStatistics,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
this.storageOptions = azureStorageOptions.Value;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public IWorkItem GetSystem(CancellationToken ct, TimeSpan timeout)
{
try
{
IWorkItem todo;
#if PRIORITIZE_SYSTEM_TASKS
if (systemQueue.TryTake(out todo, timeout))
#else
if (mainQueue.TryTake(out todo, timeout))
#endif
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemDequeue();
}
#endif
return(todo);
}
return(null);
}
catch (InvalidOperationException)
{
return(null);
}
}
public OrleansTaskScheduler(
IOptions <SchedulingOptions> options,
ExecutorService executorService,
ILoggerFactory loggerFactory,
SchedulerStatisticsGroup schedulerStatistics,
IOptions <StatisticsOptions> statisticsOptions)
{
this.loggerFactory = loggerFactory;
this.schedulerStatistics = schedulerStatistics;
this.statisticsOptions = statisticsOptions;
this.logger = loggerFactory.CreateLogger <OrleansTaskScheduler>();
cancellationTokenSource = new CancellationTokenSource();
WorkItemGroup.ActivationSchedulingQuantum = options.Value.ActivationSchedulingQuantum;
applicationTurnsStopped = false;
TurnWarningLengthThreshold = options.Value.TurnWarningLengthThreshold;
this.MaxPendingItemsSoftLimit = options.Value.MaxPendingWorkItemsSoftLimit;
this.MaxPendingItemsHardLimit = options.Value.MaxPendingWorkItemsHardLimit;
workgroupDirectory = new ConcurrentDictionary <ISchedulingContext, WorkItemGroup>();
const int maxSystemThreads = 2;
var maxActiveThreads = options.Value.MaxActiveThreads;
maximumConcurrencyLevel = maxActiveThreads + maxSystemThreads;
OrleansSchedulerAsynchAgent CreateSchedulerAsynchAgent(string agentName, bool drainAfterCancel, int degreeOfParallelism)
{
return(new OrleansSchedulerAsynchAgent(
agentName,
executorService,
degreeOfParallelism,
options.Value.DelayWarningThreshold,
options.Value.TurnWarningLengthThreshold,
this,
drainAfterCancel,
loggerFactory));
}
mainAgent = CreateSchedulerAsynchAgent("Scheduler.LevelOne.MainQueue", false, maxActiveThreads);
systemAgent = CreateSchedulerAsynchAgent("Scheduler.LevelOne.SystemQueue", true, maxSystemThreads);
this.taskWorkItemLogger = loggerFactory.CreateLogger <TaskWorkItem>();
logger.Info("Starting OrleansTaskScheduler with {0} Max Active application Threads and 2 system thread.", maxActiveThreads);
IntValueStatistic.FindOrCreate(StatisticNames.SCHEDULER_WORKITEMGROUP_COUNT, () => WorkItemGroupCount);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_INSTANTANEOUS_PER_QUEUE, "Scheduler.LevelOne"), () => RunQueueLength);
if (!schedulerStatistics.CollectShedulerQueuesStats)
{
return;
}
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageRunQueueLengthLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageEnqueuedLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, "Scheduler.LevelTwo.Average"), () => AverageArrivalRateLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumRunQueueLengthLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumEnqueuedLevelTwo);
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, "Scheduler.LevelTwo.Sum"), () => SumArrivalRateLevelTwo);
}
/// <summary>
/// Adds a task to this activation.
/// If we're adding it to the run list and we used to be waiting, now we're runnable.
/// </summary>
/// <param name="task">The work item to add.</param>
public void EnqueueTask(Task task)
{
lock (lockable)
{
#if DEBUG
if (log.IsVerbose2)
{
log.Verbose2("EnqueueWorkItem {0} into {1} when TaskScheduler.Current={2}", task, SchedulingContext, TaskScheduler.Current);
}
#endif
if (state == WorkGroupStatus.Shutdown)
{
ReportWorkGroupProblem(
String.Format("Enqueuing task {0} to a stopped work item group. Going to ignore and not execute it. "
+ "The likely reason is that the task is not being 'awaited' properly.", task),
ErrorCode.SchedulerNotEnqueuWorkWhenShutdown);
task.Ignore(); // Ignore this Task, so in case it is faulted it will not cause UnobservedException.
return;
}
long thisSequenceNumber = totalItemsEnQueued++;
int count = WorkItemCount;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectShedulerQueuesStats)
{
queueTracking.OnEnQueueRequest(1, count);
}
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemEnqueue();
}
#endif
workItems.Enqueue(task);
int maxPendingItemsLimit = masterScheduler.MaxPendingItemsLimit.SoftLimitThreshold;
if (maxPendingItemsLimit > 0 && count > maxPendingItemsLimit)
{
log.Warn(ErrorCode.SchedulerTooManyPendingItems, String.Format("{0} pending work items for group {1}, exceeding the warning threshold of {2}",
count, Name, maxPendingItemsLimit));
}
if (state != WorkGroupStatus.Waiting)
{
return;
}
state = WorkGroupStatus.Runnable;
#if DEBUG
if (log.IsVerbose3)
{
log.Verbose3("Add to RunQueue {0}, #{1}, onto {2}", task, thisSequenceNumber, SchedulingContext);
}
#endif
masterScheduler.RunQueue.Add(this);
}
}
protected override void OnEnqueue(IWorkItem request)
{
base.OnEnqueue(request);
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemEnqueue();
}
#endif
}
public ClosureWorkItem(Action closure)
{
continuation = closure;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnClosureWorkItemsCreated();
}
#endif
}
public override void Execute()
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnClosureWorkItemsExecuted();
}
#endif
continuation();
}
internal static OrleansTaskScheduler InitializeSchedulerForTesting(ISchedulingContext context, ICorePerformanceMetrics performanceMetrics, ILoggerFactory loggerFactory)
{
StatisticsCollector.StatisticsCollectionLevel = StatisticsLevel.Info;
SchedulerStatisticsGroup.Init(loggerFactory);
var scheduler = OrleansTaskScheduler.CreateTestInstance(4, performanceMetrics, loggerFactory);
scheduler.Start();
WorkItemGroup ignore = scheduler.RegisterWorkContext(context);
return(scheduler);
}
public AsyncClosureWorkItem(Func <Task> closure, string name = null)
{
this.continuation = closure;
this.name = name;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnClosureWorkItemsCreated();
}
#endif
}
internal static OrleansTaskScheduler InitializeSchedulerForTesting(ISchedulingContext context)
{
StatisticsCollector.StatisticsCollectionLevel = StatisticsLevel.Info;
SchedulerStatisticsGroup.Init();
var scheduler = OrleansTaskScheduler.CreateTestInstance(4);
scheduler.Start();
WorkItemGroup ignore = scheduler.RegisterWorkContext(context);
return(scheduler);
}
public override void OnActionExecuted(ExecutionContext context)
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectTurnsStats)
{
if (agent.GetWorkItemState(context).ItemType != WorkItemType.WorkItemGroup)
{
SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(context.WorkItem.ExecutionStart));
}
}
#endif
}
internal SiloStatisticsManager(GlobalConfiguration globalConfig, NodeConfiguration nodeConfig)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(globalConfig.ResponseTimeout);
SchedulerStatisticsGroup.Init();
StorageStatisticsGroup.Init();
runtimeStats = new RuntimeStatisticsGroup();
logStatistics = new LogStatistics(nodeConfig.StatisticsLogWriteInterval, true);
MetricsTable = new SiloPerformanceMetrics(runtimeStats, nodeConfig);
perfCountersPublisher = new PerfCountersStatistics(nodeConfig.StatisticsPerfCountersWriteInterval);
}
public void Add(IWorkItem workItem)
{
workItem.TimeQueued = DateTime.UtcNow;
try
{
#if PRIORITIZE_SYSTEM_TASKS
if (workItem.IsSystemPriority)
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectShedulerQueuesStats)
{
systemQueueTracking.OnEnQueueRequest(1, systemQueue.Count);
}
#endif
systemQueue.Add(workItem);
}
else
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectShedulerQueuesStats)
{
mainQueueTracking.OnEnQueueRequest(1, mainQueue.Count);
}
#endif
mainQueue.Add(workItem);
}
#else
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectQueueStats)
{
mainQueueTracking.OnEnQueueRequest(1, mainQueue.Count);
}
#endif
mainQueue.Add(task);
#endif
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemEnqueue();
}
#endif
}
catch (InvalidOperationException)
{
// Queue has been stopped; ignore the exception
}
}
internal static ILoggerFactory InitSchedulerLogging()
{
var filters = new LoggerFilterOptions();
filters.AddFilter("Scheduler", LogLevel.Trace);
filters.AddFilter("Scheduler.WorkerPoolThread", LogLevel.Trace);
var orleansConfig = new ClusterConfiguration();
orleansConfig.StandardLoad();
NodeConfiguration config = orleansConfig.CreateNodeConfigurationForSilo("Primary");
var loggerFactory = TestingUtils.CreateDefaultLoggerFactory(TestingUtils.CreateTraceFileName(config.SiloName, orleansConfig.Globals.ClusterId), filters);
StatisticsCollector.Initialize(StatisticsLevel.Info);
SchedulerStatisticsGroup.Init(loggerFactory);
return(loggerFactory);
}
internal WorkerPoolThread(WorkerPool gtp, OrleansTaskScheduler sched, bool system = false)
: base(system ? "System" : null)
{
pool = gtp;
scheduler = sched;
ownsSemaphore = false;
IsSystem = system;
maxWorkQueueWait = IsSystem ? Constants.INFINITE_TIMESPAN : gtp.MaxWorkQueueWait;
OnFault = FaultBehavior.IgnoreFault;
CurrentStateStarted = DateTime.UtcNow;
CurrentWorkItem = null;
if (StatisticsCollector.CollectTurnsStats)
{
WorkerThreadStatisticsNumber = SchedulerStatisticsGroup.RegisterWorkingThread(Name);
}
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init();
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public SiloStatisticsManager(NodeConfiguration nodeConfiguration, ILocalSiloDetails siloDetails, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory, IOptions <MessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
runtimeStats = new RuntimeStatisticsGroup(loggerFactory);
this.logStatistics = new LogStatistics(nodeConfiguration.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, nodeConfiguration);
this.countersPublisher = new CountersStatistics(nodeConfiguration.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
internal WorkerPoolThread(WorkerPool gtp, OrleansTaskScheduler sched, ICorePerformanceMetrics performanceMetrics, int threadNumber, bool system = false)
: base((system ? "System." : "") + threadNumber)
{
pool = gtp;
scheduler = sched;
this.performanceMetrics = performanceMetrics;
ownsSemaphore = false;
IsSystem = system;
maxWorkQueueWait = IsSystem ? Constants.INFINITE_TIMESPAN : gtp.MaxWorkQueueWait;
OnFault = FaultBehavior.IgnoreFault;
currentWorkItemStarted = DateTime.UtcNow;
currentTaskStarted = DateTime.UtcNow;
CurrentWorkItem = null;
if (StatisticsCollector.CollectTurnsStats)
{
WorkerThreadStatisticsNumber = SchedulerStatisticsGroup.RegisterWorkingThread(Name);
}
}
internal WorkerPoolThread(WorkerPool gtp, OrleansTaskScheduler sched, ExecutorService executorService, ILoggerFactory loggerFactory, IHostEnvironmentStatistics hostEnvironmentStatistics, int threadNumber, bool system = false)
: base((system ? "System." : "") + threadNumber, executorService, loggerFactory)
{
pool = gtp;
scheduler = sched;
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
ownsSemaphore = false;
IsSystem = system;
maxWorkQueueWait = IsSystem ? Constants.INFINITE_TIMESPAN : gtp.MaxWorkQueueWait;
OnFault = FaultBehavior.IgnoreFault;
currentWorkItemStarted = DateTime.UtcNow;
currentTaskStarted = DateTime.UtcNow;
CurrentWorkItem = null;
if (StatisticsCollector.CollectTurnsStats)
{
WorkerThreadStatisticsNumber = SchedulerStatisticsGroup.RegisterWorkingThread(Name);
}
}
public SiloStatisticsManager(SiloInitializationParameters initializationParams)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(initializationParams.GlobalConfig.ResponseTimeout);
SchedulerStatisticsGroup.Init();
StorageStatisticsGroup.Init();
runtimeStats = new RuntimeStatisticsGroup();
this.logStatistics = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, initializationParams.NodeConfig);
this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval);
initializationParams.ClusterConfig.OnConfigChange(
"Defaults/LoadShedding",
() => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
false);
}
private void TrackWorkItemDequeue(ExecutionContext context)
{
#if TRACK_DETAILED_STATS
var todo = agent.GetWorkItemState(context);
if (todo.ItemType != WorkItemType.WorkItemGroup)
{
if (StatisticsCollector.CollectTurnsStats)
{
SchedulerStatisticsGroup.OnThreadStartsTurnExecution(context.ThreadIndex, todo.SchedulingContext);
}
}
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemDequeue();
}
#endif
}
/// <summary>
/// Shuts down this work item group so that it will not process any additional work items, even if they
/// have already been queued.
/// </summary>
internal void Stop()
{
lock (lockable)
{
if (IsActive)
{
ReportWorkGroupProblem(
String.Format("WorkItemGroup is being stoped while still active. workItemCount = {0}."
+ "The likely reason is that the task is not being 'awaited' properly.", WorkItemCount),
ErrorCode.SchedulerWorkGroupStopping);
}
if (state == WorkGroupStatus.Shutdown)
{
log.Warn(ErrorCode.SchedulerWorkGroupShuttingDown, "WorkItemGroup is already shutting down {0}", this.ToString());
return;
}
state = WorkGroupStatus.Shutdown;
if (StatisticsCollector.CollectPerWorkItemStats)
{
SchedulerStatisticsGroup.UnRegisterWorkItemGroup(workItemGroupStatisticsNumber);
}
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemDrop(WorkItemCount);
}
if (StatisticsCollector.CollectShedulerQueuesStats)
{
queueTracking.OnStopExecution();
}
foreach (Task task in workItems)
{
// Ignore all queued Tasks, so in case they are faulted they will not cause UnobservedException.
task.Ignore();
}
workItems.Clear();
}
}
public SiloStatisticsManager(SiloInitializationParameters initializationParams, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(initializationParams.ClusterConfig.Globals.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = new LoggerWrapper <SiloStatisticsManager>(loggerFactory);
runtimeStats = new RuntimeStatisticsGroup(loggerFactory);
this.logStatistics = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, initializationParams.NodeConfig);
this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
initializationParams.ClusterConfig.OnConfigChange(
"Defaults/LoadShedding",
() => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
false);
}
public override async void Execute()
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnClosureWorkItemsExecuted();
}
#endif
try
{
RequestContext.Clear();
await this.continuation();
this.completion.TrySetResult(true);
}
catch (Exception exception)
{
this.completion.TrySetException(exception);
}
}
internal static void InitSchedulerLogging()
{
LogManager.UnInitialize();
//LogManager.LogConsumers.Add(new LogWriterToConsole());
var traceLevels = new[]
{
Tuple.Create("Scheduler", Severity.Verbose3),
Tuple.Create("Scheduler.WorkerPoolThread", Severity.Verbose2),
};
LogManager.SetTraceLevelOverrides(new List <Tuple <string, Severity> >(traceLevels));
var orleansConfig = new ClusterConfiguration();
orleansConfig.StandardLoad();
NodeConfiguration config = orleansConfig.CreateNodeConfigurationForSilo("Primary");
StatisticsCollector.Initialize(config);
SchedulerStatisticsGroup.Init();
}
