internal static void Init(TimeSpan responseTimeout)
{
if (!StatisticsCollector.CollectApplicationRequestsStats)
{
return;
}
const CounterStorage storage = CounterStorage.LogAndTable;
appRequestsLatencyHistogram = ExponentialHistogramValueStatistic.Create_ExponentialHistogram_ForTiming(
StatisticNames.APP_REQUESTS_LATENCY_HISTOGRAM, NUM_APP_REQUESTS_EXP_LATENCY_HISTOGRAM_CATEGORIES);
timedOutRequests = CounterStatistic.FindOrCreate(StatisticNames.APP_REQUESTS_TIMED_OUT, storage);
totalAppRequests = CounterStatistic.FindOrCreate(StatisticNames.APP_REQUESTS_TOTAL_NUMBER_OF_REQUESTS, storage);
appRequestsTotalLatency = CounterStatistic.FindOrCreate(StatisticNames.APP_REQUESTS_LATENCY_TOTAL, false, storage, true);
appRequestsAverageLatency = FloatValueStatistic.FindOrCreate(
StatisticNames.APP_REQUESTS_LATENCY_AVERAGE,
() =>
{
long totalLatencyInTicks = appRequestsTotalLatency.GetCurrentValue();
if (totalLatencyInTicks == 0)
{
return(0);
}
long numReqs = totalAppRequests.GetCurrentValue();
long averageLatencyInTicks = (long)((double)totalLatencyInTicks / (double)numReqs);
return((float)Utils.TicksToMilliSeconds(averageLatencyInTicks));
}, storage);
}
private void TrackContextSwitches()
{
PerformanceCounterCategory allThreadsWithPerformanceCounters = new PerformanceCounterCategory("Thread");
PerformanceCounter[] performanceCountersForThisThread = null;
// Iterate over all "Thread" category performance counters on system (includes numerous processes)
foreach (string threadName in allThreadsWithPerformanceCounters.GetInstanceNames())
{
// Obtain those performance counters for the OrleansHost
if (threadName.Contains("OrleansHost") && threadName.EndsWith("/" + Thread.CurrentThread.ManagedThreadId))
{
performanceCountersForThisThread = allThreadsWithPerformanceCounters.GetCounters(threadName);
break;
}
}
// In the case that the performance was not obtained correctly (this condition is null), we simply will not have stats for context switches
if (performanceCountersForThisThread == null)
{
return;
}
// Look at all performance counters for this thread
foreach (PerformanceCounter performanceCounter in performanceCountersForThisThread)
{
// Find performance counter for context switches
if (performanceCounter.CounterName == CONTEXT_SWTICH_COUNTER_NAME)
{
// Use raw value for logging, should show total context switches
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_CONTEXT_SWITCHES, Name), () => (float)performanceCounter.RawValue, CounterStorage.LogOnly);
}
}
}
internal void Start()
{
if (!countersAvailable)
{
logger.Warn(ErrorCode.PerfCounterNotRegistered,
"CPU & Memory perf counters did not initialize correctly - try repairing Windows perf counter config on this machine with 'lodctr /r' command");
return;
}
cpuUsageTimer = new SafeTimer(CheckCpuUsage, null, CPU_CHECK_PERIOD, CPU_CHECK_PERIOD);
try
{
// Read initial value of CPU Usage counter
CpuUsage = cpuCounter.NextValue();
}
catch (InvalidOperationException)
{
// Can sometimes get exception accessing CPU Usage counter for first time in some runtime environments
CpuUsage = 0;
}
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_CPUUSAGE, () => CpuUsage);
IntValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_TOTALMEMORYKB, () => (MemoryUsage + 1023) / 1024); // Round up
#if LOG_MEMORY_PERF_COUNTERS // print GC stats in the silo log file.
StringValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_GENCOLLECTIONCOUNT, () => GCGenCollectionCount);
StringValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_GENSIZESKB, () => GCGenSizes);
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_PERCENTOFTIMEINGC, () => timeInGC.NextValue());
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_ALLOCATEDBYTESINKBPERSEC, () => allocatedBytesPerSec.NextValue() / 1024f);
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_PROMOTEDMEMORYFROMGEN1KB, () => promotedMemoryFromGen1.NextValue() / 1024f);
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_LARGEOBJECTHEAPSIZEKB, () => largeObjectHeapSize.NextValue() / 11024f);
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_PROMOTEDMEMORYFROMGEN0KB, () => promotedFinalizationMemoryFromGen0.NextValue() / 1024f);
FloatValueStatistic.FindOrCreate(StatisticNames.RUNTIME_GC_NUMBEROFINDUCEDGCS, () => numberOfInducedGCs.NextValue());
#endif
IntValueStatistic.FindOrCreate(StatisticNames.RUNTIME_DOT_NET_THREADPOOL_INUSE_WORKERTHREADS, () =>
{
int maXworkerThreads;
int maXcompletionPortThreads;
ThreadPool.GetMaxThreads(out maXworkerThreads, out maXcompletionPortThreads);
int workerThreads;
int completionPortThreads;
// GetAvailableThreads Retrieves the difference between the maximum number of thread pool threads
// and the number currently active.
// So max-Available is the actual number in use. If it goes beyond min, it means we are stressing the thread pool.
ThreadPool.GetAvailableThreads(out workerThreads, out completionPortThreads);
return(maXworkerThreads - workerThreads);
});
IntValueStatistic.FindOrCreate(StatisticNames.RUNTIME_DOT_NET_THREADPOOL_INUSE_COMPLETIONPORTTHREADS, () =>
{
int maXworkerThreads;
int maXcompletionPortThreads;
ThreadPool.GetMaxThreads(out maXworkerThreads, out maXcompletionPortThreads);
int workerThreads;
int completionPortThreads;
ThreadPool.GetAvailableThreads(out workerThreads, out completionPortThreads);
return(maXcompletionPortThreads - completionPortThreads);
});
}
public QueueTrackingStatistic(string queueName)
{
if (StatisticsCollector.CollectQueueStats)
{
const CounterStorage storage = CounterStorage.LogAndTable;
averageQueueSizeCounter = AverageValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, queueName), storage);
numEnqueuedRequestsCounter = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, queueName), false, storage);
if (TrackExtraStats)
{
totalExecutionTime = TimeIntervalFactory.CreateTimeInterval(true);
averageArrivalRate = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, queueName),
() =>
{
TimeSpan totalTime = totalExecutionTime.Elapsed;
if (totalTime.Ticks == 0)
{
return(0);
}
long numReqs = numEnqueuedRequestsCounter.GetCurrentValue();
return((float)((((double)numReqs * (double)TimeSpan.TicksPerSecond)) / (double)totalTime.Ticks));
}, storage);
}
averageTimeInQueue = AverageValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_AVERAGE_MILLIS_PER_QUEUE, queueName), storage);
averageTimeInQueue.AddValueConverter(Utils.AverageTicksToMilliSeconds);
if (averageTimeInAllQueues == null)
{
averageTimeInAllQueues = AverageValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_AVERAGE_MILLIS_PER_QUEUE, "AllQueues"), storage);
averageTimeInAllQueues.AddValueConverter(Utils.AverageTicksToMilliSeconds);
}
if (totalTimeInAllQueues == null)
{
totalTimeInAllQueues = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_TOTAL_MILLIS_PER_QUEUE, "AllQueues"), false, storage);
totalTimeInAllQueues.AddValueConverter(Utils.TicksToMilliSeconds);
}
}
}
static private AverageValueStatistic FindOrCreate_Impl(StatisticName name, CounterStorage storage, bool multiThreaded)
{
AverageValueStatistic stat;
#if COLLECT_AVERAGE
if (multiThreaded)
{
stat = new MultiThreadedAverageValueStatistic(name);
}
else
{
stat = new SingleThreadedAverageValueStatistic(name);
}
stat.average = FloatValueStatistic.FindOrCreate(name,
() => stat.GetAverageValue(), storage);
#else
stat = new AverageValueStatistic(name);
#endif
return(stat);
}
/// <summary>
/// Keep track of thread statistics, mainly timing, can be created outside the thread to be tracked.
/// </summary>
/// <param name="threadName">Name used for logging the collected stastistics</param>
/// <param name="storage"></param>
public ThreadTrackingStatistic(string threadName)
{
ExecutingCpuCycleTime = new TimeIntervalThreadCycleCounterBased();
ExecutingWallClockTime = TimeIntervalFactory.CreateTimeInterval(true);
ProcessingCpuCycleTime = new TimeIntervalThreadCycleCounterBased();
ProcessingWallClockTime = TimeIntervalFactory.CreateTimeInterval(true);
NumRequests = 0;
firstStart = true;
Name = threadName;
CounterStorage storage = StatisticsCollector.ReportDetailedThreadTimeTrackingStats ? CounterStorage.LogOnly : CounterStorage.DontStore;
CounterStorage aggrCountersStorage = CounterStorage.LogOnly;
// 4 direct counters
allExecutingCpuCycleTime.Add(
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_EXECUTION_TIME_TOTAL_CPU_CYCLES, threadName),
() => (float)ExecutingCpuCycleTime.Elapsed.TotalMilliseconds, storage));
allExecutingWallClockTime.Add(
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_EXECUTION_TIME_TOTAL_WALL_CLOCK, threadName),
() => (float)ExecutingWallClockTime.Elapsed.TotalMilliseconds, storage));
allProcessingCpuCycleTime.Add(
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_PROCESSING_TIME_TOTAL_CPU_CYCLES, threadName),
() => (float)ProcessingCpuCycleTime.Elapsed.TotalMilliseconds, storage));
allProcessingWallClockTime.Add(
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_PROCESSING_TIME_TOTAL_WALL_CLOCK, threadName),
() => (float)ProcessingWallClockTime.Elapsed.TotalMilliseconds, storage));
// numRequests
allNumProcessedRequests.Add(
FloatValueStatistic.FindOrCreate(new StatisticName(StatisticNames.THREADS_PROCESSED_REQUESTS_PER_THREAD, threadName),
() => (float)NumRequests, storage));
// aggregate stats
if (totalExecutingCpuCycleTime == null)
{
totalExecutingCpuCycleTime = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.THREADS_EXECUTION_TIME_AVERAGE_CPU_CYCLES, "AllThreads"),
() => CalculateTotalAverage(allExecutingCpuCycleTime, totalNumProcessedRequests), aggrCountersStorage);
}
if (totalExecutingWallClockTime == null)
{
totalExecutingWallClockTime = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.THREADS_EXECUTION_TIME_AVERAGE_WALL_CLOCK, "AllThreads"),
() => CalculateTotalAverage(allExecutingWallClockTime, totalNumProcessedRequests), aggrCountersStorage);
}
if (totalProcessingCpuCycleTime == null)
{
totalProcessingCpuCycleTime = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.THREADS_PROCESSING_TIME_AVERAGE_CPU_CYCLES, "AllThreads"),
() => CalculateTotalAverage(allProcessingCpuCycleTime, totalNumProcessedRequests), aggrCountersStorage);
}
if (totalProcessingWallClockTime == null)
{
totalProcessingWallClockTime = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.THREADS_PROCESSING_TIME_AVERAGE_WALL_CLOCK, "AllThreads"),
() => CalculateTotalAverage(allProcessingWallClockTime, totalNumProcessedRequests), aggrCountersStorage);
}
if (totalNumProcessedRequests == null)
{
totalNumProcessedRequests = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.THREADS_PROCESSED_REQUESTS_PER_THREAD, "AllThreads"),
() => (float)allNumProcessedRequests.Select(cs => cs.GetCurrentValue()).Sum(), aggrCountersStorage);
}
if (StatisticsCollector.PerformStageAnalysis)
{
globalStageAnalyzer.AddTracking(this);
}
}