public QueueTrackingStatistic(string queueName)
{
if (StatisticsCollector.CollectQueueStats)
{
const CounterStorage storage = CounterStorage.LogAndTable;
averageQueueSizeCounter = AverageValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_QUEUE_SIZE_AVERAGE_PER_QUEUE, queueName), storage);
numEnqueuedRequestsCounter = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_ENQUEUED_PER_QUEUE, queueName), false, storage);
if (TrackExtraStats)
{
totalExecutionTime = TimeIntervalFactory.CreateTimeInterval(true);
averageArrivalRate = FloatValueStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_AVERAGE_ARRIVAL_RATE_PER_QUEUE, queueName),
() =>
{
TimeSpan totalTime = totalExecutionTime.Elapsed;
if (totalTime.Ticks == 0)
{
return(0);
}
long numReqs = numEnqueuedRequestsCounter.GetCurrentValue();
return((float)((((double)numReqs * (double)TimeSpan.TicksPerSecond)) / (double)totalTime.Ticks));
}, storage);
}
averageTimeInQueue = AverageValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_AVERAGE_MILLIS_PER_QUEUE, queueName), storage);
averageTimeInQueue.AddValueConverter(Utils.AverageTicksToMilliSeconds);
if (averageTimeInAllQueues == null)
{
averageTimeInAllQueues = AverageValueStatistic.FindOrCreate(new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_AVERAGE_MILLIS_PER_QUEUE, "AllQueues"), storage);
averageTimeInAllQueues.AddValueConverter(Utils.AverageTicksToMilliSeconds);
}
if (totalTimeInAllQueues == null)
{
totalTimeInAllQueues = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.QUEUES_TIME_IN_QUEUE_TOTAL_MILLIS_PER_QUEUE, "AllQueues"), false, storage);
totalTimeInAllQueues.AddValueConverter(Utils.TicksToMilliSeconds);
}
}
}
internal static int RegisterWorkingThread(string threadName)
{
lock (lockable)
{
int i = workerThreadCounter;
workerThreadCounter++;
if (i == turnsExecutedPerWorkerThreadApplicationTurns.Length)
{
// need to resize the array
Array.Resize(ref turnsExecutedPerWorkerThreadApplicationTurns, 2 * turnsExecutedPerWorkerThreadApplicationTurns.Length);
Array.Resize(ref turnsExecutedPerWorkerThreadSystemTurns, 2 * turnsExecutedPerWorkerThreadSystemTurns.Length);
Array.Resize(ref turnsExecutedPerWorkerThreadNull, 2 * turnsExecutedPerWorkerThreadNull.Length);
}
turnsExecutedPerWorkerThreadApplicationTurns[i] = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.SCHEDULER_TURNSEXECUTED_APPLICATION_PERTHREAD, threadName));
turnsExecutedPerWorkerThreadSystemTurns[i] = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.SCHEDULER_TURNSEXECUTED_SYSTEM_PERTHREAD, threadName));
turnsExecutedPerWorkerThreadNull[i] = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.SCHEDULER_TURNSEXECUTED_NULL_PERTHREAD, threadName));
return(i);
}
}
internal async Task DumpCounters()
{
List <ICounter> allCounters = new List <ICounter>();
CounterStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
IntValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
StringValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
FloatValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
AverageTimeSpanStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
foreach (var stat in allCounters.Where(cs => cs.Storage != CounterStorage.DontStore).OrderBy(cs => cs.Name))
{
WriteStatsLogEntry(stat.GetDisplayString());
}
List <ICounter> additionalCounters = GenerateAdditionalCounters();
// NOTE: For now, we don't want to bother logging these counters -- AG 11/20/2012
foreach (var stat in additionalCounters.OrderBy(cs => cs.Name))
{
WriteStatsLogEntry(stat.GetDisplayString());
}
WriteStatsLogEntry(null); // Write any remaining log data
try
{
if (StatsTablePublisher != null && allCounters.Count > 0)
{
await StatsTablePublisher.ReportStats(allCounters);
}
}
catch (Exception exc)
{
var e = exc.GetBaseException();
logger.Error(ErrorCode.AzureTable_35, "Exception occurred during Stats reporter.", e);
}
// Reset current value for counter that have delta.
// Do it ONLY after all counters have been logged.
foreach (ICounter stat in allCounters.Where(cs => cs.Storage != CounterStorage.DontStore).Union(additionalCounters).Where(cs => cs.IsValueDelta))
{
stat.ResetCurrent();
}
}
public SchedulerStatisticsGroup(IOptions <StatisticsOptions> statisticsOptions, ILogger <SchedulerStatisticsGroup> logger)
{
this.logger = logger;
this.collectionLevel = statisticsOptions.Value.CollectionLevel;
this.CollectTurnsStats = collectionLevel.CollectTurnsStats();
this.CollectPerWorkItemStats = collectionLevel.CollectPerWorkItemStats();
this.CollectShedulerQueuesStats = collectionLevel.CollectShedulerQueuesStats();
workItemGroupStatuses = new StringValueStatistic[1];
workItemGroupCounter = 0;
lockable = new object();
if (this.CollectTurnsStats)
{
turnsExecutedPerWorkItemGroup ??= new CounterStatistic[1];
}
NumLongRunningTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_NUM_LONG_RUNNING_TURNS);
NumLongQueueWaitTimes = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_NUM_LONG_QUEUE_WAIT_TIMES);
}
internal static void Init()
{
workItemGroupStatuses = new StringValueStatistic[1];
workerThreadCounter = 0;
workItemGroupCounter = 0;
lockable = new object();
if (StatisticsCollector.CollectGlobalShedulerStats)
{
totalPendingWorkItems = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_PENDINGWORKITEMS, false);
turnsEnQueuedTotal = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_ITEMS_ENQUEUED_TOTAL);
turnsDeQueuedTotal = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_ITEMS_DEQUEUED_TOTAL);
turnsDroppedTotal = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_ITEMS_DROPPED_TOTAL);
closureWorkItemsCreated = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_CLOSURE_WORK_ITEMS_CREATED);
closureWorkItemsExecuted = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_CLOSURE_WORK_ITEMS_EXECUTED);
}
if (StatisticsCollector.CollectTurnsStats)
{
turnsExecutedByAllWorkerThreadsTotalApplicationTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_APPLICATION_BYALLWORKERTHREADS);
turnsExecutedByAllWorkerThreadsTotalSystemTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_SYSTEM_BYALLWORKERTHREADS);
turnsExecutedByAllWorkerThreadsTotalNullTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_NULL_BYALLWORKERTHREADS);
turnsExecutedByAllWorkItemGroupsTotalApplicationTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_APPLICATION_BYALLWORKITEMGROUPS);
turnsExecutedByAllWorkItemGroupsTotalSystem = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_SYSTEM_BYALLWORKITEMGROUPS);
turnLengthHistogram = ExponentialHistogramValueStatistic.Create_ExponentialHistogram_ForTiming(StatisticNames.SCHEDULER_TURN_LENGTH_HISTOGRAM, TURN_LENGTH_HISTOGRAM_SIZE);
turnsExecutedStartTotal = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_TOTAL_START);
turnsExecutedEndTotal = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_TURNSEXECUTED_TOTAL_END);
turnsExecutedPerWorkerThreadApplicationTurns = new CounterStatistic[1];
turnsExecutedPerWorkerThreadSystemTurns = new CounterStatistic[1];
turnsExecutedPerWorkerThreadNull = new CounterStatistic[1];
turnsExecutedPerWorkItemGroup = new CounterStatistic[1];
}
NumLongRunningTurns = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_NUM_LONG_RUNNING_TURNS);
NumLongQueueWaitTimes = CounterStatistic.FindOrCreate(StatisticNames.SCHEDULER_NUM_LONG_QUEUE_WAIT_TIMES);
logger = TraceLogger.GetLogger("SchedulerStatisticsGroup", TraceLogger.LoggerType.Runtime);
}
internal void DumpCounters()
{
List <ICounter> allCounters = new List <ICounter>();
CounterStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
IntValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
StringValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
FloatValueStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
AverageTimeSpanStatistic.AddCounters(allCounters, cs => cs.Storage != CounterStorage.DontStore);
foreach (var stat in allCounters.Where(cs => cs.Storage != CounterStorage.DontStore).OrderBy(cs => cs.Name))
{
WriteStatsLogEntry(stat.GetDisplayString());
}
WriteStatsLogEntry(null); // Write any remaining log data
// Reset current value for counter that have delta.
// Do it ONLY after all counters have been logged.
foreach (ICounter stat in allCounters.Where(cs => cs.Storage != CounterStorage.DontStore).Where(cs => cs.IsValueDelta))
{
stat.ResetCurrent();
}
}
/// <summary>
/// Creates a buffer pool.
/// </summary>
/// <param name="bufferSize">The size, in bytes, of each buffer.</param>
/// <param name="maxBuffers">The maximum number of buffers to keep around, unused; by default, the number of unused buffers is unbounded.</param>
private BufferPool(int bufferSize, int maxBuffers, int preallocationSize, string name)
{
Name = name;
byteBufferSize = bufferSize;
maxBuffersCount = maxBuffers;
limitBuffersCount = maxBuffers > 0;
buffers = new ConcurrentBag <byte[]>();
var globalPoolSizeStat = IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_BUFFERS_INPOOL,
() => Count);
allocatedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_ALLOCATED_BUFFERS);
checkedOutBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_OUT_BUFFERS);
checkedInBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_IN_BUFFERS);
droppedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_BUFFERS);
droppedTooLargeBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_TOO_LARGE_BUFFERS);
// Those 2 counters should be equal. If not, it means we don't release all buffers.
IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_CHECKED_OUT_NOT_CHECKED_IN_BUFFERS,
() => checkedOutBufferCounter.GetCurrentValue()
- checkedInBufferCounter.GetCurrentValue()
- droppedBufferCounter.GetCurrentValue());
IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_ALLOCATED_NOT_INPOOL_BUFFERS,
() => allocatedBufferCounter.GetCurrentValue()
- globalPoolSizeStat.GetCurrentValue()
- droppedBufferCounter.GetCurrentValue());
if (preallocationSize <= 0)
{
return;
}
var dummy = GetMultiBuffer(preallocationSize * Size);
Release(dummy);
}
/// <summary>
/// Resolve target address for a message
/// - use transaction info
/// - check ordering info in message and sending activation
/// - use sender's placement strategy
/// </summary>
/// <param name="message"></param>
/// <returns>Resolve when message is addressed (modifies message fields)</returns>
private async Task AddressMessage(Message message)
{
var targetAddress = message.TargetAddress;
if (targetAddress.IsComplete)
{
return;
}
// placement strategy is determined by searching for a specification. first, we check for a strategy associated with the grain reference,
// second, we check for a strategy associated with the target's interface. third, we check for a strategy associated with the activation sending the
// message.
var strategy = targetAddress.Grain.IsGrain ? catalog.GetGrainPlacementStrategy(targetAddress.Grain) : null;
var request = message.IsUsingInterfaceVersions
? message.GetDeserializedBody(this.serializationManager) as InvokeMethodRequest
: null;
var target = new PlacementTarget(message.TargetGrain, request?.InterfaceId ?? 0, request?.InterfaceVersion ?? 0);
var placementResult = await this.placementDirectorsManager.SelectOrAddActivation(
message.SendingAddress, target, this.catalog, strategy);
if (placementResult.IsNewPlacement && targetAddress.Grain.IsClient)
{
logger.Error(ErrorCode.Dispatcher_AddressMsg_UnregisteredClient, String.Format("AddressMessage could not find target for client pseudo-grain {0}", message));
throw new KeyNotFoundException(String.Format("Attempting to send a message {0} to an unregistered client pseudo-grain {1}", message, targetAddress.Grain));
}
message.SetTargetPlacement(placementResult);
if (placementResult.IsNewPlacement)
{
CounterStatistic.FindOrCreate(StatisticNames.DISPATCHER_NEW_PLACEMENT).Increment();
}
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.Dispatcher_AddressMsg_SelectTarget, "AddressMessage Placement SelectTarget {0}", message);
}
}
private static void AgentThreadProc(Object obj)
{
var agent = obj as AsynchAgent;
if (agent == null)
{
var log = LogManager.GetLogger("RuntimeCore.AsynchAgent");
log.Error(ErrorCode.Runtime_Error_100022, "Agent thread started with incorrect parameter type");
return;
}
try
{
LogStatus(agent.Log, "Starting AsyncAgent {0} on managed thread {1}", agent.Name, Thread.CurrentThread.ManagedThreadId);
CounterStatistic.SetOrleansManagedThread(); // do it before using CounterStatistic.
CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_PERAGENTTYPE, agent.type)).Increment();
CounterStatistic.FindOrCreate(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_TOTAL_THREADS_CREATED).Increment();
agent.Run();
}
catch (Exception exc)
{
if (agent.State == ThreadState.Running) // If we're stopping, ignore exceptions
{
var log = agent.Log;
switch (agent.OnFault)
{
case FaultBehavior.CrashOnFault:
Console.WriteLine(
"The {0} agent has thrown an unhandled exception, {1}. The process will be terminated.",
agent.Name, exc);
log.Error(ErrorCode.Runtime_Error_100023,
"AsynchAgent Run method has thrown an unhandled exception. The process will be terminated.",
exc);
log.Fail(ErrorCode.Runtime_Error_100024, "Terminating process because of an unhandled exception caught in AsynchAgent.Run.");
break;
case FaultBehavior.IgnoreFault:
log.Error(ErrorCode.Runtime_Error_100025, "AsynchAgent Run method has thrown an unhandled exception. The agent will exit.",
exc);
agent.State = ThreadState.Stopped;
break;
case FaultBehavior.RestartOnFault:
log.Error(ErrorCode.Runtime_Error_100026,
"AsynchAgent Run method has thrown an unhandled exception. The agent will be restarted.",
exc);
agent.State = ThreadState.Stopped;
try
{
agent.Start();
}
catch (Exception ex)
{
log.Error(ErrorCode.Runtime_Error_100027, "Unable to restart AsynchAgent", ex);
agent.State = ThreadState.Stopped;
}
break;
}
}
}
finally
{
CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_PERAGENTTYPE, agent.type)).DecrementBy(1);
agent.Log.Info(ErrorCode.Runtime_Error_100328, "Stopping AsyncAgent {0} that runs on managed thread {1}", agent.Name, Thread.CurrentThread.ManagedThreadId);
}
}
internal static void Init(bool silo)
{
if (silo)
{
LocalMessagesSent = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_SENT_LOCALMESSAGES);
ConnectedClientCount = CounterStatistic.FindOrCreate(StatisticNames.GATEWAY_CONNECTED_CLIENTS, false);
}
MessagesSentTotal = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_SENT_MESSAGES_TOTAL);
MessagesSentPerDirection = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
foreach (var direction in Enum.GetValues(typeof(Message.Directions)))
{
MessagesSentPerDirection[(int)direction] = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.MESSAGING_SENT_MESSAGES_PER_DIRECTION, Enum.GetName(typeof(Message.Directions), direction)));
}
MessagesReceived = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_RECEIVED_MESSAGES_TOTAL);
MessagesReceivedPerDirection = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
foreach (var direction in Enum.GetValues(typeof(Message.Directions)))
{
MessagesReceivedPerDirection[(int)direction] = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.MESSAGING_RECEIVED_MESSAGES_PER_DIRECTION, Enum.GetName(typeof(Message.Directions), direction)));
}
TotalBytesSent = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_SENT_BYTES_TOTAL);
totalBytesReceived = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_RECEIVED_BYTES_TOTAL);
HeaderBytesSent = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_SENT_BYTES_HEADER);
headerBytesReceived = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_RECEIVED_BYTES_HEADER);
FailedSentMessages = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
DroppedSentMessages = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
RejectedMessages = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
ReroutedMessages = new CounterStatistic[Enum.GetValues(typeof(Message.Directions)).Length];
foreach (var direction in Enum.GetValues(typeof(Message.Directions)))
{
ReroutedMessages[(int)direction] = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.MESSAGING_REROUTED_PER_DIRECTION, Enum.GetName(typeof(Message.Directions), direction)));
}
sentMsgSizeHistogram = ExponentialHistogramValueStatistic.Create_ExponentialHistogram(StatisticNames.MESSAGING_SENT_MESSAGESIZEHISTOGRAM, NUM_MSG_SIZE_HISTOGRAM_CATEGORIES);
receiveMsgSizeHistogram = ExponentialHistogramValueStatistic.Create_ExponentialHistogram(StatisticNames.MESSAGING_RECEIVED_MESSAGESIZEHISTOGRAM, NUM_MSG_SIZE_HISTOGRAM_CATEGORIES);
expiredAtSendCounter = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_EXPIRED_ATSENDER);
expiredAtReceiveCounter = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_EXPIRED_ATRECEIVER);
expiredAtDispatchCounter = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_EXPIRED_ATDISPATCH);
expiredAtInvokeCounter = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_EXPIRED_ATINVOKE);
expiredAtRespondCounter = CounterStatistic.FindOrCreate(StatisticNames.MESSAGING_EXPIRED_ATRESPOND);
perSocketDirectionStatsSend = new PerSocketDirectionStats[Enum.GetValues(typeof(SocketDirection)).Length];
perSocketDirectionStatsReceive = new PerSocketDirectionStats[Enum.GetValues(typeof(SocketDirection)).Length];
if (silo)
{
perSocketDirectionStatsSend[(int)SocketDirection.SiloToSilo] = new PerSocketDirectionStats(true, SocketDirection.SiloToSilo);
perSocketDirectionStatsSend[(int)SocketDirection.GatewayToClient] = new PerSocketDirectionStats(true, SocketDirection.GatewayToClient);
perSocketDirectionStatsReceive[(int)SocketDirection.SiloToSilo] = new PerSocketDirectionStats(false, SocketDirection.SiloToSilo);
perSocketDirectionStatsReceive[(int)SocketDirection.GatewayToClient] = new PerSocketDirectionStats(false, SocketDirection.GatewayToClient);
}
else
{
perSocketDirectionStatsSend[(int)SocketDirection.ClientToGateway] = new PerSocketDirectionStats(true, SocketDirection.ClientToGateway);
perSocketDirectionStatsReceive[(int)SocketDirection.ClientToGateway] = new PerSocketDirectionStats(false, SocketDirection.ClientToGateway);
}
perSiloSendCounters = new ConcurrentDictionary <string, CounterStatistic>();
perSiloReceiveCounters = new ConcurrentDictionary <string, CounterStatistic>();
perSiloPingSendCounters = new ConcurrentDictionary <string, CounterStatistic>();
perSiloPingReceiveCounters = new ConcurrentDictionary <string, CounterStatistic>();
perSiloPingReplyReceivedCounters = new ConcurrentDictionary <string, CounterStatistic>();
perSiloPingReplyMissedCounters = new ConcurrentDictionary <string, CounterStatistic>();
}
internal MultiThreadedAverageValueStatistic(StatisticName name)
: base(name)
{
totalSum = CounterStatistic.FindOrCreate(new StatisticName(String.Format("{0}.{1}", name.Name, "TotalSum.Hidden")), false, CounterStorage.DontStore);
numItems = CounterStatistic.FindOrCreate(new StatisticName(String.Format("{0}.{1}", name.Name, "NumItems.Hidden")), false, CounterStorage.DontStore);
}
