public async Task GetMonitoredSingleEndpointQueryTest(int numberOfInstances, int numberOfMessageTypes, int sendReportEvery, int numberOfEntriesInReport, int queryEveryInMilliseconds, int numberOfQueries)
{
var instances = BuildInstances(1, numberOfInstances);
foreach (var instance in instances)
{
endpointRegistry.Record(instance);
}
var endpointName = instances.First().EndpointName;
for (var i = 0; i < numberOfMessageTypes; i++)
{
var messageType = new Guid(i, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).ToString();
messageTypeRegistry.Record(new EndpointMessageType(endpointName, messageType));
}
var source = new CancellationTokenSource();
var messageTypes = messageTypeRegistry.GetForEndpointName(endpointName).ToArray();
long counter = 0;
Func <EndpointMessageType> getter = () =>
{
var value = Interlocked.Increment(ref counter) % messageTypes.Length;
return(messageTypes[value]);
};
var reporters =
new[]
{
BuildReporters(sendReportEvery, numberOfEntriesInReport, instances, source, (e, i) => criticalTimeStore.Store(e, i, getter())),
BuildReporters(sendReportEvery, numberOfEntriesInReport, instances, source, (e, i) => processingTimeStore.Store(e, i, getter())),
BuildReporters(sendReportEvery, numberOfEntriesInReport, instances, source, (e, i) => retriesStore.Store(e, i, getter())),
BuildReporters(sendReportEvery, numberOfEntriesInReport, instances, source, (e, i) => queueLengthProvider.Process(i, new TaggedLongValueOccurrence {
Entries = e
}))
}.SelectMany(i => i).ToArray();
var histogram = CreateTimeHistogram();
for (var i = 0; i < numberOfQueries; i++)
{
var start = Stopwatch.GetTimestamp();
await GetMonitoredSingleEndpoint(endpointName).ConfigureAwait(false);
var elapsed = Stopwatch.GetTimestamp() - start;
histogram.RecordValue(elapsed);
await Task.Delay(queryEveryInMilliseconds).ConfigureAwait(false);
}
source.Cancel();
await Task.WhenAll(reporters).ConfigureAwait(false);
var reportFinalHistogram = MergeHistograms(reporters);
Report("Querying", histogram, TimeSpan.FromMilliseconds(25));
Report("Reporters", reportFinalHistogram, TimeSpan.FromMilliseconds(20));
}
/// <summary>
/// Initializes the metric API module.
/// </summary>
public MonitoredEndpointsModule(IProvideBreakdown[] breakdownProviders, EndpointRegistry endpointRegistry, EndpointInstanceActivityTracker activityTracker, MessageTypeRegistry messageTypeRegistry)
{
var metricByInstanceLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointInstanceId> >().ToDictionary(i => i.GetType());
var metricByQueueLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointInputQueue> >().ToDictionary(i => i.GetType());
var metricByMessageTypeLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointMessageType> >().ToDictionary(i => i.GetType());
var instanceMetrics = new[]
{
CreateMetric <EndpointInstanceId, ProcessingTimeStore>("ProcessingTime", Aggregator.ToAverages),
CreateMetric <EndpointInstanceId, CriticalTimeStore>("CriticalTime", Aggregator.ToAverages),
CreateMetric <EndpointInstanceId, RetriesStore>("Retries", Aggregator.ToTotalMeasurementsPerSecond),
CreateMetric <EndpointInstanceId, ProcessingTimeStore>("Throughput", Aggregator.ToTotalMeasurementsPerSecond)
};
var queueMetrics = new[]
{
CreateMetric <EndpointInputQueue, QueueLengthStore>("QueueLength", Aggregator.ToRoundedSumOfBreakdownAverages)
};
var messageTypeMetrics = new[]
{
CreateMetric <EndpointMessageType, ProcessingTimeStore>("ProcessingTime", Aggregator.ToAverages),
CreateMetric <EndpointMessageType, CriticalTimeStore>("CriticalTime", Aggregator.ToAverages),
CreateMetric <EndpointMessageType, RetriesStore>("Retries", Aggregator.ToTotalMeasurementsPerSecond),
CreateMetric <EndpointMessageType, ProcessingTimeStore>("Throughput", Aggregator.ToTotalMeasurementsPerSecond)
};
var detailedMetrics = new HashSet <string>
{
"Throughput",
"QueueLength",
"ProcessingTime",
"CriticalTime",
"Retries"
};
Get["/monitored-endpoints"] = parameters =>
{
var endpoints = GetMonitoredEndpoints(endpointRegistry, activityTracker);
var period = ExtractHistoryPeriod();
foreach (var metric in instanceMetrics)
{
var store = metricByInstanceLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow).ToLookup(k => k.Id.EndpointName);
foreach (var endpoint in endpoints)
{
var values = metric.Aggregate(intervals[endpoint.Name].ToList(), period);
endpoint.Metrics.Add(metric.ReturnName, values);
}
}
foreach (var metric in queueMetrics)
{
var store = metricByQueueLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow).ToLookup(k => k.Id.EndpointName);
foreach (var endpoint in endpoints)
{
var values = metric.Aggregate(intervals[endpoint.Name].ToList(), period);
endpoint.Metrics.Add(metric.ReturnName, values);
}
}
return(Negotiate.WithModel(endpoints));
};
Get["/monitored-endpoints/{endpointName}"] = parameters =>
{
var endpointName = (string)parameters.EndpointName;
var period = ExtractHistoryPeriod();
var instances = GetMonitoredEndpointInstances(endpointRegistry, endpointName, activityTracker);
var digest = new MonitoredEndpointDigest();
var metricDetails = new MonitoredEndpointMetricDetails();
foreach (var metric in instanceMetrics)
{
var store = metricByInstanceLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow);
var intervalsByEndpoint = intervals.ToLookup(k => k.Id.EndpointName);
var endpointValues = metric.Aggregate(intervalsByEndpoint[endpointName].ToList(), period);
if (detailedMetrics.Contains(metric.ReturnName))
{
var details = new MonitoredValuesWithTimings
{
Points = endpointValues.Points,
Average = endpointValues.Average,
TimeAxisValues = GetTimeAxisValues(intervalsByEndpoint[endpointName])
};
metricDetails.Metrics.Add(metric.ReturnName, details);
}
var metricDigest = new MonitoredEndpointMetricDigest
{
Latest = endpointValues.Points.LastOrDefault(),
Average = endpointValues.Average
};
digest.Metrics.Add(metric.ReturnName, metricDigest);
var intervalsByInstanceId = intervals.ToLookup(k => k.Id);
foreach (var instance in instances)
{
var instanceId = new EndpointInstanceId(endpointName, instance.Id, instance.Name);
var instanceValues = metric.Aggregate(intervalsByInstanceId[instanceId].ToList(), period);
instance.Metrics.Add(metric.ReturnName, instanceValues);
}
}
foreach (var metric in queueMetrics)
{
var store = metricByQueueLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow);
var intervalsByEndpoint = intervals.ToLookup(k => k.Id.EndpointName);
var endpointValues = metric.Aggregate(intervalsByEndpoint[endpointName].ToList(), period);
if (detailedMetrics.Contains(metric.ReturnName))
{
var details = new MonitoredValuesWithTimings
{
Points = endpointValues.Points,
Average = endpointValues.Average,
TimeAxisValues = GetTimeAxisValues(intervalsByEndpoint[endpointName])
};
metricDetails.Metrics.Add(metric.ReturnName, details);
}
var metricDigest = new MonitoredEndpointMetricDigest
{
Latest = endpointValues.Points.LastOrDefault(),
Average = endpointValues.Average
};
digest.Metrics.Add(metric.ReturnName, metricDigest);
}
var messageTypes = GetMonitoredMessageTypes(messageTypeRegistry.GetForEndpointName(endpointName));
foreach (var metric in messageTypeMetrics)
{
var store = metricByMessageTypeLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow).ToLookup(k => k.Id);
foreach (var messageType in messageTypes)
{
var values = metric.Aggregate(intervals[new EndpointMessageType(endpointName, messageType.Id)].ToList(), period);
messageType.Metrics.Add(metric.ReturnName, values);
}
}
var data = new MonitoredEndpointDetails
{
Digest = digest,
Instances = instances,
MessageTypes = messageTypes,
MetricDetails = metricDetails
};
return(Negotiate.WithModel(data));
};
}
