public void MetricAggregationByInstanceIsScopedToLogicalEndpoint()
{
var processingTimeStore = new ProcessingTimeStore();
var endpointRegistry = new EndpointRegistry();
var apiController = CreateConroller(processingTimeStore, endpointRegistry);
var instanceAId = new EndpointInstanceId("EndpointA", "instance");
var instanceBId = new EndpointInstanceId("EndpointB", "instance");
endpointRegistry.Record(instanceAId);
endpointRegistry.Record(instanceBId);
var period = HistoryPeriod.FromMinutes(DiagramApiController.DefaultHistory);
var now = DateTime.UtcNow.Subtract(new TimeSpan(period.IntervalSize.Ticks * period.DelayedIntervals));
var dataA = new RawMessage.Entry {
DateTicks = now.Ticks, Value = 5
};
var dataB = new RawMessage.Entry {
DateTicks = now.Ticks, Value = 10
};
processingTimeStore.Store(new[] { dataA }, instanceAId, EndpointMessageType.Unknown(instanceAId.EndpointName));
processingTimeStore.Store(new[] { dataB }, instanceBId, EndpointMessageType.Unknown(instanceBId.EndpointName));
var result = apiController.GetSingleEndpointMetrics(instanceAId.EndpointName);
var contentResult = result as OkNegotiatedContentResult <MonitoredEndpointDetails>;
var model = contentResult.Content;
Assert.AreEqual(5, model.Instances[0].Metrics["ProcessingTime"].Average);
}
public Task Handle(EndpointMetadataReport message, IMessageHandlerContext context)
{
var instanceId = EndpointInstanceId.From(context.MessageHeaders);
queueLengthProvider.Process(instanceId, message);
return(TaskEx.Completed);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
endpointQueueMappings.AddOrUpdate(
endpointInstanceId,
id => metadataReport.LocalAddress,
(id, old) => metadataReport.LocalAddress
);
}
public Task Handle(MetricReport message, IMessageHandlerContext context)
{
var endpointInstanceId = EndpointInstanceId.From(context.MessageHeaders);
if (legacyEndpoints.TryAdd(endpointInstanceId.InstanceId))
{
Logger.Warn($"Legacy queue length report received from {endpointInstanceId.InstanceName} instance of {endpointInstanceId.EndpointName}");
}
return(TaskEx.Completed);
}
public string ClassifyFailure(ClassifiableMessageDetails failureDetails)
{
if (failureDetails.ProcessingAttempt == null)
{
return(null);
}
var instanceId = EndpointInstanceId.From(failureDetails.ProcessingAttempt.Headers);
return(instanceId?.EndpointName);
}
public void Filters_endpoint_instances_by_endpoint_name()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointB", "instance2");
registry.Record(instanceA);
registry.Record(instanceB);
var instances = registry.GetForEndpointName("EndpointA");
CollectionAssert.AreEquivalent(new [] { instanceA }, instances);
}
public void Returns_two_different_stored_logical_endpoints()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointB", "instance2");
registry.Record(instanceA);
registry.Record(instanceB);
var endpoints = registry.GetGroupedByEndpointName();
CollectionAssert.AreEquivalent(new[] { "EndpointA", "EndpointB" }, endpoints.Keys);
}
public void Returns_distinct_endpoint_names()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointA", "instance1");
registry.Record(instanceA);
registry.Record(instanceB);
var endpoints = registry.GetGroupedByEndpointName();
CollectionAssert.AreEquivalent(new[] { "EndpointA" }, endpoints.Keys);
}
public void Returns_distinct_endpoint_instances()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointA", "instance1");
registry.Record(instanceA);
registry.Record(instanceB);
var instances = registry.GetForEndpointName("EndpointA");
CollectionAssert.AreEquivalent(new[] { instanceA }, instances);
}
public Task Handle(TaggedLongValueOccurrence message, IMessageHandlerContext context)
{
var instanceId = EndpointInstanceId.From(context.MessageHeaders);
var messageType = context.MessageHeaders[MetricHeaders.MetricType];
if (messageType == QueueLengthMessageType)
{
queueLengthProvider.Process(instanceId, message);
}
return(TaskEx.Completed);
}
public void Returns_all_physical_instances_for_logical_endpoint_name()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointB", "instance2");
var instanceC = new EndpointInstanceId("EndpointA", "instance3");
registry.Record(instanceA);
registry.Record(instanceB);
registry.Record(instanceC);
var instances = registry.GetForEndpointName("EndpointA");
CollectionAssert.AreEquivalent(new [] { instanceA, instanceC }, instances);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
var endpointInstanceQueue = new EndpointInputQueue(endpointInstanceId.EndpointName, metadataReport.LocalAddress);
var queueName = metadataReport.LocalAddress;
endpointQueues.AddOrUpdate(endpointInstanceQueue, _ => queueName, (_, currentValue) =>
{
if (currentValue != queueName)
{
sizes.TryRemove(currentValue, out var _);
}
return(queueName);
});
}
public void When_known_endpoint_instance_changes_name_existing_entry_is_used_and_udpated()
{
var registry = new EndpointRegistry();
var originalId = new EndpointInstanceId("LogicalName", "instance-id", "original-name");
var renamedId = new EndpointInstanceId(originalId.EndpointName, originalId.InstanceId, "renamed-name");
registry.Record(originalId);
registry.Record(renamedId);
var records = registry.GetForEndpointName(originalId.EndpointName).ToArray();
Assert.AreEqual(1, records.Length, "Existing entry should be reused");
Assert.AreEqual("renamed-name", records[0].InstanceName);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
var endpointInputQueue = new EndpointInputQueue(endpointInstanceId.EndpointName, metadataReport.LocalAddress);
var queueName = QueueNameSanitizer.Sanitize(metadataReport.LocalAddress);
var queueClient = CloudStorageAccount.Parse(connectionString).CreateCloudQueueClient();
var emptyQueueLength = new QueueLengthValue
{
QueueName = queueName,
Length = 0,
QueueReference = queueClient.GetQueueReference(queueName)
};
queueLengths.AddOrUpdate(endpointInputQueue, _ => emptyQueueLength, (_, existingQueueLength) => existingQueueLength);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
var endpointInputQueue = new EndpointInputQueue(endpointInstanceId.EndpointName, metadataReport.LocalAddress);
var queue = QueueNameHelper.GetSqsQueueName(metadataReport.LocalAddress, queueNamePrefix);
queues.AddOrUpdate(endpointInputQueue, _ => queue, (_, currentQueue) =>
{
if (currentQueue != queue)
{
sizes.TryRemove(currentQueue, out var _);
}
return(queue);
});
sizes.TryAdd(queue, 0);
}
public void Returns_only_not_removed_instances()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB1 = new EndpointInstanceId("EndpointB", "instance1");
var instanceB2 = new EndpointInstanceId("EndpointB", "instance2");
var instanceC = new EndpointInstanceId("EndpointC", "instance2");
registry.Record(instanceA);
registry.Record(instanceB1);
registry.Record(instanceB2);
registry.Record(instanceC);
registry.RemoveEndpointInstance("EndpointB", "instance2");
var instances = registry.GetForEndpointName("EndpointB");
CollectionAssert.AreEquivalent(new[] { instanceB1 }, instances);
}
public void Removing_all_instances_removes_the_logical_endpoint()
{
var instanceA = new EndpointInstanceId("EndpointA", "instance1");
var instanceB = new EndpointInstanceId("EndpointA", "instance2");
var instanceC = new EndpointInstanceId("EndpointA", "instance3");
registry.Record(instanceA);
registry.Record(instanceB);
registry.Record(instanceC);
registry.RemoveEndpointInstance("EndpointA", "instance1");
registry.RemoveEndpointInstance("EndpointA", "instance2");
registry.RemoveEndpointInstance("EndpointA", "instance3");
var instances = registry.GetForEndpointName("EndpointA");
CollectionAssert.IsEmpty(instances);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
var endpointInputQueue = new EndpointInputQueue(endpointInstanceId.EndpointName, metadataReport.LocalAddress);
var localAddress = metadataReport.LocalAddress;
var sqlTable = SqlTable.Parse(localAddress, defaultSchema);
tableNames.AddOrUpdate(endpointInputQueue, _ => sqlTable, (_, currentSqlTable) =>
{
if (currentSqlTable.Equals(sqlTable) == false)
{
tableSizes.TryRemove(currentSqlTable, out var _);
}
return(sqlTable);
});
tableSizes.TryAdd(sqlTable, 0);
}
public Task Handle(TaggedLongValueOccurrence message, IMessageHandlerContext context)
{
var instanceId = EndpointInstanceId.From(context.MessageHeaders);
var messageType = new EndpointMessageType(instanceId.EndpointName, message.TagValue);
var metricType = context.MessageHeaders[MetricHeaders.MetricType];
switch (metricType)
{
case ProcessingTimeMessageType:
processingTimeStore.Store(message.Entries, instanceId, messageType);
break;
case CriticalTimeMessageType:
criticalTimeStore.Store(message.Entries, instanceId, messageType);
break;
}
return(TaskEx.Completed);
}
public async Task When_endpoint_removed_should_stay_removed()
{
var monitor = new EndpointInstanceMonitoring(new FakeDomainEvents());
var monitoredEndpoint = new EndpointInstanceId("MonitoredEndpoint", "HostName", Guid.NewGuid());
var lastHeartbeat = DateTime.UtcNow;
monitor.RecordHeartbeat(monitoredEndpoint, lastHeartbeat);
await monitor.CheckEndpoints(lastHeartbeat);
Assert.IsTrue(monitor.HasEndpoint(monitoredEndpoint.UniqueId), "Monitored Endpoint should be recorded");
monitor.RemoveEndpoint(monitoredEndpoint.UniqueId);
Assert.IsFalse(monitor.HasEndpoint(monitoredEndpoint.UniqueId), "Monitored Endpoint should be removed");
await monitor.CheckEndpoints(lastHeartbeat);
Assert.IsFalse(monitor.HasEndpoint(monitoredEndpoint.UniqueId), "Monitored Endpoint should not be added back");
}
public async Task MetricAggregationByInstanceIsScopedToLogicalEndpoint()
{
var processingTimeStore = new ProcessingTimeStore();
var endpointRegistry = new EndpointRegistry();
var queryAction = CreateQuery(processingTimeStore, endpointRegistry);
var instanceAId = new EndpointInstanceId("EndpointA", "instance");
var instanceBId = new EndpointInstanceId("EndpointB", "instance");
endpointRegistry.Record(instanceAId);
endpointRegistry.Record(instanceBId);
var period = HistoryPeriod.FromMinutes(MonitoredEndpointsModule.DefaultHistory);
var now = DateTime.UtcNow.Subtract(new TimeSpan(period.IntervalSize.Ticks * period.DelayedIntervals));
var dataA = new RawMessage.Entry {
DateTicks = now.Ticks, Value = 5
};
var dataB = new RawMessage.Entry {
DateTicks = now.Ticks, Value = 10
};
processingTimeStore.Store(new[] { dataA }, instanceAId, EndpointMessageType.Unknown(instanceAId.EndpointName));
processingTimeStore.Store(new[] { dataB }, instanceBId, EndpointMessageType.Unknown(instanceBId.EndpointName));
var result = await queryAction(new
{
instanceAId.EndpointName
}.ToDynamic(), new CancellationToken());
var model = (MonitoredEndpointDetails)result.NegotiationContext.DefaultModel;
Assert.AreEqual(5, model.Instances[0].Metrics["ProcessingTime"].Average);
}
public IHttpActionResult GetSingleEndpointMetrics(string endpointName)
{
var metricByInstanceLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointInstanceId> >().ToDictionary(i => i.GetType());
var metricByQueueLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointInputQueue> >().ToDictionary(i => i.GetType());
var metricByMessageTypeLookup = breakdownProviders.OfType <IProvideBreakdownBy <EndpointMessageType> >().ToDictionary(i => i.GetType());
var period = ExtractHistoryPeriod();
var instances = GetMonitoredEndpointInstances(endpointRegistry, endpointName, activityTracker);
var digest = new MonitoredEndpointDigest();
var metricDetails = new MonitoredEndpointMetricDetails();
foreach (var metric in InstanceMetrics)
{
var store = metricByInstanceLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow);
var intervalsByEndpoint = intervals.ToLookup(k => k.Id.EndpointName);
var endpointValues = metric.Aggregate(intervalsByEndpoint[endpointName].ToList(), period);
if (DetailedMetrics.Contains(metric.ReturnName))
{
var details = new MonitoredValuesWithTimings
{
Points = endpointValues.Points,
Average = endpointValues.Average,
TimeAxisValues = GetTimeAxisValues(intervalsByEndpoint[endpointName])
};
metricDetails.Metrics.Add(metric.ReturnName, details);
}
var metricDigest = new MonitoredEndpointMetricDigest
{
Latest = endpointValues.Points.LastOrDefault(),
Average = endpointValues.Average
};
digest.Metrics.Add(metric.ReturnName, metricDigest);
var intervalsByInstanceId = intervals.ToLookup(k => k.Id);
foreach (var instance in instances)
{
var instanceId = new EndpointInstanceId(endpointName, instance.Id, instance.Name);
var instanceValues = metric.Aggregate(intervalsByInstanceId[instanceId].ToList(), period);
instance.Metrics.Add(metric.ReturnName, instanceValues);
}
}
foreach (var metric in QueueMetrics)
{
var store = metricByQueueLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow);
var intervalsByEndpoint = intervals.ToLookup(k => k.Id.EndpointName);
var endpointValues = metric.Aggregate(intervalsByEndpoint[endpointName].ToList(), period);
if (DetailedMetrics.Contains(metric.ReturnName))
{
var details = new MonitoredValuesWithTimings
{
Points = endpointValues.Points,
Average = endpointValues.Average,
TimeAxisValues = GetTimeAxisValues(intervalsByEndpoint[endpointName])
};
metricDetails.Metrics.Add(metric.ReturnName, details);
}
var metricDigest = new MonitoredEndpointMetricDigest
{
Latest = endpointValues.Points.LastOrDefault(),
Average = endpointValues.Average
};
digest.Metrics.Add(metric.ReturnName, metricDigest);
}
var messageTypes = GetMonitoredMessageTypes(messageTypeRegistry.GetForEndpointName(endpointName));
foreach (var metric in MessageTypeMetrics)
{
var store = metricByMessageTypeLookup[metric.StoreType];
var intervals = store.GetIntervals(period, DateTime.UtcNow).ToLookup(k => k.Id);
foreach (var messageType in messageTypes)
{
var values = metric.Aggregate(intervals[new EndpointMessageType(endpointName, messageType.Id)].ToList(), period);
messageType.Metrics.Add(metric.ReturnName, values);
}
}
var data = new MonitoredEndpointDetails
{
Digest = digest,
Instances = instances,
MessageTypes = messageTypes,
MetricDetails = metricDetails
};
return(Ok(data));
}
public void Process(EndpointInstanceId endpointInstanceId, TaggedLongValueOccurrence metricsReport)
{
var endpointInputQueue = new EndpointInputQueue(endpointInstanceId.EndpointName, metricsReport.TagValue);
queueLengthStore.Store(metricsReport.Entries, endpointInputQueue);
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
// HINT: Not every queue length provider requires metadata reports
}
static Task <LongHistogram> StartReporter(int sendReportEvery, int numberOfEntriesInReport, CancellationTokenSource source, EndpointInstanceId instance, Action <RawMessage.Entry[], EndpointInstanceId> store)
{
return(Task.Run(async() =>
{
var entries = new RawMessage.Entry[numberOfEntriesInReport];
var histogram = CreateTimeHistogram();
while (source.IsCancellationRequested == false)
{
var now = DateTime.UtcNow;
for (var i = 0; i < entries.Length; i++)
{
entries[i].DateTicks = now.AddMilliseconds(100 * i).Ticks;
entries[i].Value = i;
}
var start = Stopwatch.GetTimestamp();
store(entries, instance);
var elapsed = Stopwatch.GetTimestamp() - start;
histogram.RecordValue(elapsed);
await Task.Delay(sendReportEvery).ConfigureAwait(false);
}
return histogram;
}));
}
public void Process(EndpointInstanceId endpointInstanceId, TaggedLongValueOccurrence metricsReport)
{
// The endpoint should not be sending this data
}
public void Process(EndpointInstanceId endpointInstanceId, EndpointMetadataReport metadataReport)
{
var key = new EndpointInputQueue(endpointInstanceId.EndpointName, metadataReport.LocalAddress);
endpointsHash.AddOrUpdate(key, key, (_, __) => key);
}
public void Process(EndpointInstanceId endpointInstanceId, TaggedLongValueOccurrence metricsReport)
{
}
public void Process(EndpointInstanceId endpointInstanceId, TaggedLongValueOccurrence metricsReport)
{
//HINT: ASQ server endpoints do not support endpoint level queue length monitoring
}
public void Process(EndpointInstanceId endpointInstanceId, TaggedLongValueOccurrence metricsReport)
{
//RabbitMQ does not support endpoint level queue length reports
}
