public async Task AggregateEmptyStateManager()
{
MockReliableStateManager stateManager = new MockReliableStateManager();
MetricAggregator target = new MetricAggregator();
IEnumerable <LoadMetric> actual = await target.Aggregate(stateManager, CancellationToken.None);
Assert.IsFalse(actual.Any());
}
public void TestBasicFunctionality()
{
MetricAggregator aggregator = new MetricAggregator(new AggregationTemplate("test_metric", "key1", new Averager()));
// metrics with 1 tag, key1, that has key values of val[1-10]. The key values don't matter for this test and are ignored by the aggregator. Only the metric value is averaged.
IEnumerable <Metric> metrics = Enumerable.Range(1, 10).Select(i => new Metric(this.now, "test_metric", i, new Dictionary <string, string> {
{ "key1", $"val{i}" }
}));
Metric[] result = aggregator.AggregateMetrics(metrics).ToArray();
Assert.Equal(5.5, result.Single().Value); // should be sum of 1-10
}
public async Task AggregateEmptyMetrics()
{
Uri name = new Uri("test://dictionary");
MockReliableStateManager stateManager = new MockReliableStateManager();
MockReliableDictionary <int, int> dictionary = new MockReliableDictionary <int, int>(name);
dictionary.OnGetLoadMetrics = () =>
new DecimalLoadMetric[0];
stateManager.SetMock(name, dictionary);
MetricAggregator target = new MetricAggregator();
IEnumerable <LoadMetric> actual = await target.Aggregate(stateManager, CancellationToken.None);
Assert.IsFalse(actual.Any());
}
public void TestKeepsNonAggregateTagsSeperate()
{
MetricAggregator aggregator = new MetricAggregator(new AggregationTemplate("test_metric", "key1", new Summer()));
// metrics with 2 tags, key1, that has key values of val[1-10], and key2, which has key values val[0-1]. The values are summed by shared key2, and key1's value is ignored.
IEnumerable <Metric> metrics = Enumerable.Range(1, 10).Select(i => new Metric(this.now, "test_metric", i, new Dictionary <string, string>
{
{ "key1", $"val{i}" },
{ "key2", $"val{i % 2}" }
}));
Metric[] results = aggregator.AggregateMetrics(metrics).ToArray();
Assert.Equal(2, results.Length);
Assert.Equal(2 + 4 + 6 + 8 + 10, results.Where(m => m.Tags.Contains(new KeyValuePair <string, string>("key2", "val0"))).Single().Value);
Assert.Equal(1 + 3 + 5 + 7 + 9, results.Where(m => m.Tags.Contains(new KeyValuePair <string, string>("key2", "val1"))).Single().Value);
}
public void TestMultipleMetricsDifferentAggregator()
{
MetricAggregator aggregator = new MetricAggregator(
new AggregationTemplate("test_metric0", "key1", new Summer()),
new AggregationTemplate("test_metric1", "key1", new Multiplier()));
// 2 metrics with 1 tag, key1, that has key values of val[1-10]. The key values don't matter for this test and are ignored by the aggregator. Only the metric values are used. Even values are test_metric0 and will be summed, odd values are test_metric1 and will be multiplied.
IEnumerable <Metric> metrics = Enumerable.Range(1, 10).Select(i => new Metric(this.now, $"test_metric{i % 2}", i, new Dictionary <string, string> {
{ "key1", $"val{i}" }
}));
Metric[] results = aggregator.AggregateMetrics(metrics).ToArray();
Assert.Equal(2, results.Length);
Assert.Equal(2 + 4 + 6 + 8 + 10, results.Where(m => m.Name == "test_metric0").Single().Value);
Assert.Equal(1 * 3 * 5 * 7 * 9, results.Where(m => m.Name == "test_metric1").Single().Value);
}
public async Task AggregateMultipleCollectionsSameMetrics()
{
string expectedMetric1 = "one";
string expectedMetric2 = "two";
double inputMetric1Value = 1.9;
double inputMetric2Value = 4.1;
int expectedMetric1Value = 2;
int expectedMetric2Value = 4;
Uri collection1Name = new Uri("test://dictionary1");
Uri collection2Name = new Uri("test://dictionary2");
MockReliableStateManager stateManager = new MockReliableStateManager();
MockReliableDictionary <int, int> dictionary1 = new MockReliableDictionary <int, int>(collection1Name);
MockReliableDictionary <int, int> dictionary2 = new MockReliableDictionary <int, int>(collection2Name);
dictionary1.OnGetLoadMetrics = () =>
new DecimalLoadMetric[]
{
new DecimalLoadMetric(expectedMetric1, inputMetric1Value / 2.0),
new DecimalLoadMetric(expectedMetric2, inputMetric2Value / 2.0)
};
dictionary2.OnGetLoadMetrics = () =>
new DecimalLoadMetric[]
{
new DecimalLoadMetric(expectedMetric1, inputMetric1Value / 2.0),
new DecimalLoadMetric(expectedMetric2, inputMetric2Value / 2.0)
};
stateManager.SetMock(collection1Name, dictionary1);
stateManager.SetMock(collection2Name, dictionary2);
MetricAggregator target = new MetricAggregator();
IEnumerable <LoadMetric> actual = await target.Aggregate(stateManager, CancellationToken.None);
Assert.AreEqual <int>(1, actual.Count(x => x.Name == expectedMetric1 && x.Value == expectedMetric1Value));
Assert.AreEqual <int>(1, actual.Count(x => x.Name == expectedMetric2 && x.Value == expectedMetric2Value));
}
public void IncrementByTwo_OneStat_TwoTicks()
{
const string metricName = "foo";
const int count1 = 100;
const int count2 = 25;
IEnumerable <DataPoint> points = null;
var mockPoster = new Mock <MockCustomMetricsPoster>();
mockPoster
.Setup(p => p.SendBatchMetricsAsync(It.IsAny <IEnumerable <DataPoint> >()))
.Callback <IEnumerable <DataPoint> >(c => points = c)
.Returns(new Task(NoOp));
var aggregator = new MetricAggregator(mockPoster.Object, 3);
for (int i = 0; i < count1; i++)
{
aggregator.Increment(metricName, 2);
}
Thread.Sleep(4500);
Assert.NotNull(points);
var dp = points.First();
Assert.NotNull(dp);
Assert.Equal(count1 * 2, (int)(double)dp.Value);
for (int i = 0; i < count2; i++)
{
aggregator.Increment(metricName, 2);
}
Thread.Sleep(4000);
Assert.NotNull(points);
dp = points.First();
Assert.NotNull(dp);
Assert.Equal(count2 * 2, (int)(double)dp.Value);
}
public void IncrementByTwo_TwoStats_TwoTicks()
{
const string metricName1 = "foo";
const string metricName2 = "bar";
const string metricName3 = "zap";
const int count1 = 100;
const long count2 = 2500L;
const long count3 = 10;
IEnumerable <DataPoint> points = null;
var mockPoster = new Mock <MockCustomMetricsPoster>();
mockPoster
.Setup(p => p.SendBatchMetricsAsync(It.IsAny <IEnumerable <DataPoint> >()))
.Callback <IEnumerable <DataPoint> >(c => points = c)
.Returns(new Task(NoOp));
var aggregator = new MetricAggregator(mockPoster.Object, 3);
var sw = Stopwatch.StartNew();
aggregator.SetupMetric(metricName3, MetricType.Total);
for (int i = 0; i < count1; i++)
{
aggregator.Increment(metricName1, 2);
aggregator.Add(metricName2, count2);
aggregator.Add(metricName3, count3);
}
sw.Stop();
Thread.Sleep(4500 - (int)sw.ElapsedMilliseconds);
Assert.NotNull(points);
Assert.Equal(count1 * 2, (int)(double)points.Single(dp => dp.Name == metricName1).Value);
Assert.Equal(count2, (double)points.Single(dp => dp.Name == metricName2).Value);
var val = points.Single(dp => dp.Name == metricName3);
Assert.Equal(count3 * count1, (double)val.Value);
}
public MetricsWorker(IMetricsScraper scraper, IMetricsStorage storage, IMetricsPublisher uploader)
{
this.scraper = Preconditions.CheckNotNull(scraper, nameof(scraper));
this.storage = Preconditions.CheckNotNull(storage, nameof(storage));
this.uploader = Preconditions.CheckNotNull(uploader, nameof(uploader));
this.metricFilter = new MetricTransformer()
.AddAllowedTags((MetricsConstants.MsTelemetry, true.ToString()))
.AddDisallowedTags(
("quantile", "0.1"),
("quantile", "0.5"),
("quantile", "0.99"))
.AddTagsToRemove(MetricsConstants.MsTelemetry, MetricsConstants.IotHubLabel, MetricsConstants.DeviceIdLabel)
.AddTagsToModify(
("id", this.ReplaceDeviceId),
("module_name", this.ReplaceModuleId),
("to", name => name.CreateSha256()),
("from", name => name.CreateSha256()),
("to_route_input", name => name.CreateSha256()),
("from_route_output", name => name.CreateSha256()));
#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter
this.metricAggregator = new MetricAggregator(
new AggregationTemplate("edgehub_gettwin_total", "id", new Summer()),
new AggregationTemplate(
"edgehub_messages_received_total",
("route_output", new Summer()),
("id", new Summer())
),
new AggregationTemplate(
"edgehub_messages_sent_total",
("from", new Summer()),
("to", new Summer()),
("from_route_output", new Summer()),
("to_route_input", new Summer())
),
new AggregationTemplate(
new string[]
{
"edgehub_message_size_bytes",
"edgehub_message_size_bytes_sum",
"edgehub_message_size_bytes_count"
},
"id",
new Averager()),
new AggregationTemplate(
new string[]
{
"edgehub_message_process_duration_seconds",
"edgehub_message_process_duration_seconds_sum",
"edgehub_message_process_duration_seconds_count",
},
("from", new Averager()),
("to", new Averager())
),
new AggregationTemplate(
"edgehub_direct_methods_total",
("from", new Summer()),
("to", new Summer())
),
new AggregationTemplate("edgehub_queue_length", "endpoint", new Summer()),
new AggregationTemplate(
new string[]
{
"edgehub_messages_dropped_total",
"edgehub_messages_unack_total",
},
("from", new Summer()),
("from_route_output", new Summer())
),
new AggregationTemplate("edgehub_client_connect_failed_total", "id", new Summer())
);
#pragma warning restore SA1111 // Closing parenthesis should be on line of last parameter
}
