public async STT.Task Stat_Aggregation_FaultTolerance_Records() { var statDataProvider = new TestStatisticalDataProvider(); var aggregator = new StatisticalDataAggregationController(statDataProvider, new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(), NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>()); // Initial state: records and aggregations simulate lack of two periods (at 2 min and 3 min). // 0:00 1:00 2:00 3:00 4:00 // | | | | | // r r r r r r r r r r r r r r r r r r r r / // <Minutely> // // The action: generating a hourly aggregation at 00:04:00. This operation's original result is the minutely aggregations // between 3 min and 4 min. // 0:00 1:00 2:00 3:00 4:00 // | | | | | // r r r r r r r r r r r r r r r r r r r r / // <Minutely> <Minutely> // // The expectation: the aggregator produces the original result but fills the gap: generates the 1 min and 2 min // aggregations too. // 0:00 1:00 2:00 3:00 4:00 // | | | | | // r r r r r r r r r r r r r r r r r r r r / // <Minutely><Minutely><Minutely><Minutely> var start = new DateTime(2021, 6, 28, 0, 0, 0); var milestone = new DateTime(2021, 6, 28, 0, 1, 0); var end = new DateTime(2021, 6, 28, 0, 4, 0); for (var now = start; now < end; now = now.AddSeconds(15)) { await GenerateWebHookRecordAsync(now, statDataProvider, CancellationToken.None); if (now == milestone) { await aggregator.AggregateAsync(now.AddSeconds(-1), TimeResolution.Minute, CancellationToken.None); } } var allAggregations = statDataProvider.Aggregations; Assert.AreEqual(16, statDataProvider.Storage.Count); Assert.AreEqual(1, allAggregations.Count); Assert.AreEqual(1, allAggregations.Count(x => x.Resolution == TimeResolution.Minute)); // ACTION var time = end.AddSeconds(-1); await aggregator.AggregateAsync(time, TimeResolution.Minute, CancellationToken.None); // ASSERT allAggregations = statDataProvider.Aggregations; Assert.AreEqual(4, allAggregations.Count); Assert.AreEqual(4, allAggregations.Count(x => x.Resolution == TimeResolution.Minute)); Assert.AreEqual("0 1 2 3", string.Join(" ", allAggregations.Select(x => x.Date.Minute.ToString()))); }
public async STT.Task Stat_Aggregation_FaultTolerance_AllAggregations() { var statDataProvider = new TestStatisticalDataProvider(); // now 2021-06-29 13:56:12 // Minute ... 2021-06-29 13:52:00 2021-06-29 13:53:00 2021-06-29 13:54:00 // Hour ... 2021-06-29 10:00:00 2021-06-29 11:00:00 2021-06-29 12:00:00 // Day ... 2021-06-26 00:00:00 2021-06-27 00:00:00 2021-06-28 00:00:00 // Month ... 2021-03-01 00:00:00 2021-04-01 00:00:00 2021-05-01 00:00:00 var now = new DateTime(2021, 6, 29, 13, 56, 12); for (int i = 0; i < 12; i++) { var date = now.AddSeconds(-i * 15 - 1); await GenerateWebHookRecordAsync(date, statDataProvider, CancellationToken.None); } for (int i = 0; i < 60 * 3; i++) { var date = now.Truncate(TimeResolution.Minute).AddMinutes(-i - 2); await GenerateWebHookAggregationAsync(date, TimeResolution.Minute, 10, statDataProvider); } for (int i = 0; i < 24 * 3; i++) { var date = now.Truncate(TimeResolution.Hour).AddHours(-i - 1); await GenerateWebHookAggregationAsync(date, TimeResolution.Hour, 10, statDataProvider); } for (int i = 0; i < 31 * 3; i++) { var date = now.Truncate(TimeResolution.Day).AddDays(-i - 1); await GenerateWebHookAggregationAsync(date, TimeResolution.Day, 10, statDataProvider); } for (int i = 0; i < 12 * 3; i++) { var date = now.Truncate(TimeResolution.Month).AddMonths(-i - 1); await GenerateWebHookAggregationAsync(date, TimeResolution.Month, 10, statDataProvider); } var aggregationCountBefore = statDataProvider.Aggregations.Count; // ACTION-1 no repair (every aggregations are present). var aggregator = new StatisticalDataAggregationController(statDataProvider, new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(), NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>()); var aggregationTime = now.Truncate(TimeResolution.Minute).AddSeconds(-1); await aggregator.AggregateAsync(aggregationTime, TimeResolution.Minute, CancellationToken.None); // ASSERT-1 Current aggregation is created. var aggregationCountAfter = statDataProvider.Aggregations.Count; Assert.AreEqual(aggregationCountBefore + 1, aggregationCountAfter); // ALIGN-2 Delete the current aggregation and two of each older aggregations var toDelete = new List <Aggregation>(); toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Minute) .OrderByDescending(x => x.Date).Take(3).ToArray()); toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Hour) .OrderByDescending(x => x.Date).Take(2).ToArray()); toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Day) .OrderByDescending(x => x.Date).Take(2).ToArray()); toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Month) .OrderByDescending(x => x.Date).Take(2).ToArray()); foreach (var item in toDelete) { statDataProvider.Aggregations.Remove(item); } aggregationCountBefore = statDataProvider.Aggregations.Count; // ACTION-2 repair 8 and generate 1 aggregator = new StatisticalDataAggregationController(statDataProvider, new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(), NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>()); aggregationTime = now.Truncate(TimeResolution.Minute).AddSeconds(-1); await aggregator.AggregateAsync(aggregationTime, TimeResolution.Minute, CancellationToken.None); // ASSERT-2 Current aggregation is created. aggregationCountAfter = statDataProvider.Aggregations.Count; Assert.AreEqual(aggregationCountBefore + 9, aggregationCountAfter); }