public async STT.Task Stat_Aggregation_FaultTolerance_Records()
{
var statDataProvider = new TestStatisticalDataProvider();
var aggregator = new StatisticalDataAggregationController(statDataProvider,
new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(),
NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>());
// Initial state: records and aggregations simulate lack of two periods (at 2 min and 3 min).
// 0:00 1:00 2:00 3:00 4:00
// | | | | |
// r r r r r r r r r r r r r r r r r r r r /
// <Minutely>
//
// The action: generating a hourly aggregation at 00:04:00. This operation's original result is the minutely aggregations
// between 3 min and 4 min.
// 0:00 1:00 2:00 3:00 4:00
// | | | | |
// r r r r r r r r r r r r r r r r r r r r /
// <Minutely> <Minutely>
//
// The expectation: the aggregator produces the original result but fills the gap: generates the 1 min and 2 min
// aggregations too.
// 0:00 1:00 2:00 3:00 4:00
// | | | | |
// r r r r r r r r r r r r r r r r r r r r /
// <Minutely><Minutely><Minutely><Minutely>
var start = new DateTime(2021, 6, 28, 0, 0, 0);
var milestone = new DateTime(2021, 6, 28, 0, 1, 0);
var end = new DateTime(2021, 6, 28, 0, 4, 0);
for (var now = start; now < end; now = now.AddSeconds(15))
{
await GenerateWebHookRecordAsync(now, statDataProvider, CancellationToken.None);
if (now == milestone)
{
await aggregator.AggregateAsync(now.AddSeconds(-1), TimeResolution.Minute, CancellationToken.None);
}
}
var allAggregations = statDataProvider.Aggregations;
Assert.AreEqual(16, statDataProvider.Storage.Count);
Assert.AreEqual(1, allAggregations.Count);
Assert.AreEqual(1, allAggregations.Count(x => x.Resolution == TimeResolution.Minute));
// ACTION
var time = end.AddSeconds(-1);
await aggregator.AggregateAsync(time, TimeResolution.Minute, CancellationToken.None);
// ASSERT
allAggregations = statDataProvider.Aggregations;
Assert.AreEqual(4, allAggregations.Count);
Assert.AreEqual(4, allAggregations.Count(x => x.Resolution == TimeResolution.Minute));
Assert.AreEqual("0 1 2 3", string.Join(" ", allAggregations.Select(x => x.Date.Minute.ToString())));
}
public async STT.Task Stat_Aggregation_FaultTolerance_AllAggregations()
{
var statDataProvider = new TestStatisticalDataProvider();
// now 2021-06-29 13:56:12
// Minute ... 2021-06-29 13:52:00 2021-06-29 13:53:00 2021-06-29 13:54:00
// Hour ... 2021-06-29 10:00:00 2021-06-29 11:00:00 2021-06-29 12:00:00
// Day ... 2021-06-26 00:00:00 2021-06-27 00:00:00 2021-06-28 00:00:00
// Month ... 2021-03-01 00:00:00 2021-04-01 00:00:00 2021-05-01 00:00:00
var now = new DateTime(2021, 6, 29, 13, 56, 12);
for (int i = 0; i < 12; i++)
{
var date = now.AddSeconds(-i * 15 - 1);
await GenerateWebHookRecordAsync(date, statDataProvider, CancellationToken.None);
}
for (int i = 0; i < 60 * 3; i++)
{
var date = now.Truncate(TimeResolution.Minute).AddMinutes(-i - 2);
await GenerateWebHookAggregationAsync(date, TimeResolution.Minute, 10, statDataProvider);
}
for (int i = 0; i < 24 * 3; i++)
{
var date = now.Truncate(TimeResolution.Hour).AddHours(-i - 1);
await GenerateWebHookAggregationAsync(date, TimeResolution.Hour, 10, statDataProvider);
}
for (int i = 0; i < 31 * 3; i++)
{
var date = now.Truncate(TimeResolution.Day).AddDays(-i - 1);
await GenerateWebHookAggregationAsync(date, TimeResolution.Day, 10, statDataProvider);
}
for (int i = 0; i < 12 * 3; i++)
{
var date = now.Truncate(TimeResolution.Month).AddMonths(-i - 1);
await GenerateWebHookAggregationAsync(date, TimeResolution.Month, 10, statDataProvider);
}
var aggregationCountBefore = statDataProvider.Aggregations.Count;
// ACTION-1 no repair (every aggregations are present).
var aggregator = new StatisticalDataAggregationController(statDataProvider,
new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(),
NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>());
var aggregationTime = now.Truncate(TimeResolution.Minute).AddSeconds(-1);
await aggregator.AggregateAsync(aggregationTime, TimeResolution.Minute, CancellationToken.None);
// ASSERT-1 Current aggregation is created.
var aggregationCountAfter = statDataProvider.Aggregations.Count;
Assert.AreEqual(aggregationCountBefore + 1, aggregationCountAfter);
// ALIGN-2 Delete the current aggregation and two of each older aggregations
var toDelete = new List <Aggregation>();
toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Minute)
.OrderByDescending(x => x.Date).Take(3).ToArray());
toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Hour)
.OrderByDescending(x => x.Date).Take(2).ToArray());
toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Day)
.OrderByDescending(x => x.Date).Take(2).ToArray());
toDelete.AddRange(statDataProvider.Aggregations.Where(x => x.Resolution == TimeResolution.Month)
.OrderByDescending(x => x.Date).Take(2).ToArray());
foreach (var item in toDelete)
{
statDataProvider.Aggregations.Remove(item);
}
aggregationCountBefore = statDataProvider.Aggregations.Count;
// ACTION-2 repair 8 and generate 1
aggregator = new StatisticalDataAggregationController(statDataProvider,
new[] { new WebHookStatisticalDataAggregator(GetOptions()) }, GetOptions(),
NullLoggerFactory.Instance.CreateLogger <StatisticalDataAggregationController>());
aggregationTime = now.Truncate(TimeResolution.Minute).AddSeconds(-1);
await aggregator.AggregateAsync(aggregationTime, TimeResolution.Minute, CancellationToken.None);
// ASSERT-2 Current aggregation is created.
aggregationCountAfter = statDataProvider.Aggregations.Count;
Assert.AreEqual(aggregationCountBefore + 9, aggregationCountAfter);
}