/// <summary>
/// Загружает подлежащие сжатию записи.
/// </summary>
/// <param name="compressSetting">Настройка сжатия статистики.</param>
/// <returns>Список записей статистики, загруженный из БД.</returns>
private List <StatisticsRecord> LoadRecordsForCompression(StatisticsCompressionSetting compressSetting)
{
// Сжиматься должны интервалы старше этого времени.
var timeLimit = AddTimeUnitsToDate(_timeService.Now, compressSetting.StatisticsAgeUnits, -compressSetting.StatisticsAgeCount);
// После сжатия должно получиться целое число новых интервалов, поэтому вычисляем точный конец сжимаемого промежутка времени.
var timeEnd = GetStartOfInterval(timeLimit, compressSetting.CompressTo);
// Формирование основного условия для выборки.
// Выбираются только записи, интервал которых меньше целевого.
var recordsQuery = _dataService.Query <StatisticsRecord>(StatisticsRecord.Views.CompressView);
var oneSecond = StatisticsInterval.OneSecond;
var tenSeconds = StatisticsInterval.TenSeconds;
var oneMinute = StatisticsInterval.OneMinute;
var fiveMinutes = StatisticsInterval.FiveMinutes;
var tenMinutes = StatisticsInterval.TenMinutes;
var hour = StatisticsInterval.Hour;
var day = StatisticsInterval.Day;
var month = StatisticsInterval.Month;
var quarter = StatisticsInterval.Quarter;
var year = StatisticsInterval.Year;
switch (compressSetting.CompressTo)
{
case StatisticsInterval.OneSecond:
recordsQuery = recordsQuery.Where(x => false);
break;
case StatisticsInterval.TenSeconds:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval == oneSecond || x.StatisticsInterval == tenSeconds);
break;
case StatisticsInterval.OneMinute:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval == oneSecond || x.StatisticsInterval == tenSeconds || x.StatisticsInterval == oneMinute);
break;
case StatisticsInterval.FiveMinutes:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval == oneSecond || x.StatisticsInterval == tenSeconds || x.StatisticsInterval == oneMinute || x.StatisticsInterval == fiveMinutes);
break;
case StatisticsInterval.TenMinutes:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval == oneSecond || x.StatisticsInterval == tenSeconds || x.StatisticsInterval == oneMinute || x.StatisticsInterval == fiveMinutes || x.StatisticsInterval == tenMinutes);
break;
case StatisticsInterval.HalfAnHour:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval != hour && x.StatisticsInterval != day && x.StatisticsInterval != month && x.StatisticsInterval != quarter && x.StatisticsInterval != year);
break;
case StatisticsInterval.Hour:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval != day && x.StatisticsInterval != month && x.StatisticsInterval != quarter && x.StatisticsInterval != year);
break;
case StatisticsInterval.Day:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval != month && x.StatisticsInterval != quarter && x.StatisticsInterval != year);
break;
case StatisticsInterval.Month:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval != quarter && x.StatisticsInterval != year);
break;
case StatisticsInterval.Quarter:
recordsQuery = recordsQuery.Where(x => x.StatisticsInterval != year);
break;
case StatisticsInterval.Year:
recordsQuery = recordsQuery.Where(x => true);
break;
default:
throw new Exception("Неизвестный интервал статистики: " + compressSetting.CompressTo.ToString());
}
recordsQuery = recordsQuery.Where(x => x.StatisticsSetting.__PrimaryKey == compressSetting.StatisticsSetting.__PrimaryKey && x.To <= timeEnd).OrderBy(x => x.Since);
if (MaxRecordsForOneCompression > 0)
{
recordsQuery = recordsQuery.Take(MaxRecordsForOneCompression);
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var dobjs = recordsQuery.ToList();
stopwatch.Stop();
long time = stopwatch.ElapsedMilliseconds;
_statisticsService.NotifyAvgTimeSql(compressSetting.StatisticsSetting.Subscription, (int)time, "DefaultStatisticsCompressorService.LoadRecordsForCompression() load StatRecords.");
return(dobjs);
}
private List <StatisticsRecord> CompressRecords(IEnumerable <StatisticsRecord> records, StatisticsCompressionSetting compressSetting)
{
// Формирование списка новых сжатых записей.
var compressedRecords = records
.GroupBy(x => GetStartOfInterval(x.Since, compressSetting.CompressTo))
.Select(x =>
new StatisticsRecord()
{
StatisticsSetting = compressSetting.StatisticsSetting,
Since = x.Key,
To = AddStatIntervalToDate(x.Key, compressSetting.CompressTo, 1),
StatisticsInterval = compressSetting.CompressTo,
SentCount = x.Sum(y => y.SentCount),
ReceivedCount = x.Sum(y => y.ReceivedCount),
ErrorsCount = x.Sum(y => y.ErrorsCount),
UniqueErrorsCount = x.OrderBy(y => y.Since).Last().UniqueErrorsCount,
QueueLength = x.OrderBy(y => y.Since).Last().QueueLength,
SentAvgTime = x.Sum(y => y.SentAvgTime) / x.Count(),
QueryAvgTime = x.Sum(y => y.QueryAvgTime) / x.Count(),
ConnectionCount = x.OrderBy(y => y.Since).Last().ConnectionCount,
})
.ToList();
return(compressedRecords);
}
public void TestPartCompression()
{
foreach (var dataService in DataServices)
{
// Arrange.
var statSetting = new StatisticsSetting();
var compressionSetting = new StatisticsCompressionSetting()
{
StatisticsSetting = statSetting,
CompressTo = StatisticsInterval.Hour,
StatisticsAgeCount = 1,
StatisticsAgeUnits = TimeUnit.Minute,
NextCompressTime = DateTime.Now,
};
var dataObjects = new DataObject[100];
var date = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, DateTime.Now.Hour - 1, 0, 0, 0);
for (int i = 0; i < 100; i++)
{
dataObjects[i] = new StatisticsRecord()
{
Since = date.AddMinutes(i),
To = date.AddMinutes(i + 1),
StatisticsInterval = StatisticsInterval.OneMinute,
SentCount = 1,
StatisticsSetting = statSetting,
};
}
var counter = 0;
var mockTimeService = new Mock <IStatisticsTimeService>();
mockTimeService
.Setup(ts => ts.Now)
.Callback(() =>
{
counter++;
if (counter == 5)
{
throw new Exception("As if something went wrong.");
}
}).Returns(DateTime.Now);
dataService.UpdateObject(statSetting);
dataService.UpdateObject(compressionSetting);
dataService.UpdateObjects(ref dataObjects);
var component = new DefaultStatisticsCompressorService(
dataService,
mockTimeService.Object,
GetMockLogger(),
GetMockStatisticsService())
{
CompressionPeriod = 1000,
MaxRecordsForOneCompression = 50,
};
// Act.
Act(component);
// Assert.
var statRecords = dataService.Query <StatisticsRecord>(StatisticsRecord.Views.CompressView)
.OrderBy(r => r.Since)
.ToArray();
var compressedRecord = statRecords.First();
var unCompressedRecord = statRecords.Skip(1).First();
Assert.Equal(41, statRecords.Count());
Assert.Equal(60, compressedRecord.SentCount);
Assert.Equal(StatisticsInterval.Hour, compressedRecord.StatisticsInterval);
Assert.Equal(StatisticsInterval.OneMinute, unCompressedRecord.StatisticsInterval);
}
}
