public async Task <IEnumerable <AggregatedDataRange> > GetAggregatedData(Guid entity, Guid attribute, int aggregationSeconds, TimeRange timeRange)
{
var series = await _model.ResolveSeries(entity, attribute, aggregationSeconds);
var query = $"SELECT Time, Value FROM deepflowdata WHERE Guid = {series} AND Time >= {timeRange.Min} AND Time < {timeRange.Max};";
var aggregationTask = _model.ResolveAggregationForSeries(series);
var timeRangesTask = GetAllTimeRanges(entity, attribute);
var rowSet = await _session.ExecuteAsync(new SimpleStatement(query)).ConfigureAwait(false);
var alTimeRanges = await timeRangesTask.ConfigureAwait(false);
var timeRanges = _filterer.FilterRanges(alTimeRanges, timeRange);
var aggregation = await aggregationTask.ConfigureAwait(false);
return(ToDataRanges(timeRanges, rowSet.Reverse().ToList(), timeRange, aggregation));
}
private async Task <Dictionary <int, IEnumerable <AggregatedDataRange> > > SaveHistoricalData(Guid entity, Guid attribute, AggregatedDataRange dataRange)
{
_aggregatedDataRangeValidator.ValidateAndThrow(dataRange);
var minAggregationSeconds = _configuration.AggregationsSeconds.Min();
if (dataRange.AggregationSeconds != minAggregationSeconds)
{
throw new Exception($"Must be lowest aggregation {minAggregationSeconds}");
}
var series = await _model.ResolveSeries(entity, attribute, minAggregationSeconds);
var maxAggregationSeconds = _configuration.AggregationsSeconds.Max();
var quantised = dataRange.TimeRange.Quantise(maxAggregationSeconds);
var semaphore = _seriesSemaphores.GetOrAdd(series, s => new SemaphoreSlim(1));
await semaphore.WaitAsync().ConfigureAwait(false);
try
{
//Console.WriteLine($"Writing data between {dataRange.TimeRange.Min} and {dataRange.TimeRange.Max}");
var persisted = await _persistence.GetAggregatedData(entity, attribute, dataRange.AggregationSeconds, quantised);
var existingTimeRanges = await _persistence.GetAllTimeRanges(entity, attribute);
if (existingTimeRanges.Any(x => x.Intersects(dataRange.TimeRange)))
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Overwriting data");
Console.ResetColor();
}
var merged = _aggregatedMerger.MergeRangeWithRanges(persisted, dataRange);
var aggregations = _aggregator.Aggregate(merged, quantised, _configuration.AggregationsSeconds);
//Console.WriteLine("Aggregations: " + JsonConvert.SerializeObject(aggregations));
var affectedAggregatedPoints = aggregations.Select(x => FilterAggregationToAffectedRanges(x.Value, dataRange.TimeRange, x.Key)).Where(x => x != null && x.Any()).ToList();
var rangesToPersist = affectedAggregatedPoints.Select(ranges => new ValueTuple <Guid, Guid, int, IEnumerable <AggregatedDataRange> >(entity, attribute, ranges.First().AggregationSeconds, ranges));
await _persistence.SaveData(rangesToPersist);
var afterTimeRanges = _timeMerger.MergeRangeWithRanges(existingTimeRanges, dataRange.TimeRange);
await _persistence.SaveTimeRanges(entity, attribute, afterTimeRanges);
var saved = await _persistence.GetTimeRanges(entity, attribute, new TimeRange(100, 4102444100));
if (JsonConvert.SerializeObject(existingTimeRanges) == JsonConvert.SerializeObject(saved))
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("No new time ranges saved");
Console.ResetColor();
}
/*Directory.CreateDirectory("timeRanges");
* File.WriteAllText($@"timeRanges\{entity}_{attribute}.json", JsonConvert.SerializeObject(afterTimeRanges));
* Console.WriteLine("Before time ranges: " + JsonConvert.SerializeObject(existingTimeRanges));
* Console.WriteLine("New time range: " + JsonConvert.SerializeObject(dataRange.TimeRange));
* Console.WriteLine("After time ranges: " + JsonConvert.SerializeObject(afterTimeRanges));*/
_telemetry.TrackTrace($"Saved historical data from {dataRange.TimeRange.Min.ToDateTime():s} to {dataRange.TimeRange.Max.ToDateTime():s} after receiving {dataRange.Data.Count / 2} points from source");
return(affectedAggregatedPoints.ToDictionary(x => x.First().AggregationSeconds, x => x));
}
finally
{
semaphore.Release();
}
}
