private async Task StoreAggregatedMetricsInternalAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
{
bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace);
if (isTraceLogLevelEnabled)
{
_logger.LogTrace("Storing {CountersCount} counters, {GaugesCount} gauges, {TimingsCount} timings.", counters.Count, gauges.Count, timings.Count);
}
try {
await Run.WithRetriesAsync(() => StoreAggregatedMetricsAsync(timeBucket, counters, gauges, timings)).AnyContext();
} catch (Exception ex) {
if (_logger.IsEnabled(LogLevel.Error))
{
_logger.LogError(ex, "Error storing aggregated metrics: {Message}", ex.Message);
}
throw;
}
await OnCountedAsync(counters.Sum(c => c.Value)).AnyContext();
if (isTraceLogLevelEnabled)
{
_logger.LogTrace("Done storing aggregated metrics");
}
}
private async Task StoreGaugeAsync(TimeBucket timeBucket, AggregatedGaugeMetric gauge)
{
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Storing gauge name={Name} count={Count} total={Total} last={Last} min={Min} max={Max} time={StartTimeUtc}", gauge.Key.Name, gauge.Count, gauge.Total, gauge.Last, gauge.Min, gauge.Max, gauge.Key.StartTimeUtc);
}
string countKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
string totalDurationKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
string lastKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Last);
string minKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
string maxKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
await Task.WhenAll(
_cache.IncrementAsync(countKey, gauge.Count, timeBucket.Ttl),
_cache.IncrementAsync(totalDurationKey, gauge.Total, timeBucket.Ttl),
_cache.SetAsync(lastKey, gauge.Last, timeBucket.Ttl),
_cache.SetIfLowerAsync(minKey, gauge.Min, timeBucket.Ttl),
_cache.SetIfHigherAsync(maxKey, gauge.Max, timeBucket.Ttl)
).AnyContext();
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Done storing gauge name={Name}", gauge.Key.Name);
}
}
protected override async Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
{
var metrics = new List <MetricDatum>();
metrics.AddRange(ConvertToDatums(counters));
metrics.AddRange(ConvertToDatums(gauges));
metrics.AddRange(ConvertToDatums(timings));
int page = 1;
int pageSize = 20; // CloudWatch only allows max 20 metrics at once.
do
{
var metricsPage = metrics.Skip((page - 1) * pageSize).Take(pageSize).ToList();
if (metricsPage.Count == 0)
{
break;
}
_logger.Trace(() => $"Sending PutMetricData to AWS for {metricsPage.Count} metric(s)");
// do retries
var response = await _client.Value.PutMetricDataAsync(new PutMetricDataRequest {
Namespace = _options.Namespace,
MetricData = metricsPage
}).AnyContext();
if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
throw new AmazonCloudWatchException("Unable to post metrics.");
}
page++;
} while (true);
}
public void GetEpochNumberFromTable(string tableName, long expectedValue)
{
var table = GetTableForName(tableName);
var actual = TimeBucket.GetEpochNumberFromTable(table);
Assert.Equal(expectedValue, actual);
}
private async Task StoreCounterAsync(TimeBucket timeBucket, AggregatedCounterMetric counter)
{
_logger.Trace(() => $"Storing counter name={counter.Key.Name} value={counter.Value} time={counter.Key.Duration}");
string bucketKey = GetBucketKey(CacheMetricNames.Counter, counter.Key.Name, counter.Key.StartTimeUtc, timeBucket.Size);
await _cache.IncrementAsync(bucketKey, counter.Value, timeBucket.Ttl).AnyContext();
_logger.Trace(() => $"Done storing counter name={counter.Key.Name}");
}
public void GetTableForDateTime()
{
var provider = new MockProvider();
var table = TimeBucket.GetTableForDateTime(provider, new DateTime(2017, 8, 5));
var name = table.Name;
Assert.Equal("foo201708", name);
}
public void ConvertToBucket()
{
// Baseline test. Will catch breaking changes in the formula.
var date = new DateTime(2019, 12, 1);
var bucket = TimeBucket.ConvertToBucket(date);
var expectedBucket = 10474560;
Assert.Equal(expectedBucket, bucket);
var date2 = TimeBucket.ConvertToDateTime(bucket);
Assert.Equal(date, date2);
}
private async Task StoreCounterAsync(TimeBucket timeBucket, AggregatedCounterMetric counter)
{
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Storing counter name={Name} value={Value} time={Duration}", counter.Key.Name, counter.Value, counter.Key.Duration);
}
string bucketKey = GetBucketKey(CacheMetricNames.Counter, counter.Key.Name, counter.Key.StartTimeUtc, timeBucket.Size);
await _cache.IncrementAsync(bucketKey, counter.Value, timeBucket.Ttl).AnyContext();
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Done storing counter name={Name}", counter.Key.Name);
}
}
private async Task StoreAggregatedMetricsInternalAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
{
_logger.Trace(() => $"Storing {counters.Count} counters, {gauges.Count} gauges, {timings.Count} timings.");
try {
await Run.WithRetriesAsync(() => StoreAggregatedMetricsAsync(timeBucket, counters, gauges, timings)).AnyContext();
} catch (Exception ex) {
_logger.Error(ex, $"Error storing aggregated metrics: {ex.Message}");
throw;
}
await OnCountedAsync(counters.Sum(c => c.Value)).AnyContext();
_logger.Trace("Done storing aggregated metrics");
}
protected override async Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
{
foreach (var counter in counters)
{
await StoreCounterAsync(timeBucket, counter).AnyContext();
}
foreach (var gauge in gauges)
{
await StoreGaugeAsync(timeBucket, gauge).AnyContext();
}
foreach (var timing in timings)
{
await StoreTimingAsync(timeBucket, timing).AnyContext();
}
}
private async Task StoreTimingAsync(TimeBucket timeBucket, AggregatedTimingMetric timing)
{
_logger.Trace(() => $"Storing timing name={timing.Key.Name} count={timing.Count} total={timing.TotalDuration} min={timing.MinDuration} max={timing.MaxDuration} time={timing.Key.StartTimeUtc}");
string countKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
await _cache.IncrementAsync(countKey, timing.Count, timeBucket.Ttl).AnyContext();
string totalDurationKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
await _cache.IncrementAsync(totalDurationKey, timing.TotalDuration, timeBucket.Ttl).AnyContext();
string maxKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
await _cache.SetIfHigherAsync(maxKey, timing.MaxDuration, timeBucket.Ttl).AnyContext();
string minKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
await _cache.SetIfLowerAsync(minKey, timing.MinDuration, timeBucket.Ttl).AnyContext();
_logger.Trace(() => $"Done storing timing name={timing.Key.Name}");
}
protected override Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
{
var tasks = new List <Task>();
foreach (var counter in counters)
{
tasks.Add(StoreCounterAsync(timeBucket, counter));
}
foreach (var gauge in gauges)
{
tasks.Add(StoreGaugeAsync(timeBucket, gauge));
}
foreach (var timing in timings)
{
tasks.Add(StoreTimingAsync(timeBucket, timing));
}
return(Task.WhenAll(tasks));
}
private async Task StoreGaugeAsync(TimeBucket timeBucket, AggregatedGaugeMetric gauge)
{
_logger.Trace(() => $"Storing gauge name={gauge.Key.Name} count={gauge.Count} total={gauge.Total} last={gauge.Last} min={gauge.Min} max={gauge.Max} time={gauge.Key.StartTimeUtc}");
string countKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
await _cache.IncrementAsync(countKey, gauge.Count, timeBucket.Ttl).AnyContext();
string totalDurationKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
await _cache.IncrementAsync(totalDurationKey, gauge.Total, timeBucket.Ttl).AnyContext();
string lastKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Last);
await _cache.SetAsync(lastKey, gauge.Last, timeBucket.Ttl).AnyContext();
string minKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
await _cache.SetIfLowerAsync(minKey, gauge.Min, timeBucket.Ttl).AnyContext();
string maxKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
await _cache.SetIfHigherAsync(maxKey, gauge.Max, timeBucket.Ttl).AnyContext();
_logger.Trace(() => $"Done storing gauge name={gauge.Key.Name}");
}
private async Task StoreTimingAsync(TimeBucket timeBucket, AggregatedTimingMetric timing)
{
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Storing timing name={Name} count={Count} total={TotalDuration} min={MinDuration} max={MaxDuration} time={StartTimeUtc}", timing.Key.Name, timing.Count, timing.TotalDuration, timing.MinDuration, timing.MaxDuration, timing.Key.StartTimeUtc);
}
string countKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
string totalDurationKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
string maxKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
string minKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
await Task.WhenAll(
_cache.IncrementAsync(countKey, timing.Count, timeBucket.Ttl),
_cache.IncrementAsync(totalDurationKey, timing.TotalDuration, timeBucket.Ttl),
_cache.SetIfHigherAsync(maxKey, timing.MaxDuration, timeBucket.Ttl),
_cache.SetIfLowerAsync(minKey, timing.MinDuration, timeBucket.Ttl)
).AnyContext();
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Done storing timing name={Name}", timing.Key.Name);
}
}
protected abstract Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings);
protected override async Task <Tuple <TimeBucket <TCalendar>, KpiValue> > Extract(KpiType kpi, TimeBucket <TCalendar> timeBucket, string targetId)
{
var parameters = SetParametersValue(kpi.Id, timeBucket, targetId, kpi.Parameters).Cast <object>().ToArray();
var commandText = new SqlCommand(kpi.Query).CommandText;
var kpiValue = _targetDbContext.Set <KpiValue>().FromSqlRaw(commandText, parameters).AsNoTracking()
.FirstOrDefault() ?? new KpiValue()
{
Value = 0,
DateTimePoint = timeBucket.Start,
KpiTypeId = kpi.Id,
ExtractedDateTime = DateTime.Now,
TargetId = targetId,
};
if (!timeBucket.IsPartial)
{
await Context.KpiValues.AddAsync(kpiValue);
await Context.SaveChangesAsync();
}
return(new Tuple <TimeBucket <TCalendar>, KpiValue>(timeBucket, kpiValue));
}