Пример #1
0
        private async Task StoreAggregatedMetricsInternalAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
        {
            bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace);

            if (isTraceLogLevelEnabled)
            {
                _logger.LogTrace("Storing {CountersCount} counters, {GaugesCount} gauges, {TimingsCount} timings.", counters.Count, gauges.Count, timings.Count);
            }

            try {
                await Run.WithRetriesAsync(() => StoreAggregatedMetricsAsync(timeBucket, counters, gauges, timings)).AnyContext();
            } catch (Exception ex) {
                if (_logger.IsEnabled(LogLevel.Error))
                {
                    _logger.LogError(ex, "Error storing aggregated metrics: {Message}", ex.Message);
                }
                throw;
            }

            await OnCountedAsync(counters.Sum(c => c.Value)).AnyContext();

            if (isTraceLogLevelEnabled)
            {
                _logger.LogTrace("Done storing aggregated metrics");
            }
        }
        private async Task StoreGaugeAsync(TimeBucket timeBucket, AggregatedGaugeMetric gauge)
        {
            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Storing gauge name={Name} count={Count} total={Total} last={Last} min={Min} max={Max} time={StartTimeUtc}", gauge.Key.Name, gauge.Count, gauge.Total, gauge.Last, gauge.Min, gauge.Max, gauge.Key.StartTimeUtc);
            }

            string countKey         = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
            string totalDurationKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
            string lastKey          = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Last);
            string minKey           = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
            string maxKey           = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);

            await Task.WhenAll(
                _cache.IncrementAsync(countKey, gauge.Count, timeBucket.Ttl),
                _cache.IncrementAsync(totalDurationKey, gauge.Total, timeBucket.Ttl),
                _cache.SetAsync(lastKey, gauge.Last, timeBucket.Ttl),
                _cache.SetIfLowerAsync(minKey, gauge.Min, timeBucket.Ttl),
                _cache.SetIfHigherAsync(maxKey, gauge.Max, timeBucket.Ttl)
                ).AnyContext();

            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Done storing gauge name={Name}", gauge.Key.Name);
            }
        }
Пример #3
0
        protected override async Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
        {
            var metrics = new List <MetricDatum>();

            metrics.AddRange(ConvertToDatums(counters));
            metrics.AddRange(ConvertToDatums(gauges));
            metrics.AddRange(ConvertToDatums(timings));

            int page     = 1;
            int pageSize = 20; // CloudWatch only allows max 20 metrics at once.

            do
            {
                var metricsPage = metrics.Skip((page - 1) * pageSize).Take(pageSize).ToList();
                if (metricsPage.Count == 0)
                {
                    break;
                }

                _logger.Trace(() => $"Sending PutMetricData to AWS for {metricsPage.Count} metric(s)");
                // do retries
                var response = await _client.Value.PutMetricDataAsync(new PutMetricDataRequest {
                    Namespace  = _options.Namespace,
                    MetricData = metricsPage
                }).AnyContext();

                if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
                {
                    throw new AmazonCloudWatchException("Unable to post metrics.");
                }

                page++;
            } while (true);
        }
Пример #4
0
        public void GetEpochNumberFromTable(string tableName, long expectedValue)
        {
            var table = GetTableForName(tableName);

            var actual = TimeBucket.GetEpochNumberFromTable(table);

            Assert.Equal(expectedValue, actual);
        }
        private async Task StoreCounterAsync(TimeBucket timeBucket, AggregatedCounterMetric counter)
        {
            _logger.Trace(() => $"Storing counter name={counter.Key.Name} value={counter.Value} time={counter.Key.Duration}");

            string bucketKey = GetBucketKey(CacheMetricNames.Counter, counter.Key.Name, counter.Key.StartTimeUtc, timeBucket.Size);
            await _cache.IncrementAsync(bucketKey, counter.Value, timeBucket.Ttl).AnyContext();

            _logger.Trace(() => $"Done storing counter name={counter.Key.Name}");
        }
Пример #6
0
        public void GetTableForDateTime()
        {
            var provider = new MockProvider();
            var table    = TimeBucket.GetTableForDateTime(provider, new DateTime(2017, 8, 5));

            var name = table.Name;

            Assert.Equal("foo201708", name);
        }
Пример #7
0
        public void ConvertToBucket()
        {
            // Baseline test. Will catch breaking changes in the formula.
            var date           = new DateTime(2019, 12, 1);
            var bucket         = TimeBucket.ConvertToBucket(date);
            var expectedBucket = 10474560;

            Assert.Equal(expectedBucket, bucket);

            var date2 = TimeBucket.ConvertToDateTime(bucket);

            Assert.Equal(date, date2);
        }
        private async Task StoreCounterAsync(TimeBucket timeBucket, AggregatedCounterMetric counter)
        {
            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Storing counter name={Name} value={Value} time={Duration}", counter.Key.Name, counter.Value, counter.Key.Duration);
            }

            string bucketKey = GetBucketKey(CacheMetricNames.Counter, counter.Key.Name, counter.Key.StartTimeUtc, timeBucket.Size);
            await _cache.IncrementAsync(bucketKey, counter.Value, timeBucket.Ttl).AnyContext();

            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Done storing counter name={Name}", counter.Key.Name);
            }
        }
        private async Task StoreAggregatedMetricsInternalAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
        {
            _logger.Trace(() => $"Storing {counters.Count} counters, {gauges.Count} gauges, {timings.Count} timings.");

            try {
                await Run.WithRetriesAsync(() => StoreAggregatedMetricsAsync(timeBucket, counters, gauges, timings)).AnyContext();
            } catch (Exception ex) {
                _logger.Error(ex, $"Error storing aggregated metrics: {ex.Message}");
                throw;
            }

            await OnCountedAsync(counters.Sum(c => c.Value)).AnyContext();

            _logger.Trace("Done storing aggregated metrics");
        }
        protected override async Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
        {
            foreach (var counter in counters)
            {
                await StoreCounterAsync(timeBucket, counter).AnyContext();
            }

            foreach (var gauge in gauges)
            {
                await StoreGaugeAsync(timeBucket, gauge).AnyContext();
            }

            foreach (var timing in timings)
            {
                await StoreTimingAsync(timeBucket, timing).AnyContext();
            }
        }
        private async Task StoreTimingAsync(TimeBucket timeBucket, AggregatedTimingMetric timing)
        {
            _logger.Trace(() => $"Storing timing name={timing.Key.Name} count={timing.Count} total={timing.TotalDuration} min={timing.MinDuration} max={timing.MaxDuration} time={timing.Key.StartTimeUtc}");

            string countKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
            await _cache.IncrementAsync(countKey, timing.Count, timeBucket.Ttl).AnyContext();

            string totalDurationKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
            await _cache.IncrementAsync(totalDurationKey, timing.TotalDuration, timeBucket.Ttl).AnyContext();

            string maxKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
            await _cache.SetIfHigherAsync(maxKey, timing.MaxDuration, timeBucket.Ttl).AnyContext();

            string minKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
            await _cache.SetIfLowerAsync(minKey, timing.MinDuration, timeBucket.Ttl).AnyContext();

            _logger.Trace(() => $"Done storing timing name={timing.Key.Name}");
        }
        protected override Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings)
        {
            var tasks = new List <Task>();

            foreach (var counter in counters)
            {
                tasks.Add(StoreCounterAsync(timeBucket, counter));
            }

            foreach (var gauge in gauges)
            {
                tasks.Add(StoreGaugeAsync(timeBucket, gauge));
            }

            foreach (var timing in timings)
            {
                tasks.Add(StoreTimingAsync(timeBucket, timing));
            }

            return(Task.WhenAll(tasks));
        }
        private async Task StoreGaugeAsync(TimeBucket timeBucket, AggregatedGaugeMetric gauge)
        {
            _logger.Trace(() => $"Storing gauge name={gauge.Key.Name} count={gauge.Count} total={gauge.Total} last={gauge.Last} min={gauge.Min} max={gauge.Max} time={gauge.Key.StartTimeUtc}");

            string countKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
            await _cache.IncrementAsync(countKey, gauge.Count, timeBucket.Ttl).AnyContext();

            string totalDurationKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
            await _cache.IncrementAsync(totalDurationKey, gauge.Total, timeBucket.Ttl).AnyContext();

            string lastKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Last);
            await _cache.SetAsync(lastKey, gauge.Last, timeBucket.Ttl).AnyContext();

            string minKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);
            await _cache.SetIfLowerAsync(minKey, gauge.Min, timeBucket.Ttl).AnyContext();

            string maxKey = GetBucketKey(CacheMetricNames.Gauge, gauge.Key.Name, gauge.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
            await _cache.SetIfHigherAsync(maxKey, gauge.Max, timeBucket.Ttl).AnyContext();

            _logger.Trace(() => $"Done storing gauge name={gauge.Key.Name}");
        }
        private async Task StoreTimingAsync(TimeBucket timeBucket, AggregatedTimingMetric timing)
        {
            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Storing timing name={Name} count={Count} total={TotalDuration} min={MinDuration} max={MaxDuration} time={StartTimeUtc}", timing.Key.Name, timing.Count, timing.TotalDuration, timing.MinDuration, timing.MaxDuration, timing.Key.StartTimeUtc);
            }

            string countKey         = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Count);
            string totalDurationKey = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Total);
            string maxKey           = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Max);
            string minKey           = GetBucketKey(CacheMetricNames.Timing, timing.Key.Name, timing.Key.StartTimeUtc, timeBucket.Size, CacheMetricNames.Min);

            await Task.WhenAll(
                _cache.IncrementAsync(countKey, timing.Count, timeBucket.Ttl),
                _cache.IncrementAsync(totalDurationKey, timing.TotalDuration, timeBucket.Ttl),
                _cache.SetIfHigherAsync(maxKey, timing.MaxDuration, timeBucket.Ttl),
                _cache.SetIfLowerAsync(minKey, timing.MinDuration, timeBucket.Ttl)
                ).AnyContext();

            if (_logger.IsEnabled(LogLevel.Trace))
            {
                _logger.LogTrace("Done storing timing name={Name}", timing.Key.Name);
            }
        }
 protected abstract Task StoreAggregatedMetricsAsync(TimeBucket timeBucket, ICollection <AggregatedCounterMetric> counters, ICollection <AggregatedGaugeMetric> gauges, ICollection <AggregatedTimingMetric> timings);
Пример #16
0
        protected override async Task <Tuple <TimeBucket <TCalendar>, KpiValue> > Extract(KpiType kpi, TimeBucket <TCalendar> timeBucket, string targetId)
        {
            var parameters  = SetParametersValue(kpi.Id, timeBucket, targetId, kpi.Parameters).Cast <object>().ToArray();
            var commandText = new SqlCommand(kpi.Query).CommandText;



            var kpiValue = _targetDbContext.Set <KpiValue>().FromSqlRaw(commandText, parameters).AsNoTracking()
                           .FirstOrDefault() ?? new KpiValue()
            {
                Value             = 0,
                DateTimePoint     = timeBucket.Start,
                KpiTypeId         = kpi.Id,
                ExtractedDateTime = DateTime.Now,
                TargetId          = targetId,
            };

            if (!timeBucket.IsPartial)
            {
                await Context.KpiValues.AddAsync(kpiValue);

                await Context.SaveChangesAsync();
            }

            return(new Tuple <TimeBucket <TCalendar>, KpiValue>(timeBucket, kpiValue));
        }