private async Task <IEnumerable <string> > GetIndicesAsync(MetricQuery query) { var indices = new List <string>(); var options = _optionsProvider.Value.ElasticSearch; if (options.DefaultIndex != null) { indices.Add(string.Format(options.DefaultIndex, DateTime.UtcNow)); } if (query.Indices.Any()) { indices.AddRange(query.Indices.Select(i => string.Format(i, DateTime.UtcNow))); } if (indices.Count < 1) { throw new InvalidOperationException($"There's no indexes provided for query '{query.Name}', ensure your configuration is correct"); } var existsTasks = indices.Select(i => _client.Indices.ExistsAsync(Indices.Index(i))); var indexExists = await Task.WhenAll(existsTasks); if (indexExists.All(r => !r.Exists)) { throw new InvalidOperationException( $"Unable to find any existing index for query '{query.Name}', ensure index format and name provided correctly"); } return(indices); }
public async Task EvaluateAsync(MetricQuery query, CancellationToken cancellation = default) { var options = _optionsProvider.Value; var timeout = query.Timeout.GetValueOrDefault(options.Metrics.Evaluation.Timeout); var interval = query.Interval.GetValueOrDefault(options.Metrics.Evaluation.Interval); try { _logger.LogInformation("Starting evaluation of query '{Query}'", query.Name); var result = await _queryExecutor.ExecuteAsync(query, timeout, cancellation); await _metricsWriter.WriteAsync(query, result, cancellation); } catch (Exception e) { _logger.LogError(e, "Query '{Query}' evaluation failed with exception", query.Name); } finally { _logger.LogInformation("Query '{Query}' evaluated. Next evaluation on '{NextOccurence}'", query.Name, (DateTime.Now + interval)); // Re-schedule _jobClient.Schedule <ScheduledMetricsEvaluator>( e => e.EvaluateAsync(query, cancellation), interval); } }
public async Task API_MetricQueryTest() { var widget = new TestWidget(); var handler = new MetricQuery(); var request = MetricQueryRequest.Create(widget); await handler.Handle(request, CancellationToken.None); Assert.AreEqual(1, widget.Value); }
/// <inheritdoc /> public async Task <IMetricQueryResult> ExecuteAsync(MetricQuery query, TimeSpan timeout, CancellationToken cancellation = default) { var result = query.Type switch { MetricQueryType.Default => ExecuteDefaultAsync((DefaultMetricQuery)query, timeout, cancellation), MetricQueryType.Raw => ExecuteRawAsync((RawMetricQuery)query, timeout, cancellation), _ => throw new InvalidOperationException($"Query type '{query.Type}' not supported") }; return(await result); }
public static IQuery GetSpecificQuery(Prometheus.MetricFactory metricFactory, MetricQuery metricQuery) { switch (metricQuery.QueryUsage) { case QueryUsage.Counter: var labelColumns = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.CounterLabel) .Select(x => new CounterGroupQuery.Column(x.Name, x.Order ?? 0, x.Label)); var valueColumn = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.Counter) .Select(x => new CounterGroupQuery.Column(x.Name, x.Order ?? 0, x.Label)) .FirstOrDefault(); return(new CounterGroupQuery(metricQuery.Name, metricQuery.Description ?? string.Empty, metricQuery.Query, labelColumns, valueColumn, metricFactory, metricQuery.MillisecondTimeout)); case QueryUsage.Gauge: var gaugeLabelColumns = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.GaugeLabel) .Select(x => new GaugeGroupQuery.Column(x.Name, x.Order ?? 0, x.Label)); var gaugeValueColumn = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.Gauge) .Select(x => new GaugeGroupQuery.Column(x.Name, x.Order ?? 0, x.Label)) .FirstOrDefault(); return(new GaugeGroupQuery(metricQuery.Name, metricQuery.Description ?? string.Empty, metricQuery.Query, gaugeLabelColumns, gaugeValueColumn, metricFactory, metricQuery.MillisecondTimeout)); case QueryUsage.Empty: var gaugeColumns = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.Gauge) .Select(x => new GenericQuery.GaugeColumn(x.Name, x.Label, x.Description ?? x.Label, metricFactory, x.DefaultValue)) .ToArray(); var counterColumns = metricQuery.Columns .Where(x => x.ColumnUsage == ColumnUsage.Counter) .Select(x => new GenericQuery.CounterColumn(x.Name, x.Label, x.Description ?? x.Label, metricFactory)) .ToArray(); return(new GenericQuery(metricQuery.Name, metricQuery.Query, gaugeColumns, counterColumns, metricQuery.MillisecondTimeout)); default: break; } throw new Exception("Undefined QueryUsage."); }
/// <inheritdoc /> public Task WriteAsync(MetricQuery query, IMetricQueryResult result, CancellationToken cancellation = default) { MetricTags CreateTags(IDictionary <string, string> additionalLabels = null) { var labels = (IDictionary <string, string>) new Dictionary <string, string> { { "query", query.Name } }; if (query.Labels.Any()) { labels = labels.MergeDifference(query.Labels); } if (additionalLabels != null) { labels = labels.MergeDifference(additionalLabels); } return(new MetricTags(labels.Keys.ToArray(), labels.Values.ToArray())); } void WriteSucceed(SucceedMetricQueryResult succeedResult) { var tags = CreateTags(); _metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.Hits, tags, succeedResult.Hits); _metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.Duration, tags, succeedResult.Duration.TotalMilliseconds); if (succeedResult.ValueAggregations.Any()) { foreach (var aggregation in succeedResult.ValueAggregations) { if (!aggregation.Value.HasValue) { continue; } var aggregationTags = CreateTags(new Dictionary <string, string> { ["aggregation"] = aggregation.Key }); _metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.ValueAggregation, aggregationTags, aggregation.Value.Value); } } } void WriteFailure(FailureMetricQueryResult failureResult) { var tags = CreateTags(); if (failureResult.Timeout) { _metrics.Measure.Counter.Increment(MetricsRegistry.Counters.Timeouts, tags); } if (failureResult.Exception != null) { _metrics.Measure.Counter.Increment(MetricsRegistry.Counters.Exceptions, tags); } } void LogFailure(FailureMetricQueryResult failureResult) { if (failureResult.Timeout) { _logger.LogWarning("Query '{Query}' - evaluation timed out.", query.Name); } if (failureResult.Exception != null) { _logger.LogError(failureResult.Exception, "Query '{Query}' - evaluation failed with exception. Server error: {Error}", query.Name, JsonConvert.SerializeObject(failureResult.ServerError)); } } switch (result) { case SucceedMetricQueryResult succeed: WriteSucceed(succeed); break; case FailureMetricQueryResult failure: WriteFailure(failure); LogFailure(failure); break; } return(Task.CompletedTask); }