private async Task <IEnumerable <string> > GetIndicesAsync(MetricQuery query)
{
var indices = new List <string>();
var options = _optionsProvider.Value.ElasticSearch;
if (options.DefaultIndex != null)
{
indices.Add(string.Format(options.DefaultIndex, DateTime.UtcNow));
}
if (query.Indices.Any())
{
indices.AddRange(query.Indices.Select(i => string.Format(i, DateTime.UtcNow)));
}
if (indices.Count < 1)
{
throw new InvalidOperationException($"There's no indexes provided for query '{query.Name}', ensure your configuration is correct");
}
var existsTasks = indices.Select(i => _client.Indices.ExistsAsync(Indices.Index(i)));
var indexExists = await Task.WhenAll(existsTasks);
if (indexExists.All(r => !r.Exists))
{
throw new InvalidOperationException(
$"Unable to find any existing index for query '{query.Name}', ensure index format and name provided correctly");
}
return(indices);
}
public async Task EvaluateAsync(MetricQuery query, CancellationToken cancellation = default)
{
var options = _optionsProvider.Value;
var timeout = query.Timeout.GetValueOrDefault(options.Metrics.Evaluation.Timeout);
var interval = query.Interval.GetValueOrDefault(options.Metrics.Evaluation.Interval);
try
{
_logger.LogInformation("Starting evaluation of query '{Query}'", query.Name);
var result = await _queryExecutor.ExecuteAsync(query, timeout, cancellation);
await _metricsWriter.WriteAsync(query, result, cancellation);
}
catch (Exception e)
{
_logger.LogError(e, "Query '{Query}' evaluation failed with exception", query.Name);
}
finally
{
_logger.LogInformation("Query '{Query}' evaluated. Next evaluation on '{NextOccurence}'", query.Name, (DateTime.Now + interval));
// Re-schedule
_jobClient.Schedule <ScheduledMetricsEvaluator>(
e => e.EvaluateAsync(query, cancellation), interval);
}
}
public async Task API_MetricQueryTest()
{
var widget = new TestWidget();
var handler = new MetricQuery();
var request = MetricQueryRequest.Create(widget);
await handler.Handle(request, CancellationToken.None);
Assert.AreEqual(1, widget.Value);
}
/// <inheritdoc />
public async Task <IMetricQueryResult> ExecuteAsync(MetricQuery query, TimeSpan timeout, CancellationToken cancellation = default)
{
var result = query.Type switch
{
MetricQueryType.Default => ExecuteDefaultAsync((DefaultMetricQuery)query, timeout, cancellation),
MetricQueryType.Raw => ExecuteRawAsync((RawMetricQuery)query, timeout, cancellation),
_ => throw new InvalidOperationException($"Query type '{query.Type}' not supported")
};
return(await result);
}
public static IQuery GetSpecificQuery(Prometheus.MetricFactory metricFactory, MetricQuery metricQuery)
{
switch (metricQuery.QueryUsage)
{
case QueryUsage.Counter:
var labelColumns =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.CounterLabel)
.Select(x => new CounterGroupQuery.Column(x.Name, x.Order ?? 0, x.Label));
var valueColumn =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.Counter)
.Select(x => new CounterGroupQuery.Column(x.Name, x.Order ?? 0, x.Label))
.FirstOrDefault();
return(new CounterGroupQuery(metricQuery.Name, metricQuery.Description ?? string.Empty, metricQuery.Query, labelColumns, valueColumn, metricFactory, metricQuery.MillisecondTimeout));
case QueryUsage.Gauge:
var gaugeLabelColumns =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.GaugeLabel)
.Select(x => new GaugeGroupQuery.Column(x.Name, x.Order ?? 0, x.Label));
var gaugeValueColumn =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.Gauge)
.Select(x => new GaugeGroupQuery.Column(x.Name, x.Order ?? 0, x.Label))
.FirstOrDefault();
return(new GaugeGroupQuery(metricQuery.Name, metricQuery.Description ?? string.Empty, metricQuery.Query, gaugeLabelColumns, gaugeValueColumn, metricFactory, metricQuery.MillisecondTimeout));
case QueryUsage.Empty:
var gaugeColumns =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.Gauge)
.Select(x => new GenericQuery.GaugeColumn(x.Name, x.Label, x.Description ?? x.Label, metricFactory, x.DefaultValue))
.ToArray();
var counterColumns =
metricQuery.Columns
.Where(x => x.ColumnUsage == ColumnUsage.Counter)
.Select(x => new GenericQuery.CounterColumn(x.Name, x.Label, x.Description ?? x.Label, metricFactory))
.ToArray();
return(new GenericQuery(metricQuery.Name, metricQuery.Query, gaugeColumns, counterColumns, metricQuery.MillisecondTimeout));
default:
break;
}
throw new Exception("Undefined QueryUsage.");
}
/// <inheritdoc />
public Task WriteAsync(MetricQuery query, IMetricQueryResult result, CancellationToken cancellation = default)
{
MetricTags CreateTags(IDictionary <string, string> additionalLabels = null)
{
var labels = (IDictionary <string, string>) new Dictionary <string, string>
{
{ "query", query.Name }
};
if (query.Labels.Any())
{
labels = labels.MergeDifference(query.Labels);
}
if (additionalLabels != null)
{
labels = labels.MergeDifference(additionalLabels);
}
return(new MetricTags(labels.Keys.ToArray(), labels.Values.ToArray()));
}
void WriteSucceed(SucceedMetricQueryResult succeedResult)
{
var tags = CreateTags();
_metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.Hits, tags, succeedResult.Hits);
_metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.Duration, tags, succeedResult.Duration.TotalMilliseconds);
if (succeedResult.ValueAggregations.Any())
{
foreach (var aggregation in succeedResult.ValueAggregations)
{
if (!aggregation.Value.HasValue)
{
continue;
}
var aggregationTags = CreateTags(new Dictionary <string, string>
{
["aggregation"] = aggregation.Key
});
_metrics.Measure.Gauge.SetValue(MetricsRegistry.Gauges.ValueAggregation, aggregationTags, aggregation.Value.Value);
}
}
}
void WriteFailure(FailureMetricQueryResult failureResult)
{
var tags = CreateTags();
if (failureResult.Timeout)
{
_metrics.Measure.Counter.Increment(MetricsRegistry.Counters.Timeouts, tags);
}
if (failureResult.Exception != null)
{
_metrics.Measure.Counter.Increment(MetricsRegistry.Counters.Exceptions, tags);
}
}
void LogFailure(FailureMetricQueryResult failureResult)
{
if (failureResult.Timeout)
{
_logger.LogWarning("Query '{Query}' - evaluation timed out.", query.Name);
}
if (failureResult.Exception != null)
{
_logger.LogError(failureResult.Exception, "Query '{Query}' - evaluation failed with exception. Server error: {Error}",
query.Name, JsonConvert.SerializeObject(failureResult.ServerError));
}
}
switch (result)
{
case SucceedMetricQueryResult succeed:
WriteSucceed(succeed);
break;
case FailureMetricQueryResult failure:
WriteFailure(failure);
LogFailure(failure);
break;
}
return(Task.CompletedTask);
}
