public MetricScrapingJob(ScrapeDefinition <AzureResourceDefinition> metric,
IMetricsDeclarationProvider metricsDeclarationProvider,
IPrometheusMetricWriter prometheusMetricWriter,
IRuntimeMetricsCollector runtimeMetricsCollector,
MetricScraperFactory metricScraperFactory,
ILogger logger, IExceptionTracker exceptionTracker)
{
Guard.NotNull(metric, nameof(metric));
Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(runtimeMetricsCollector, nameof(runtimeMetricsCollector));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(logger, nameof(logger));
Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
_metric = metric;
_metricsDeclarationProvider = metricsDeclarationProvider;
_prometheusMetricWriter = prometheusMetricWriter;
_runtimeMetricsCollector = runtimeMetricsCollector;
_exceptionTracker = exceptionTracker;
_logger = logger;
_metricScraperFactory = metricScraperFactory;
ConfigureJob();
}
private void ReportScrapingOutcome(ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, bool isSuccessful)
{
// We reset all values, by default
double successfulMetricValue = 0;
double unsuccessfulMetricValue = 0;
// Based on the result, we reflect that in the metric
if (isSuccessful)
{
successfulMetricValue = 1;
}
else
{
unsuccessfulMetricValue = 1;
}
// Enrich with context
var labels = new Dictionary <string, string>
{
{ "metric_name", scrapeDefinition.PrometheusMetricDefinition.Name },
{ "resource_group", scrapeDefinition.ResourceGroupName },
{ "resource_name", scrapeDefinition.Resource.ResourceName },
{ "resource_type", scrapeDefinition.Resource.ResourceType.ToString() },
{ "subscription_id", scrapeDefinition.SubscriptionId }
};
// Report!
AzureScrapingPrometheusMetricsCollector.WriteGaugeMeasurement(RuntimeMetricNames.ScrapeSuccessful, ScrapeSuccessfulMetricDescription, successfulMetricValue, labels);
AzureScrapingPrometheusMetricsCollector.WriteGaugeMeasurement(RuntimeMetricNames.ScrapeError, ScrapeErrorMetricDescription, unsuccessfulMetricValue, labels);
}
private async Task ScrapeMetric(ScrapeDefinition <IAzureResourceDefinition> metricDefinitionDefinition)
{
Logger.LogInformation("Scraping {MetricName} for resource type {ResourceType}", metricDefinitionDefinition.PrometheusMetricDefinition.Name, metricDefinitionDefinition.Resource.ResourceType);
var scraper = _metricScraperFactory.CreateScraper(metricDefinitionDefinition.Resource.ResourceType, _metricSinkWriter, _azureMonitorClient);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, ScrapeDefinition <AzureResourceDefinition> metricDefinitionDefinition)
{
_logger.LogInformation("Scraping '{MetricName}' for resource type '{ResourceType}'", metricDefinitionDefinition.PrometheusMetricDefinition.Name, metricDefinitionDefinition.Resource.ResourceType);
var scraper = _metricScraperFactory.CreateScraper(metricDefinitionDefinition.Resource.ResourceType, azureMetadata, _prometheusMetricWriter, _runtimeMetricsCollector);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
public MetricScrapingJob(string jobName,
ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition,
MetricSinkWriter metricSinkWriter,
IPrometheusMetricWriter prometheusMetricWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClient azureMonitorClient,
ILogger <MetricScrapingJob> logger)
{
Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(logger, nameof(logger));
Name = jobName;
_metricScrapeDefinition = metricScrapeDefinition;
_prometheusMetricWriter = prometheusMetricWriter;
_metricSinkWriter = metricSinkWriter;
_logger = logger;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClient = azureMonitorClient;
}
/// <inheritdoc />
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, TResourceDefinition resource)
{
var slotName = DetermineSlotName(resource);
var resourceUri = BuildResourceUriWithoutDeploymentSlot(subscriptionId, scrapeDefinition, resource);
// Production slot should not be suffixed in resource URI
if (slotName != "production")
{
resourceUri += $"/slots/{slotName}";
}
return(resourceUri);
}
private void LogMeasuredMetrics(ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, ScrapeResult scrapedMetricResult, TimeSpan?aggregationInterval)
{
foreach (var measuredMetric in scrapedMetricResult.MetricValues)
{
if (measuredMetric.IsDimensional)
{
_logger.LogInformation("Found value {MetricValue} for metric {MetricName} with dimension {DimensionValue} as part of {DimensionName} dimension with aggregation interval {AggregationInterval}", measuredMetric.Value, scrapeDefinition.PrometheusMetricDefinition.Name, measuredMetric.DimensionValue, measuredMetric.DimensionName, aggregationInterval);
}
else
{
_logger.LogInformation("Found value {MetricValue} for metric {MetricName} with aggregation interval {AggregationInterval}", measuredMetric.Value, scrapeDefinition.PrometheusMetricDefinition.Name, aggregationInterval);
}
}
}
public async Task ScrapeAsync(ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition)
{
if (scrapeDefinition == null)
{
throw new ArgumentNullException(nameof(scrapeDefinition));
}
var aggregationInterval = scrapeDefinition.AzureMetricConfiguration?.Aggregation?.Interval;
if (aggregationInterval == null)
{
throw new ArgumentNullException(nameof(scrapeDefinition));
}
try
{
var castedMetricDefinition = scrapeDefinition.Resource as TResourceDefinition;
if (castedMetricDefinition == null)
{
throw new ArgumentException($"Could not cast metric definition of type {scrapeDefinition.Resource.ResourceType} to {typeof(TResourceDefinition)}. Payload: {JsonConvert.SerializeObject(scrapeDefinition)}");
}
var aggregationType = scrapeDefinition.AzureMetricConfiguration.Aggregation.Type;
var scrapedMetricResult = await ScrapeResourceAsync(
scrapeDefinition.SubscriptionId,
scrapeDefinition,
castedMetricDefinition,
aggregationType,
aggregationInterval.Value);
LogMeasuredMetrics(scrapeDefinition, scrapedMetricResult, aggregationInterval);
await _metricSinkWriter.ReportMetricAsync(scrapeDefinition.PrometheusMetricDefinition.Name, scrapeDefinition.PrometheusMetricDefinition.Description, scrapedMetricResult);
ReportScrapingOutcome(scrapeDefinition, isSuccessful: true);
}
catch (ErrorResponseException errorResponseException)
{
HandleErrorResponseException(errorResponseException, scrapeDefinition.PrometheusMetricDefinition.Name);
ReportScrapingOutcome(scrapeDefinition, isSuccessful: false);
}
catch (Exception exception)
{
Logger.LogCritical(exception, "Failed to scrape resource for metric '{MetricName}'", scrapeDefinition.PrometheusMetricDefinition.Name);
ReportScrapingOutcome(scrapeDefinition, isSuccessful: false);
}
}
private static string GenerateResourceScrapingJobName(ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, IAzureResourceDefinition resource)
{
var jobNameBuilder = new StringBuilder();
jobNameBuilder.Append(scrapeDefinition.SubscriptionId);
jobNameBuilder.Append("-");
jobNameBuilder.Append(scrapeDefinition.ResourceGroupName);
jobNameBuilder.Append("-");
jobNameBuilder.Append(scrapeDefinition.PrometheusMetricDefinition.Name);
jobNameBuilder.Append("-");
jobNameBuilder.Append(resource.UniqueName);
jobNameBuilder.Append("-");
jobNameBuilder.Append(Guid.NewGuid().ToString());
return(jobNameBuilder.ToString());
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, MySqlResourceDefinition resource)
{
var serverDefinition = (MySqlResourceDefinition)scrapeDefinition.Resource;
switch (serverDefinition.Type)
{
case MySqlServerType.Single:
return(string.Format(SingleServerResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.ServerName));
case MySqlServerType.Flexible:
return(string.Format(FlexibleServerResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.ServerName));
default:
throw new ArgumentOutOfRangeException($"Server type '{serverDefinition.Type}' is not supported for now");
}
}
public ResourceScrapingJob(string jobName,
ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClient azureMonitorClient,
ILogger <ResourceScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
_metricScrapeDefinition = metricScrapeDefinition;
_metricSinkWriter = metricSinkWriter;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClient = azureMonitorClient;
}
private async Task ScrapeMetric(ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition)
{
// this runs in a separate thread, must trap exceptions
try
{
var resourceSubscriptionId = !string.IsNullOrWhiteSpace(scrapeDefinition.Resource.SubscriptionId)
? scrapeDefinition.Resource.SubscriptionId
: _metricsDeclaration.AzureMetadata.SubscriptionId;
var azureMonitorClient = _azureMonitorClientFactory.CreateIfNotExists(_metricsDeclaration.AzureMetadata.Cloud, _metricsDeclaration.AzureMetadata.TenantId,
resourceSubscriptionId, _metricSinkWriter, _azureScrapingPrometheusMetricsCollector, _resourceMetricDefinitionMemoryCache, _configuration,
_azureMonitorIntegrationConfiguration, _azureMonitorLoggingConfiguration, _loggerFactory);
var scraper = _metricScraperFactory.CreateScraper(scrapeDefinition.Resource.ResourceType, _metricSinkWriter, _azureScrapingPrometheusMetricsCollector, azureMonitorClient);
await scraper.ScrapeAsync(scrapeDefinition);
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed to scrape metric {MetricName} for resource {ResourceName}.",
scrapeDefinition.PrometheusMetricDefinition.Name, scrapeDefinition.Resource.ResourceName);
}
}
public MetricScrapingJob(ScrapeDefinition <IAzureResourceDefinition> metric,
IMetricsDeclarationProvider metricsDeclarationProvider,
IPrometheusMetricWriter prometheusMetricWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClient azureMonitorClient,
ILogger <MetricScrapingJob> logger)
{
Guard.NotNull(metric, nameof(metric));
Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(logger, nameof(logger));
_metric = metric;
_metricsDeclarationProvider = metricsDeclarationProvider;
_prometheusMetricWriter = prometheusMetricWriter;
_logger = logger;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClient = azureMonitorClient;
ConfigureJob();
}
public async Task ScrapeAsync(ScrapeDefinition <AzureResourceDefinition> scrapeDefinition)
{
try
{
if (scrapeDefinition == null)
{
throw new ArgumentNullException(nameof(scrapeDefinition));
}
var castedMetricDefinition = scrapeDefinition.Resource as TResourceDefinition;
if (castedMetricDefinition == null)
{
throw new ArgumentException($"Could not cast metric definition of type '{scrapeDefinition.Resource.ResourceType}' to {typeof(TResourceDefinition)}. Payload: {JsonConvert.SerializeObject(scrapeDefinition)}");
}
var aggregationInterval = scrapeDefinition.AzureMetricConfiguration.Aggregation.Interval;
var aggregationType = scrapeDefinition.AzureMetricConfiguration.Aggregation.Type;
var scrapedMetricResult = await ScrapeResourceAsync(
AzureMetadata.SubscriptionId,
scrapeDefinition,
castedMetricDefinition,
aggregationType,
aggregationInterval.Value);
_logger.LogInformation("Found value '{MetricValue}' for metric '{MetricName}' with aggregation interval '{AggregationInterval}'", scrapedMetricResult, scrapeDefinition.PrometheusMetricDefinition.Name, aggregationInterval);
_prometheusMetricWriter.ReportMetric(scrapeDefinition.PrometheusMetricDefinition, scrapedMetricResult);
}
catch (ErrorResponseException errorResponseException)
{
HandleErrorResponseException(errorResponseException);
}
catch (Exception exception)
{
_exceptionTracker.Track(exception);
}
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, ContainerRegistryResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.RegistryName));
}
protected override async Task <ScrapeResult> ScrapeResourceAsync(string subscriptionId, ScrapeDefinition <AzureResourceDefinition> scrapeDefinition, CosmosDbResourceDefinition resource, AggregationType aggregationType, TimeSpan aggregationInterval)
{
var resourceUri = string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.DbName);
var metricName = scrapeDefinition.AzureMetricConfiguration.MetricName;
var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, aggregationType, aggregationInterval, resourceUri);
return(new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, resource.DbName, resourceUri, foundMetricValue));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, StorageAccountResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.AccountName));
}
/// <summary>
/// Scrapes the configured resource
/// </summary>
/// <param name="subscriptionId">Metric subscription Id</param>
/// <param name="scrapeDefinition">Contains all the information needed to scrape the resource.</param>
/// <param name="resourceDefinition">Contains the resource cast to the specific resource type.</param>
/// <param name="aggregationType">Aggregation for the metric to use</param>
/// <param name="aggregationInterval">Interval that is used to aggregate metrics</param>
protected abstract Task <ScrapeResult> ScrapeResourceAsync(
string subscriptionId,
ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition,
TResourceDefinition resourceDefinition,
AggregationType aggregationType,
TimeSpan aggregationInterval);
/// <inheritdoc />
protected override async Task <ScrapeResult> ScrapeResourceAsync(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, TResourceDefinition resourceDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
{
var resourceUri = BuildResourceUri(AzureMetadata.SubscriptionId, scrapeDefinition, resourceDefinition);
var metricFilter = DetermineMetricFilter(resourceDefinition);
var metricName = scrapeDefinition.AzureMetricConfiguration.MetricName;
var dimensionName = scrapeDefinition.AzureMetricConfiguration.Dimension?.Name;
var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, dimensionName, aggregationType, aggregationInterval, resourceUri, metricFilter);
var instanceName = resourceDefinition.GetResourceName();
var metricLabels = DetermineMetricLabels(resourceDefinition);
return(new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, instanceName, resourceUri, foundMetricValue, metricLabels));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, ServiceBusQueueResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.Namespace));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, NetworkInterfaceResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.NetworkInterfaceName));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, VirtualMachineResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.VirtualMachineName));
}
protected override async Task <ScrapeResult> ScrapeResourceAsync(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, StorageQueueResourceDefinition resource, AggregationType aggregationType, TimeSpan aggregationInterval)
{
Guard.NotNull(scrapeDefinition, nameof(scrapeDefinition));
Guard.NotNull(scrapeDefinition.AzureMetricConfiguration, nameof(scrapeDefinition.AzureMetricConfiguration));
Guard.NotNull(resource.SasToken, nameof(resource.SasToken));
Guard.NotNullOrEmpty(scrapeDefinition.AzureMetricConfiguration.MetricName, nameof(scrapeDefinition.AzureMetricConfiguration.MetricName));
var resourceUri = string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.AccountName);
var sasToken = resource.SasToken.GetSecretValue();
double foundMetricValue;
switch (scrapeDefinition.AzureMetricConfiguration.MetricName.ToLowerInvariant())
{
case AzureStorageConstants.Queues.Metrics.TimeSpentInQueue:
foundMetricValue = await _azureStorageQueueClient.GetQueueMessageTimeSpentInQueueAsync(resource.AccountName, resource.QueueName, sasToken);
break;
case AzureStorageConstants.Queues.Metrics.MessageCount:
foundMetricValue = await _azureStorageQueueClient.GetQueueMessageCountAsync(resource.AccountName, resource.QueueName, sasToken);
break;
default:
throw new InvalidMetricNameException(scrapeDefinition.AzureMetricConfiguration.MetricName, resource.ResourceType.ToString());
}
var labels = new Dictionary <string, string>
{
{ "queue_name", resource.QueueName }
};
var measuredMetrics = new List <MeasuredMetric>
{
MeasuredMetric.CreateWithoutDimension(foundMetricValue)
};
return(new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, resource.AccountName, resourceUri, measuredMetrics, labels));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, SqlDatabaseResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, AzureMetadata.SubscriptionId, scrapeDefinition.ResourceGroupName, resource.ServerName, resource.DatabaseName));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, SqlManagedInstanceResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.InstanceName));
}
/// <summary> /// Builds the URI of the resource to scrape /// </summary> /// <param name="subscriptionId">Subscription id in which the resource lives</param> /// <param name="scrapeDefinition">Contains all the information needed to scrape the resource.</param> /// <param name="resource">Contains the resource cast to the specific resource type.</param> /// <returns>Uri of Azure resource</returns> protected abstract string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, TResourceDefinition resource);
protected override async Task <ScrapeResult> ScrapeResourceAsync(string subscriptionId, ScrapeDefinition <AzureResourceDefinition> scrapeDefinition, ServiceBusQueueResourceDefinition resource, AggregationType aggregationType, TimeSpan aggregationInterval)
{
var resourceUri = string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.Namespace);
var filter = $"EntityName eq '{resource.QueueName}'";
var metricName = scrapeDefinition.AzureMetricConfiguration.MetricName;
var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, aggregationType, aggregationInterval, resourceUri, filter);
var labels = new Dictionary <string, string>
{
{ "entity_name", resource.QueueName }
};
return(new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, resource.Namespace, resourceUri, foundMetricValue, labels));
}
/// <inheritdoc />
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, KubernetesServiceResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.ClusterName));
}
protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, ExpressRouteCircuitResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.ExpressRouteCircuitName));
}
protected override string BuildResourceUriWithoutDeploymentSlot(string subscriptionId, ScrapeDefinition <IAzureResourceDefinition> scrapeDefinition, FunctionAppResourceDefinition resource)
{
return(string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.FunctionAppName));
}
