private static MetricsDeclaration InterpretYamlStream(YamlStream metricsDeclarationYamlStream)
{
var document = metricsDeclarationYamlStream.Documents.First();
var rootNode = (YamlMappingNode)document.RootNode;
AzureMetadata azureMetadata = null;
if (rootNode.Children.ContainsKey("azureMetadata"))
{
var azureMetadataNode = (YamlMappingNode)rootNode.Children[new YamlScalarNode("azureMetadata")];
var azureMetadataSerializer = new AzureMetadataDeserializer();
azureMetadata = azureMetadataSerializer.Deserialize(azureMetadataNode);
}
List <MetricDefinition> metrics = null;
if (rootNode.Children.ContainsKey("metrics"))
{
var metricsNode = (YamlSequenceNode)rootNode.Children[new YamlScalarNode("metrics")];
var metricsDeserializer = new MetricsDeserializer();
metrics = metricsDeserializer.Deserialize(metricsNode);
}
var metricsDeclaration = new MetricsDeclaration
{
AzureMetadata = azureMetadata,
Metrics = metrics
};
return(metricsDeclaration);
}
private static AzureMonitorClient ConfigureAzureMonitorClient(MetricsDeclaration metricsDeclaration, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
var azureCredentials = DetermineAzureCredentials(configuration);
var azureMetadata = metricsDeclaration.AzureMetadata;
var azureMonitorClient = new AzureMonitorClient(azureMetadata.Cloud, azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, runtimeMetricCollector, loggerFactory);
return(azureMonitorClient);
}
private void LogMetricsDeclaration(MetricsDeclaration metricsDeclaration)
{
metricsDeclaration.Metrics.ForEach(SanitizeStorageQueueDeclaration);
var serializer = YamlSerialization.CreateSerializer();
var rawDeclaration = serializer.Serialize(metricsDeclaration);
Logger.LogInformation("Following metrics configuration was configured:\n{Configuration}", rawDeclaration);
}
public static string Serialize(MetricsDeclaration metricsDeclaration)
{
Guard.NotNull(metricsDeclaration, nameof(metricsDeclaration));
var serializer = YamlSerialization.CreateSerializer();
var rawMetricsDeclaration = serializer.Serialize(metricsDeclaration);
return(rawMetricsDeclaration);
}
public string Build()
{
var metricsDeclaration = new MetricsDeclaration
{
AzureMetadata = _azureMetadata,
Metrics = _metrics
};
return(ConfigurationSerializer.Serialize(metricsDeclaration));
}
public string Build()
{
var metricsDeclaration = new MetricsDeclaration
{
AzureMetadata = _azureMetadata,
Metrics = _metrics
};
var configurationSerializer = new ConfigurationSerializer(NullLogger.Instance);
return(configurationSerializer.Serialize(metricsDeclaration));
}
public string Build()
{
var metricsDeclaration = new MetricsDeclaration
{
AzureMetadata = azureMetadata,
Metrics = metrics
};
var serializer = YamlSerialization.CreateSerializer();
return(serializer.Serialize(metricsDeclaration));
}
private static void ScheduleResourcesScraping(MetricsDeclaration metricsDeclaration,
MetricSinkWriter metricSinkWriter,
AzureMonitorClientFactory azureMonitorClientFactory,
IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector,
IMemoryCache resourceMetricDefinitionMemoryCache,
IScrapingMutex scrapingTaskMutex,
IConfiguration configuration,
IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration,
IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration,
ILoggerFactory loggerFactory,
ILogger <Startup> logger,
IServiceCollection services)
{
var jobName = GenerateResourceScrapingJobName(metricsDeclaration, logger);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
new ResourcesScrapingJob(jobName,
metricsDeclaration,
jobServices.GetService <ResourceDiscoveryRepository>(),
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClientFactory,
azureScrapingPrometheusMetricsCollector,
resourceMetricDefinitionMemoryCache,
scrapingTaskMutex,
configuration,
azureMonitorIntegrationConfiguration,
azureMonitorLoggingConfiguration,
loggerFactory,
jobServices.GetService <ILogger <ResourcesScrapingJob> >()),
schedulerOptions =>
{
schedulerOptions.CronSchedule = metricsDeclaration.Metrics[0].Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.AddUnobservedTaskExceptionHandler(s =>
{
return
((_, exceptionEventArgs) =>
{
var exceptionLogger = s.GetService <ILogger <BackgroundJobMonitor> >();
BackgroundJobMonitor.HandleException(jobName, exceptionEventArgs, exceptionLogger);
});
});
});
logger.LogInformation("Scheduled scraping job {JobName}.", jobName);
}
/// <summary>
/// Create a metrics scraping job for one or more resources, either enumerated specifically or
/// identified via resource definition groups. All metrics included are expected to have
/// the same scraping schedule.
/// </summary>
/// <param name="jobName">name of scheduled job</param>
/// <param name="metricsDeclaration">declaration of which metrics to collect from which resources</param>
/// <param name="resourceDiscoveryRepository">repository source for discovering resources via criteria</param>
/// <param name="metricSinkWriter">destination for metric reporting output</param>
/// <param name="metricScraperFactory">means of obtaining a metrics scraper for a particular type of resource</param>
/// <param name="azureMonitorClientFactory">means of obtaining a Azure Monitor client</param>
/// <param name="azureScrapingPrometheusMetricsCollector">metrics collector to write metrics to Prometheus</param>
/// <param name="resourceMetricDefinitionMemoryCache">cache of metric definitions by resource ID</param>
/// <param name="scrapingTaskMutex">semaphore used to limit concurrency of tasks if configured, or null for no limiting</param>
/// <param name="configuration">Promitor configuration</param>
/// <param name="azureMonitorIntegrationConfiguration">options for Azure Monitor integration</param>
/// <param name="azureMonitorLoggingConfiguration">options for Azure Monitor logging</param>
/// <param name="loggerFactory">means to obtain a logger</param>
/// <param name="logger">logger to use for scraping detail</param>
public ResourcesScrapingJob(string jobName,
MetricsDeclaration metricsDeclaration,
ResourceDiscoveryRepository resourceDiscoveryRepository,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClientFactory azureMonitorClientFactory,
IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector,
IMemoryCache resourceMetricDefinitionMemoryCache,
IScrapingMutex scrapingTaskMutex,
IConfiguration configuration,
IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration,
IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration,
ILoggerFactory loggerFactory,
ILogger <ResourcesScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(metricsDeclaration, nameof(metricsDeclaration));
Guard.NotNull(metricsDeclaration.AzureMetadata, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.AzureMetadata)}");
Guard.NotNull(metricsDeclaration.Metrics, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.Metrics)}");
Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory));
Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector));
Guard.NotNull(resourceMetricDefinitionMemoryCache, nameof(resourceMetricDefinitionMemoryCache));
Guard.NotNull(configuration, nameof(configuration));
Guard.NotNull(azureMonitorIntegrationConfiguration, nameof(azureMonitorIntegrationConfiguration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(loggerFactory, nameof(loggerFactory));
// all metrics must have the same scraping schedule or it is not possible for them to share the same job
VerifyAllMetricsHaveSameScrapingSchedule(metricsDeclaration);
_metricsDeclaration = metricsDeclaration;
_resourceDiscoveryRepository = resourceDiscoveryRepository;
_metricSinkWriter = metricSinkWriter;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClientFactory = azureMonitorClientFactory;
_azureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector;
_resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache;
_scrapingTaskMutex = scrapingTaskMutex;
_configuration = configuration;
_azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration;
_azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration;
_loggerFactory = loggerFactory;
}
private static void VerifyAllMetricsHaveSameScrapingSchedule(MetricsDeclaration metricsDeclaration)
{
if (metricsDeclaration.Metrics.Count > 1)
{
var metrics = metricsDeclaration.Metrics;
var commonScraping = metrics[0].Scraping;
for (var i = 1; i < metrics.Count; i++)
{
if (!Equals(commonScraping, metrics[i].Scraping))
{
throw new ArgumentException(
$"The \"{metrics[i].Scraping?.Schedule}\" scraping schedule for {nameof(metricsDeclaration)}.{nameof(metricsDeclaration.Metrics)}[{i}] does not share the common scraping schedule \"{commonScraping?.Schedule}\".");
}
}
}
}
public void YamlSerialization_SerializeAndDeserializeValidConfigForServiceBus_SucceedsWithIdenticalOutput()
{
// Arrange
var azureMetadata = GenerateBogusAzureMetadata();
var serviceBusMetricDefinition = GenerateBogusServiceBusMetricDefinition();
var scrapingConfiguration = new MetricsDeclaration
{
AzureMetadata = azureMetadata,
Metrics = new List <Core.Scraping.Configuration.Model.Metrics.MetricDefinition>
{
serviceBusMetricDefinition
}
};
// Act
var serializedConfiguration = ConfigurationSerializer.Serialize(scrapingConfiguration);
var deserializedConfiguration = ConfigurationSerializer.Deserialize(serializedConfiguration);
// Assert
Assert.NotNull(deserializedConfiguration);
Assert.NotNull(deserializedConfiguration.AzureMetadata);
Assert.Equal(azureMetadata.TenantId, deserializedConfiguration.AzureMetadata.TenantId);
Assert.Equal(azureMetadata.ResourceGroupName, deserializedConfiguration.AzureMetadata.ResourceGroupName);
Assert.Equal(azureMetadata.SubscriptionId, deserializedConfiguration.AzureMetadata.SubscriptionId);
Assert.NotNull(deserializedConfiguration.Metrics);
Assert.Single(deserializedConfiguration.Metrics);
var deserializedMetricDefinition = deserializedConfiguration.Metrics.FirstOrDefault();
Assert.NotNull(deserializedMetricDefinition);
Assert.Equal(serviceBusMetricDefinition.Name, deserializedMetricDefinition.Name);
Assert.Equal(serviceBusMetricDefinition.Description, deserializedMetricDefinition.Description);
Assert.Equal(serviceBusMetricDefinition.ResourceType, deserializedMetricDefinition.ResourceType);
var deserializedServiceBusMetricDefinition = deserializedMetricDefinition as ServiceBusQueueMetricDefinition;
Assert.NotNull(deserializedServiceBusMetricDefinition);
Assert.Equal(serviceBusMetricDefinition.Namespace, deserializedServiceBusMetricDefinition.Namespace);
Assert.Equal(serviceBusMetricDefinition.QueueName, deserializedServiceBusMetricDefinition.QueueName);
Assert.NotNull(deserializedMetricDefinition.AzureMetricConfiguration);
Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.MetricName, deserializedMetricDefinition.AzureMetricConfiguration.MetricName);
Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.Aggregation, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation);
}
private static ICollection <MetricsDeclaration> GroupMetricsByScrapingInterval(MetricsDeclaration allMetrics)
{
var metricsGroupings = new Dictionary <Scraping, MetricsDeclaration>();
foreach (var metric in allMetrics.Metrics)
{
if (!metricsGroupings.TryGetValue(metric.Scraping, out var metricsGroup))
{
metricsGroup = new MetricsDeclaration
{
AzureMetadata = allMetrics.AzureMetadata,
MetricDefaults = allMetrics.MetricDefaults
};
metricsGroupings.Add(metric.Scraping, metricsGroup);
}
metricsGroup.Metrics.Add(metric);
}
return(metricsGroupings.Values);
}
private static string GenerateResourceScrapingJobName(MetricsDeclaration metricsDeclaration, ILogger <Startup> logger)
{
string scheduleDescription;
try
{
scheduleDescription = CronExpressionDescriptor.ExpressionDescriptor.GetDescription(metricsDeclaration.Metrics[0].Scraping.Schedule);
scheduleDescription = scheduleDescription.Replace(", ", "_").Replace(",", "_").Replace(" ", "_");
}
catch (Exception ex)
{
var metricName = metricsDeclaration.Metrics[0].AzureMetricConfiguration.MetricName;
logger.LogError(ex, $"Failure to create job name for scraping resources for list of metrics which includes {metricName}.");
scheduleDescription = $"UnparsedScheduleForMetricsIncluding{metricName}";
}
var metricsCount = metricsDeclaration.Metrics.Count;
var uniqueness = Guid.NewGuid().ToString("N");
return($"Scraper-{metricsCount}Metrics-{scheduleDescription}-{uniqueness}");
}
private static void ScheduleResourceScraping(IServiceCollection services, IAzureResourceDefinition resource, MetricsDeclaration metrics, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, MetricDefinition metric, ILogger <Startup> logger)
{
var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? metrics.AzureMetadata.SubscriptionId : resource.SubscriptionId;
var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(metrics.AzureMetadata.Cloud, metrics.AzureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
var scrapeDefinition = metric.CreateScrapeDefinition(resource, metrics.AzureMetadata);
var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceScrapingJob(jobName, scrapeDefinition,
metricSinkWriter,
jobServices.GetService <IPrometheusMetricWriter>(),
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClient,
jobServices.GetService <ILogger <ResourceScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.GetUniqueName(), scrapeDefinition.PrometheusMetricDefinition?.Name);
}