private static void ScheduleResourceScraping(IAzureResourceDefinition resource, AzureMetadata azureMetadata, MetricDefinition metric, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services)
{
var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId;
var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(azureMetadata.Cloud, azureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
var scrapeDefinition = metric.CreateScrapeDefinition(resource, azureMetadata);
var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceScrapingJob(jobName, scrapeDefinition,
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClient,
jobServices.GetService <ILogger <ResourceScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.UniqueName, scrapeDefinition.PrometheusMetricDefinition?.Name);
}
private async Task GetDiscoveryGroupScrapeDefinitions(string resourceDiscoveryGroupName, MetricDefinition metricDefinition, ConcurrentBag <ScrapeDefinition <IAzureResourceDefinition> > scrapeDefinitions)
{
// this runs in a separate thread, must trap exceptions
try
{
Logger.LogInformation("Scraping resource collection {ResourceDiscoveryGroup}.", resourceDiscoveryGroupName);
var discoveredResources = await _resourceDiscoveryRepository.GetResourceDiscoveryGroupAsync(resourceDiscoveryGroupName);
if (discoveredResources == null)
{
Logger.LogWarning("Discovered no resources for resource collection {ResourceDiscoveryGroup}.", resourceDiscoveryGroupName);
return;
}
Logger.LogInformation("Discovered {ResourceCount} resources for resource collection {ResourceDiscoveryGroup}.", discoveredResources.Count, resourceDiscoveryGroupName);
foreach (var discoveredResource in discoveredResources)
{
Logger.LogDebug("Discovered resource {DiscoveredResource}.", discoveredResource);
var scrapeDefinition = metricDefinition.CreateScrapeDefinition(discoveredResource, _metricsDeclaration.AzureMetadata);
scrapeDefinitions.Add(scrapeDefinition);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed to discover resources for group {GroupName}.", resourceDiscoveryGroupName);
}
}
public void CreateScrapeDefinition_ResourceHasEmptyResourceGroupName_UsesGlobalName()
{
// Arrange
var resource = new ContainerInstanceResourceDefinition("subscription", string.Empty, "containerGroup");
var definition = new MetricDefinition(_prometheusMetricDefinition, new Promitor.Core.Scraping.Configuration.Model.Scraping(), new AzureMetricConfiguration(), ResourceType.ContainerInstance, new List <AzureResourceDefinition> {
resource
});
// Act
var scrapeDefinition = definition.CreateScrapeDefinition(resource, _azureMetadata);
// Assert
Assert.Equal(_azureMetadata.ResourceGroupName, scrapeDefinition.ResourceGroupName);
}
public void CreateScrapeDefinition_ResourceDoesNotSpecifySubscription_UsesGlobalName()
{
// Arrange
var resource = new ContainerInstanceResourceDefinition(null, "containerInstanceResourceGroup", "containerGroup");
var definition = new MetricDefinition(_prometheusMetricDefinition, new Promitor.Core.Scraping.Configuration.Model.Scraping(), new AzureMetricConfiguration(), ResourceType.ContainerInstance, new List <AzureResourceDefinition> {
resource
});
// Act
var scrapeDefinition = definition.CreateScrapeDefinition(resource, _azureMetadata);
// Assert
Assert.Equal(_azureMetadata.SubscriptionId, scrapeDefinition.SubscriptionId);
}
public void CreateScrapeDefinition_ResourceOverridesResourceGroupName_UsesOverriddenName()
{
// Arrange
var resource = new ContainerInstanceResourceDefinition(null, "containerInstanceResourceGroup", "containerGroup");
var definition = new MetricDefinition(_prometheusMetricDefinition, new Promitor.Core.Scraping.Configuration.Model.Scraping(), new AzureMetricConfiguration(), ResourceType.ContainerInstance, new List <AzureResourceDefinition> {
resource
});
// Act
var scrapeDefinition = definition.CreateScrapeDefinition(resource, _azureMetadata);
// Assert
Assert.Equal(resource.ResourceGroupName, scrapeDefinition.ResourceGroupName);
}
private async Task ScrapeResourceAsync(IAzureResourceDefinition discoveredResource, AzureMonitorClient azureMonitorClient)
{
try
{
var scrapingDefinition = _metricDefinition.CreateScrapeDefinition(discoveredResource, _azureMetadata);
var scraper = _metricScraperFactory.CreateScraper(scrapingDefinition.Resource.ResourceType, _metricSinkWriter, azureMonitorClient);
await scraper.ScrapeAsync(scrapingDefinition);
}
catch (Exception exception)
{
Logger.LogCritical(exception, "Failed to scrape resource collection {Resource}: {Exception}", discoveredResource.UniqueName, exception.Message);
}
}
private void GetResourceScrapeDefinition(IAzureResourceDefinition resourceDefinition, MetricDefinition metricDefinition, ConcurrentBag <ScrapeDefinition <IAzureResourceDefinition> > scrapeDefinitions)
{
var scrapeDefinition = metricDefinition.CreateScrapeDefinition(resourceDefinition, _metricsDeclaration.AzureMetadata);
scrapeDefinitions.Add(scrapeDefinition);
}