public ResourceDiscoveryGroupScrapingJob(string jobName, string resourceDiscoveryGroupName, AzureMetadata azureMetadata, MetricDefinition metricDefinition, ResourceDiscoveryRepository resourceDiscoveryRepository,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClientFactory azureMonitorClientFactory, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory,
ILogger <ResourceDiscoveryGroupScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNullOrWhitespace(resourceDiscoveryGroupName, nameof(resourceDiscoveryGroupName));
Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository));
Guard.NotNull(metricDefinition, nameof(metricDefinition));
Guard.NotNull(azureMetadata, nameof(azureMetadata));
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory));
Guard.NotNull(runtimeMetricCollector, nameof(runtimeMetricCollector));
Guard.NotNull(configuration, nameof(configuration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(loggerFactory, nameof(loggerFactory));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
ResourceDiscoveryGroupName = resourceDiscoveryGroupName;
_azureMetadata = azureMetadata;
_metricDefinition = metricDefinition;
_resourceDiscoveryRepository = resourceDiscoveryRepository;
_metricSinkWriter = metricSinkWriter;
_runtimeMetricCollector = runtimeMetricCollector;
_azureMonitorClientFactory = azureMonitorClientFactory;
_configuration = configuration;
_azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration;
_loggerFactory = loggerFactory;
_metricScraperFactory = metricScraperFactory;
}
public MetricScrapingJob(ScrapeDefinition <AzureResourceDefinition> metric,
IMetricsDeclarationProvider metricsDeclarationProvider,
IPrometheusMetricWriter prometheusMetricWriter,
IRuntimeMetricsCollector runtimeMetricsCollector,
MetricScraperFactory metricScraperFactory,
ILogger logger, IExceptionTracker exceptionTracker)
{
Guard.NotNull(metric, nameof(metric));
Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(runtimeMetricsCollector, nameof(runtimeMetricsCollector));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(logger, nameof(logger));
Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
_metric = metric;
_metricsDeclarationProvider = metricsDeclarationProvider;
_prometheusMetricWriter = prometheusMetricWriter;
_runtimeMetricsCollector = runtimeMetricsCollector;
_exceptionTracker = exceptionTracker;
_logger = logger;
_metricScraperFactory = metricScraperFactory;
ConfigureJob();
}
public MetricScrapingJob(MetricDefinition metric,
IMetricsDeclarationProvider metricsDeclarationProvider,
IRuntimeMetricsCollector runtimeMetricsCollector,
ILogger logger, IExceptionTracker exceptionTracker)
{
Guard.NotNull(metric, nameof(metric));
Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
Guard.NotNull(logger, nameof(logger));
_metric = metric;
_metricsDeclarationProvider = metricsDeclarationProvider;
_runtimeMetricsCollector = runtimeMetricsCollector;
_exceptionTracker = exceptionTracker;
_logger = logger;
ConfigureJob();
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param>
/// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param>
/// <param name="applicationId">Id of the application that is being used to interact with Azure Resource Manager</param>
/// <param name="metricsCollector">Metrics collector to write metrics to</param>
/// <param name="logger">Logger to write telemetry to</param>
public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, string applicationId, IRuntimeMetricsCollector metricsCollector, ILogger logger)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNullOrWhitespace(applicationId, nameof(applicationId));
Guard.NotNull(metricsCollector, nameof(metricsCollector));
Guard.NotNull(logger, nameof(logger));
_logger = logger;
_metricsCollector = metricsCollector;
_metricLabels = new Dictionary <string, string>
{
{ "tenant_id", tenantId },
{ "subscription_id", subscriptionId },
{ "app_id", applicationId }
};
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param>
/// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param>
/// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param>
/// <param name="metricSinkWriter">Metrics writer to all sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="logger">Logger to write telemetry to</param>
public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILogger logger)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(metricsCollector, nameof(metricsCollector));
Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo));
Guard.NotNull(logger, nameof(logger));
_logger = logger;
_metricSinkWriter = metricSinkWriter;
_metricsCollector = metricsCollector;
var id = DetermineApplicationId(azureAuthenticationInfo);
_metricLabels = new Dictionary <string, string>
{
{ "tenant_id", tenantId },
{ "subscription_id", subscriptionId },
{ "app_id", id },
};
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
/// <param name="logger">General logger</param>
/// <param name="exceptionTracker">Tracker used to log exceptions</param>
public static IScraper <MetricDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger, IExceptionTracker exceptionTracker)
{
var azureCredentials = DetermineAzureCredentials();
var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, azureCredentials, runtimeMetricsCollector, logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.Generic:
return(new GenericScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
default:
throw new ArgumentOutOfRangeException();
}
}
private static AzureMonitorClient ConfigureAzureMonitorClient(MetricsDeclaration metricsDeclaration, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
var azureCredentials = DetermineAzureCredentials(configuration);
var azureMetadata = metricsDeclaration.AzureMetadata;
var azureMonitorClient = new AzureMonitorClient(azureMetadata.Cloud, azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, runtimeMetricCollector, loggerFactory);
return(azureMonitorClient);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureCloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param>
/// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
/// <param name="logger">Logger to use during interaction with Azure Monitor</param>
public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNullOrWhitespace(applicationId, nameof(applicationId));
Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret));
var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud);
var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, runtimeMetricsCollector, logger);
_authenticatedAzureSubscription = Azure.Configure().WithDelegatingHandler(monitorHandler).Authenticate(credentials).WithSubscription(subscriptionId);
_logger = logger;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureCloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param>
/// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILoggerFactory loggerFactory)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNullOrWhitespace(applicationId, nameof(applicationId));
Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
_logger = loggerFactory.CreateLogger <AzureMonitorClient>();
_authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, applicationId, applicationSecret, azureMonitorLoggingConfiguration, loggerFactory, metricSinkWriter, metricsCollector);
}
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector)
{
var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud);
var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>();
var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, metricSinkWriter, metricsCollector, throttlingLogger);
var azureClientConfiguration = Azure.Configure()
.WithDelegatingHandler(monitorHandler);
var azureMonitorLogging = azureMonitorLoggingConfiguration.Value;
if (azureMonitorLogging.IsEnabled)
{
var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>();
ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger));
ServiceClientTracing.IsEnabled = true;
azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler())
.WithLogLevel(azureMonitorLogging.InformationLevel);
}
return(azureClientConfiguration
.Authenticate(credentials)
.WithSubscription(subscriptionId));
}
private static AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger)
{
var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, logger);
return(azureMonitorClient);
}
private AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, IRuntimeMetricsCollector runtimeMetricsCollector)
{
var azureCredentials = DetermineAzureCredentials();
var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, _logger);
return(azureMonitorClient);
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
public IScraper <AzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector)
{
var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, runtimeMetricsCollector);
var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, prometheusMetricWriter, _logger, _exceptionTracker);
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.AzureSqlDatabase:
return(new AzureSqlDatabaseScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Provides an Azure Monitor client
/// </summary>
/// <param name="cloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="configuration">Configuration of Promitor</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
if (_azureMonitorClients.ContainsKey(subscriptionId))
{
return(_azureMonitorClients[subscriptionId]);
}
var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, metricsCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
_azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient);
return(azureMonitorClient);
}
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
var azureCredentials = DetermineAzureCredentials(configuration);
var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, metricSinkWriter, metricsCollector, loggerFactory);
return(azureMonitorClient);
}
private static void ScheduleResourceScraping(IAzureResourceDefinition resource, AzureMetadata azureMetadata, MetricDefinition metric, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services)
{
var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId;
var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(azureMetadata.Cloud, azureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
var scrapeDefinition = metric.CreateScrapeDefinition(resource, azureMetadata);
var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceScrapingJob(jobName, scrapeDefinition,
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClient,
jobServices.GetService <ILogger <ResourceScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.UniqueName, scrapeDefinition.PrometheusMetricDefinition?.Name);
}
private static void ScheduleResourceDiscoveryGroupScraping(AzureResourceDiscoveryGroup resourceDiscoveryGroup, AzureMetadata azureMetadata, MetricDefinition metricDefinition, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services)
{
var jobName = GenerateResourceDiscoveryGroupScrapingJobName(metricDefinition.PrometheusMetricDefinition?.Name, resourceDiscoveryGroup.Name);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceDiscoveryGroupScrapingJob(jobName, resourceDiscoveryGroup.Name, azureMetadata, metricDefinition,
jobServices.GetService <ResourceDiscoveryRepository>(),
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClientFactory,
runtimeMetricCollector,
configuration,
azureMonitorLoggingConfiguration,
loggerFactory,
jobServices.GetService <ILogger <ResourceDiscoveryGroupScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = metricDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource collection {ResourceDiscoveryGroup} which will be reported as metric {MetricName}", jobName, resourceDiscoveryGroup.Name, metricDefinition.PrometheusMetricDefinition?.Name);
}
