public MetricScrapingJob(string jobName,
ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition,
MetricSinkWriter metricSinkWriter,
IPrometheusMetricWriter prometheusMetricWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClient azureMonitorClient,
ILogger <MetricScrapingJob> logger)
{
Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(logger, nameof(logger));
Name = jobName;
_metricScrapeDefinition = metricScrapeDefinition;
_prometheusMetricWriter = prometheusMetricWriter;
_metricSinkWriter = metricSinkWriter;
_logger = logger;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClient = azureMonitorClient;
}
public ResourceDiscoveryGroupScrapingJob(string jobName, string resourceDiscoveryGroupName, AzureMetadata azureMetadata, MetricDefinition metricDefinition, ResourceDiscoveryRepository resourceDiscoveryRepository,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClientFactory azureMonitorClientFactory, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory,
ILogger <ResourceDiscoveryGroupScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNullOrWhitespace(resourceDiscoveryGroupName, nameof(resourceDiscoveryGroupName));
Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository));
Guard.NotNull(metricDefinition, nameof(metricDefinition));
Guard.NotNull(azureMetadata, nameof(azureMetadata));
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory));
Guard.NotNull(runtimeMetricCollector, nameof(runtimeMetricCollector));
Guard.NotNull(configuration, nameof(configuration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(loggerFactory, nameof(loggerFactory));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
ResourceDiscoveryGroupName = resourceDiscoveryGroupName;
_azureMetadata = azureMetadata;
_metricDefinition = metricDefinition;
_resourceDiscoveryRepository = resourceDiscoveryRepository;
_metricSinkWriter = metricSinkWriter;
_runtimeMetricCollector = runtimeMetricCollector;
_azureMonitorClientFactory = azureMonitorClientFactory;
_configuration = configuration;
_azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration;
_loggerFactory = loggerFactory;
_metricScraperFactory = metricScraperFactory;
}
/// <summary>
/// Constructor
/// </summary>
protected Scraper(ScraperConfiguration scraperConfiguration)
{
Guard.NotNull(scraperConfiguration, nameof(scraperConfiguration));
_logger = scraperConfiguration.Logger;
_metricSinkWriter = scraperConfiguration.MetricSinkWriter;
AzureMonitorClient = scraperConfiguration.AzureMonitorClient;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="logger">General logger</param>
public ScraperConfiguration(AzureMonitorClient azureMonitorClient, MetricSinkWriter metricSinkWriter, ILogger logger)
{
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(logger, nameof(logger));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
AzureMonitorClient = azureMonitorClient;
Logger = logger;
MetricSinkWriter = metricSinkWriter;
}
/// <summary>
/// Constructor
/// </summary>
protected Scraper(ScraperConfiguration scraperConfiguration)
{
Guard.NotNull(scraperConfiguration, nameof(scraperConfiguration));
_metricSinkWriter = scraperConfiguration.MetricSinkWriter;
Logger = scraperConfiguration.Logger;
AzureMonitorClient = scraperConfiguration.AzureMonitorClient;
AzureScrapingPrometheusMetricsCollector = scraperConfiguration.AzureScrapingPrometheusMetricsCollector;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="azureScrapingPrometheusMetricsCollector">Collector to send metrics related to the runtime</param>
/// <param name="logger">General logger</param>
public ScraperConfiguration(AzureMonitorClient azureMonitorClient, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, ILogger logger)
{
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(logger, nameof(logger));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector));
AzureMonitorClient = azureMonitorClient;
Logger = logger;
MetricSinkWriter = metricSinkWriter;
AzureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector;
}
public async Task ReportMetricAsync_WriteToNoSinks_Succeeds()
{
// Arrange
var metricName = BogusGenerator.Name.FirstName();
var metricDescription = BogusGenerator.Lorem.Sentence();
var metricValue = BogusGenerator.Random.Double();
var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink>(), NullLogger <MetricSinkWriter> .Instance);
// Act & Assert
await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult);
}
public async Task ReportMetricAsync_WriteToOneSinkWithoutMetricDescription_Succeeds(string metricDescription)
{
// Arrange
var metricName = BogusGenerator.Name.FirstName();
var metricValue = BogusGenerator.Random.Double();
var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
var metricSink = new Mock <IMetricSink>();
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
metricSink.Object
}, NullLogger <MetricSinkWriter> .Instance);
// Act & Assert
await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult);
}
private static void ScheduleResourcesScraping(MetricsDeclaration metricsDeclaration,
MetricSinkWriter metricSinkWriter,
AzureMonitorClientFactory azureMonitorClientFactory,
IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector,
IMemoryCache resourceMetricDefinitionMemoryCache,
IScrapingMutex scrapingTaskMutex,
IConfiguration configuration,
IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration,
IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration,
ILoggerFactory loggerFactory,
ILogger <Startup> logger,
IServiceCollection services)
{
var jobName = GenerateResourceScrapingJobName(metricsDeclaration, logger);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
new ResourcesScrapingJob(jobName,
metricsDeclaration,
jobServices.GetService <ResourceDiscoveryRepository>(),
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClientFactory,
azureScrapingPrometheusMetricsCollector,
resourceMetricDefinitionMemoryCache,
scrapingTaskMutex,
configuration,
azureMonitorIntegrationConfiguration,
azureMonitorLoggingConfiguration,
loggerFactory,
jobServices.GetService <ILogger <ResourcesScrapingJob> >()),
schedulerOptions =>
{
schedulerOptions.CronSchedule = metricsDeclaration.Metrics[0].Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.AddUnobservedTaskExceptionHandler(s =>
{
return
((_, exceptionEventArgs) =>
{
var exceptionLogger = s.GetService <ILogger <BackgroundJobMonitor> >();
BackgroundJobMonitor.HandleException(jobName, exceptionEventArgs, exceptionLogger);
});
});
});
logger.LogInformation("Scheduled scraping job {JobName}.", jobName);
}
public async Task ReportMetricAsync_WriteToOneSinkWithoutMetricName_ThrowsException(string metricName)
{
// Arrange
var metricDescription = _bogus.Lorem.Sentence();
var metricValue = _bogus.Random.Double();
var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
var metricSink = new Mock <IMetricSink>();
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
metricSink.Object
}, NullLogger <MetricSinkWriter> .Instance);
// Act & Assert
// ReSharper disable once ExpressionIsAlwaysNull
await Assert.ThrowsAsync <ArgumentException>(() => metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult));
}
public async Task ReportMetricAsync_WriteToOneSinkWithoutScrapeResult_ThrowsException()
{
// Arrange
var metricName = BogusGenerator.Name.FirstName();
var metricDescription = BogusGenerator.Lorem.Sentence();
ScrapeResult scrapeResult = null;
var metricSink = new Mock <IMetricSink>();
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
metricSink.Object
}, NullLogger <MetricSinkWriter> .Instance);
// Act & Assert
// ReSharper disable once ExpressionIsAlwaysNull
await Assert.ThrowsAsync <ArgumentNullException>(() => metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult));
}
/// <summary>
/// Create a metrics scraping job for one or more resources, either enumerated specifically or
/// identified via resource definition groups. All metrics included are expected to have
/// the same scraping schedule.
/// </summary>
/// <param name="jobName">name of scheduled job</param>
/// <param name="metricsDeclaration">declaration of which metrics to collect from which resources</param>
/// <param name="resourceDiscoveryRepository">repository source for discovering resources via criteria</param>
/// <param name="metricSinkWriter">destination for metric reporting output</param>
/// <param name="metricScraperFactory">means of obtaining a metrics scraper for a particular type of resource</param>
/// <param name="azureMonitorClientFactory">means of obtaining a Azure Monitor client</param>
/// <param name="azureScrapingPrometheusMetricsCollector">metrics collector to write metrics to Prometheus</param>
/// <param name="resourceMetricDefinitionMemoryCache">cache of metric definitions by resource ID</param>
/// <param name="scrapingTaskMutex">semaphore used to limit concurrency of tasks if configured, or null for no limiting</param>
/// <param name="configuration">Promitor configuration</param>
/// <param name="azureMonitorIntegrationConfiguration">options for Azure Monitor integration</param>
/// <param name="azureMonitorLoggingConfiguration">options for Azure Monitor logging</param>
/// <param name="loggerFactory">means to obtain a logger</param>
/// <param name="logger">logger to use for scraping detail</param>
public ResourcesScrapingJob(string jobName,
MetricsDeclaration metricsDeclaration,
ResourceDiscoveryRepository resourceDiscoveryRepository,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClientFactory azureMonitorClientFactory,
IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector,
IMemoryCache resourceMetricDefinitionMemoryCache,
IScrapingMutex scrapingTaskMutex,
IConfiguration configuration,
IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration,
IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration,
ILoggerFactory loggerFactory,
ILogger <ResourcesScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(metricsDeclaration, nameof(metricsDeclaration));
Guard.NotNull(metricsDeclaration.AzureMetadata, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.AzureMetadata)}");
Guard.NotNull(metricsDeclaration.Metrics, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.Metrics)}");
Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory));
Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector));
Guard.NotNull(resourceMetricDefinitionMemoryCache, nameof(resourceMetricDefinitionMemoryCache));
Guard.NotNull(configuration, nameof(configuration));
Guard.NotNull(azureMonitorIntegrationConfiguration, nameof(azureMonitorIntegrationConfiguration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(loggerFactory, nameof(loggerFactory));
// all metrics must have the same scraping schedule or it is not possible for them to share the same job
VerifyAllMetricsHaveSameScrapingSchedule(metricsDeclaration);
_metricsDeclaration = metricsDeclaration;
_resourceDiscoveryRepository = resourceDiscoveryRepository;
_metricSinkWriter = metricSinkWriter;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClientFactory = azureMonitorClientFactory;
_azureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector;
_resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache;
_scrapingTaskMutex = scrapingTaskMutex;
_configuration = configuration;
_azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration;
_azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration;
_loggerFactory = loggerFactory;
}
public async Task ReportMetricAsync_WriteToOneSink_Succeeds()
{
// Arrange
var metricName = BogusGenerator.Name.FirstName();
var metricDescription = BogusGenerator.Lorem.Sentence();
var metricValue = BogusGenerator.Random.Double();
var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
var metricSink = new Mock <IMetricSink>();
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
metricSink.Object
}, NullLogger <MetricSinkWriter> .Instance);
// Act
await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult);
// Assert
metricSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, scrapeResult), Times.Once());
}
public ResourceScrapingJob(string jobName,
ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClient azureMonitorClient,
ILogger <ResourceScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
_metricScrapeDefinition = metricScrapeDefinition;
_metricSinkWriter = metricSinkWriter;
_metricScraperFactory = metricScraperFactory;
_azureMonitorClient = azureMonitorClient;
}
public async Task ReportMetricAsync_WriteToStatsDSink_Succeeds()
{
// Arrange
var metricName = _bogus.Name.FirstName();
var metricDescription = _bogus.Lorem.Sentence();
var metricValue = _bogus.Random.Double();
var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
var statsDPublisherMock = new Mock <IStatsDPublisher>();
var statsdMetricSink = new StatsdMetricSink(statsDPublisherMock.Object, NullLogger <StatsdMetricSink> .Instance);
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
statsdMetricSink
}, NullLogger <MetricSinkWriter> .Instance);
// Act
await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult);
// Assert
statsDPublisherMock.Verify(mock => mock.Gauge(metricValue, metricName), Times.Once());
}
public async Task ReportMetricAsync_WriteToMultipleSinks_Succeeds()
{
// Arrange
var metricName = _bogus.Name.FirstName();
var metricDescription = _bogus.Lorem.Sentence();
var metricValue = _bogus.Random.Double();
var scrapeResult = GenerateScrapeResult(metricValue);
var firstSink = new Mock <IMetricSink>();
var secondSink = new Mock <IMetricSink>();
var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> {
firstSink.Object, secondSink.Object
}, NullLogger <MetricSinkWriter> .Instance);
// Act
await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult);
// Assert
firstSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, It.IsAny <MeasuredMetric>()), Times.Once());
secondSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, It.IsAny <MeasuredMetric>()), Times.Once());
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param>
/// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param>
/// <param name="applicationId">Id of the application that is being used to interact with Azure Resource Manager</param>
/// <param name="metricSinkWriter">Metrics writer to all sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="logger">Logger to write telemetry to</param>
public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, string applicationId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILogger logger)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNullOrWhitespace(applicationId, nameof(applicationId));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(metricsCollector, nameof(metricsCollector));
Guard.NotNull(logger, nameof(logger));
_logger = logger;
_metricSinkWriter = metricSinkWriter;
_metricsCollector = metricsCollector;
_metricLabels = new Dictionary <string, string>
{
{ "tenant_id", tenantId },
{ "subscription_id", subscriptionId },
{ "app_id", applicationId }
};
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param>
/// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param>
/// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param>
/// <param name="metricSinkWriter">Metrics writer to all sinks</param>
/// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="logger">Logger to write telemetry to</param>
public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, ILogger logger)
: base(azureScrapingPrometheusMetricsCollector, logger)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector));
Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo));
_metricSinkWriter = metricSinkWriter;
_azureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector;
var id = DetermineApplicationId(azureAuthenticationInfo);
_metricLabels = new Dictionary <string, string>
{
{ "tenant_id", tenantId },
{ "subscription_id", subscriptionId },
{ "app_id", id },
};
}
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration)
{
var credentials = AzureAuthenticationFactory.CreateAzureAuthentication(azureCloud, tenantId, azureAuthenticationInfo, _azureCredentialsFactory);
var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>();
var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, azureAuthenticationInfo, metricSinkWriter, azureScrapingPrometheusMetricsCollector, throttlingLogger);
var azureClientConfiguration = Microsoft.Azure.Management.Fluent.Azure.Configure()
.WithDelegatingHandler(monitorHandler);
var azureMonitorLogging = azureMonitorLoggingConfiguration.Value;
if (azureMonitorLogging.IsEnabled)
{
var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>();
ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger));
ServiceClientTracing.IsEnabled = true;
azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler())
.WithLogLevel(azureMonitorLogging.InformationLevel);
}
return(azureClientConfiguration
.Authenticate(credentials)
.WithSubscription(subscriptionId));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureCloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="resourceMetricDefinitionMemoryCache">Memory cache to store items in for performance optimizations</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
/// <param name="azureMonitorIntegrationConfiguration">Options for Azure Monitor logging</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor integration</param>
public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, ILoggerFactory loggerFactory, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo));
Guard.NotNull(azureMonitorIntegrationConfiguration, nameof(azureMonitorIntegrationConfiguration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(resourceMetricDefinitionMemoryCache, nameof(resourceMetricDefinitionMemoryCache));
_resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache;
_azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration;
_logger = loggerFactory.CreateLogger <AzureMonitorClient>();
_authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, azureAuthenticationInfo, loggerFactory, metricSinkWriter, azureScrapingPrometheusMetricsCollector, azureMonitorLoggingConfiguration);
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="metricSinkWriter">Writer to send metrics to all sinks</param>
/// <param name="azureScrapingPrometheusMetricsCollector">Collector to send metrics related to the runtime</param>
/// <param name="azureMonitorClient">Client to interact with Azure Monitor</param>
public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, AzureMonitorClient azureMonitorClient)
{
var scraperConfiguration = new ScraperConfiguration(azureMonitorClient, metricSinkWriter, azureScrapingPrometheusMetricsCollector, _logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ApiManagement:
return(new ApiManagementScraper(scraperConfiguration));
case ResourceType.ApplicationGateway:
return(new ApplicationGatewayScraper(scraperConfiguration));
case ResourceType.ApplicationInsights:
return(new ApplicationInsightsScraper(scraperConfiguration));
case ResourceType.AppPlan:
return(new AppPlanScraper(scraperConfiguration));
case ResourceType.AutomationAccount:
return(new AutomationAccountScraper(scraperConfiguration));
case ResourceType.BlobStorage:
return(new BlobStorageScraper(scraperConfiguration));
case ResourceType.Cdn:
return(new CdnScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.DataFactory:
return(new DataFactoryScraper(scraperConfiguration));
case ResourceType.DataShare:
return(new DataShareScraper(scraperConfiguration));
case ResourceType.DeviceProvisioningService:
return(new DeviceProvisioningServiceScraper(scraperConfiguration));
case ResourceType.EventHubs:
return(new EventHubsScraper(scraperConfiguration));
case ResourceType.ExpressRouteCircuit:
return(new ExpressRouteCircuitScraper(scraperConfiguration));
case ResourceType.FileStorage:
return(new FileStorageScraper(scraperConfiguration));
case ResourceType.FrontDoor:
return(new FrontDoorScraper(scraperConfiguration));
case ResourceType.FunctionApp:
return(new FunctionAppScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.IoTHub:
return(new IoTHubScraper(scraperConfiguration));
case ResourceType.KeyVault:
return(new KeyVaultScraper(scraperConfiguration));
case ResourceType.KubernetesService:
return(new KubernetesServiceScraper(scraperConfiguration));
case ResourceType.LoadBalancer:
return(new LoadBalancerScraper(scraperConfiguration));
case ResourceType.LogicApp:
return(new LogicAppScraper(scraperConfiguration));
case ResourceType.MariaDb:
return(new MariaDbScraper(scraperConfiguration));
case ResourceType.MonitorAutoscale:
return(new MonitorAutoscaleScraper(scraperConfiguration));
case ResourceType.MySql:
return(new MySqlScraper(scraperConfiguration));
case ResourceType.NetworkGateway:
return(new NetworkGatewayScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.RedisEnterpriseCache:
return(new RedisEnterpriseCacheScraper(scraperConfiguration));
case ResourceType.ServiceBusNamespace:
return(new ServiceBusNamespaceScraper(scraperConfiguration));
case ResourceType.SqlDatabase:
return(new SqlDatabaseScraper(scraperConfiguration));
case ResourceType.SqlElasticPool:
return(new SqlElasticPoolScraper(scraperConfiguration));
case ResourceType.SqlManagedInstance:
return(new SqlManagedInstanceScraper(scraperConfiguration));
case ResourceType.SqlServer:
return(new SqlServerScraper(scraperConfiguration));
case ResourceType.StorageAccount:
return(new StorageAccountScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.SynapseApacheSparkPool:
return(new SynapseApacheSparkPoolScraper(scraperConfiguration));
case ResourceType.SynapseSqlPool:
return(new SynapseSqlPoolScraper(scraperConfiguration));
case ResourceType.SynapseWorkspace:
return(new SynapseWorkspaceScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.VirtualMachineScaleSet:
return(new VirtualMachineScaleSetScraper(scraperConfiguration));
case ResourceType.VirtualNetwork:
return(new VirtualNetworkScraper(scraperConfiguration));
case ResourceType.WebApp:
return(new WebAppScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException(nameof(metricDefinitionResourceType), metricDefinitionResourceType, "Matching scraper not found");
}
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="metricSinkWriter">Writer to send metrics to all sinks</param>
/// <param name="azureMonitorClient">Client to interact with Azure Monitor</param>
public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, MetricSinkWriter metricSinkWriter, AzureMonitorClient azureMonitorClient)
{
var scraperConfiguration = new ScraperConfiguration(azureMonitorClient, metricSinkWriter, _logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ApiManagement:
return(new ApiManagementScraper(scraperConfiguration));
case ResourceType.ApplicationGateway:
return(new ApplicationGatewayScraper(scraperConfiguration));
case ResourceType.AppPlan:
return(new AppPlanScraper(scraperConfiguration));
case ResourceType.BlobStorage:
return(new BlobStorageScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.DeviceProvisioningService:
return(new DeviceProvisioningServiceScraper(scraperConfiguration));
case ResourceType.EventHubs:
return(new EventHubsScraper(scraperConfiguration));
case ResourceType.ExpressRouteCircuit:
return(new ExpressRouteCircuitScraper(scraperConfiguration));
case ResourceType.FileStorage:
return(new FileStorageScraper(scraperConfiguration));
case ResourceType.FunctionApp:
return(new FunctionAppScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.IoTHub:
return(new IoTHubScraper(scraperConfiguration));
case ResourceType.KeyVault:
return(new KeyVaultScraper(scraperConfiguration));
case ResourceType.NetworkGateway:
return(new NetworkGatewayScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.LogicApp:
return(new LogicAppScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(scraperConfiguration));
case ResourceType.SqlDatabase:
return(new SqlDatabaseScraper(scraperConfiguration));
case ResourceType.SqlManagedInstance:
return(new SqlManagedInstanceScraper(scraperConfiguration));
case ResourceType.SqlServer:
return(new SqlServerScraper(scraperConfiguration));
case ResourceType.StorageAccount:
return(new StorageAccountScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.VirtualMachineScaleSet:
return(new VirtualMachineScaleSetScraper(scraperConfiguration));
case ResourceType.WebApp:
return(new WebAppScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException();
}
}
private static void ScheduleResourceDiscoveryGroupScraping(AzureResourceDiscoveryGroup resourceDiscoveryGroup, AzureMetadata azureMetadata, MetricDefinition metricDefinition, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services)
{
var jobName = GenerateResourceDiscoveryGroupScrapingJobName(metricDefinition.PrometheusMetricDefinition?.Name, resourceDiscoveryGroup.Name);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceDiscoveryGroupScrapingJob(jobName, resourceDiscoveryGroup.Name, azureMetadata, metricDefinition,
jobServices.GetService <ResourceDiscoveryRepository>(),
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClientFactory,
runtimeMetricCollector,
configuration,
azureMonitorLoggingConfiguration,
loggerFactory,
jobServices.GetService <ILogger <ResourceDiscoveryGroupScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = metricDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource collection {ResourceDiscoveryGroup} which will be reported as metric {MetricName}", jobName, resourceDiscoveryGroup.Name, metricDefinition.PrometheusMetricDefinition?.Name);
}
private static void ScheduleResourceScraping(IAzureResourceDefinition resource, AzureMetadata azureMetadata, MetricDefinition metric, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services)
{
var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId;
var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(azureMetadata.Cloud, azureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
var scrapeDefinition = metric.CreateScrapeDefinition(resource, azureMetadata);
var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource);
services.AddScheduler(builder =>
{
builder.AddJob(jobServices =>
{
return(new ResourceScrapingJob(jobName, scrapeDefinition,
metricSinkWriter,
jobServices.GetService <MetricScraperFactory>(),
azureMonitorClient,
jobServices.GetService <ILogger <ResourceScrapingJob> >()));
}, schedulerOptions =>
{
schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule;
schedulerOptions.RunImmediately = true;
},
jobName: jobName);
builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services);
});
logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.UniqueName, scrapeDefinition.PrometheusMetricDefinition?.Name);
}
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
var azureCredentials = AzureAuthenticationFactory.GetConfiguredAzureAuthentication(configuration);
var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials, metricSinkWriter, azureScrapingPrometheusMetricsCollector, resourceMetricDefinitionMemoryCache, loggerFactory, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration);
return(azureMonitorClient);
}
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector)
{
var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud);
var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>();
var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, metricSinkWriter, metricsCollector, throttlingLogger);
var azureClientConfiguration = Azure.Configure()
.WithDelegatingHandler(monitorHandler);
var azureMonitorLogging = azureMonitorLoggingConfiguration.Value;
if (azureMonitorLogging.IsEnabled)
{
var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>();
ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger));
ServiceClientTracing.IsEnabled = true;
azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler())
.WithLogLevel(azureMonitorLogging.InformationLevel);
}
return(azureClientConfiguration
.Authenticate(credentials)
.WithSubscription(subscriptionId));
}
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
var azureCredentials = DetermineAzureCredentials(configuration);
var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, metricSinkWriter, metricsCollector, loggerFactory);
return(azureMonitorClient);
}
/// <summary>
/// Provides an Azure Monitor client
/// </summary>
/// <param name="cloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="configuration">Configuration of Promitor</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
if (_azureMonitorClients.ContainsKey(subscriptionId))
{
return(_azureMonitorClients[subscriptionId]);
}
var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, metricsCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
_azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient);
return(azureMonitorClient);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureCloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param>
/// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILoggerFactory loggerFactory)
{
Guard.NotNullOrWhitespace(tenantId, nameof(tenantId));
Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId));
Guard.NotNullOrWhitespace(applicationId, nameof(applicationId));
Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
_logger = loggerFactory.CreateLogger <AzureMonitorClient>();
_authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, applicationId, applicationSecret, azureMonitorLoggingConfiguration, loggerFactory, metricSinkWriter, metricsCollector);
}
/// <summary>
/// Provides an Azure Monitor client
/// </summary>
/// <param name="cloud">Name of the Azure cloud to interact with</param>
/// <param name="tenantId">Id of the tenant that owns the Azure subscription</param>
/// <param name="subscriptionId">Id of the Azure subscription</param>
/// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param>
/// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param>
/// <param name="resourceMetricDefinitionMemoryCache">Memory cache to store items in</param>
/// <param name="configuration">Configuration of Promitor</param>
/// <param name="azureMonitorIntegrationConfiguration">Options for Azure Monitor integration</param>
/// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param>
/// <param name="loggerFactory">Factory to create loggers with</param>
public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory)
{
if (_azureMonitorClients.ContainsKey(subscriptionId))
{
return(_azureMonitorClients[subscriptionId]);
}
var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, azureScrapingPrometheusMetricsCollector, resourceMetricDefinitionMemoryCache, configuration, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration, loggerFactory);
_azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient);
return(azureMonitorClient);
}
