public MetricScrapingJob(string jobName, ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition, MetricSinkWriter metricSinkWriter, IPrometheusMetricWriter prometheusMetricWriter, MetricScraperFactory metricScraperFactory, AzureMonitorClient azureMonitorClient, ILogger <MetricScrapingJob> logger) { Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition)); Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient)); Guard.NotNullOrWhitespace(jobName, nameof(jobName)); Guard.NotNull(logger, nameof(logger)); Name = jobName; _metricScrapeDefinition = metricScrapeDefinition; _prometheusMetricWriter = prometheusMetricWriter; _metricSinkWriter = metricSinkWriter; _logger = logger; _metricScraperFactory = metricScraperFactory; _azureMonitorClient = azureMonitorClient; }
public ResourceDiscoveryGroupScrapingJob(string jobName, string resourceDiscoveryGroupName, AzureMetadata azureMetadata, MetricDefinition metricDefinition, ResourceDiscoveryRepository resourceDiscoveryRepository, MetricSinkWriter metricSinkWriter, MetricScraperFactory metricScraperFactory, AzureMonitorClientFactory azureMonitorClientFactory, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <ResourceDiscoveryGroupScrapingJob> logger) : base(jobName, logger) { Guard.NotNullOrWhitespace(resourceDiscoveryGroupName, nameof(resourceDiscoveryGroupName)); Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository)); Guard.NotNull(metricDefinition, nameof(metricDefinition)); Guard.NotNull(azureMetadata, nameof(azureMetadata)); Guard.NotNullOrWhitespace(jobName, nameof(jobName)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory)); Guard.NotNull(runtimeMetricCollector, nameof(runtimeMetricCollector)); Guard.NotNull(configuration, nameof(configuration)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); ResourceDiscoveryGroupName = resourceDiscoveryGroupName; _azureMetadata = azureMetadata; _metricDefinition = metricDefinition; _resourceDiscoveryRepository = resourceDiscoveryRepository; _metricSinkWriter = metricSinkWriter; _runtimeMetricCollector = runtimeMetricCollector; _azureMonitorClientFactory = azureMonitorClientFactory; _configuration = configuration; _azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration; _loggerFactory = loggerFactory; _metricScraperFactory = metricScraperFactory; }
/// <summary> /// Constructor /// </summary> protected Scraper(ScraperConfiguration scraperConfiguration) { Guard.NotNull(scraperConfiguration, nameof(scraperConfiguration)); _logger = scraperConfiguration.Logger; _metricSinkWriter = scraperConfiguration.MetricSinkWriter; AzureMonitorClient = scraperConfiguration.AzureMonitorClient; }
/// <summary> /// Constructor /// </summary> /// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="logger">General logger</param> public ScraperConfiguration(AzureMonitorClient azureMonitorClient, MetricSinkWriter metricSinkWriter, ILogger logger) { Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient)); Guard.NotNull(logger, nameof(logger)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); AzureMonitorClient = azureMonitorClient; Logger = logger; MetricSinkWriter = metricSinkWriter; }
/// <summary> /// Constructor /// </summary> protected Scraper(ScraperConfiguration scraperConfiguration) { Guard.NotNull(scraperConfiguration, nameof(scraperConfiguration)); _metricSinkWriter = scraperConfiguration.MetricSinkWriter; Logger = scraperConfiguration.Logger; AzureMonitorClient = scraperConfiguration.AzureMonitorClient; AzureScrapingPrometheusMetricsCollector = scraperConfiguration.AzureScrapingPrometheusMetricsCollector; }
/// <summary> /// Constructor /// </summary> /// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="azureScrapingPrometheusMetricsCollector">Collector to send metrics related to the runtime</param> /// <param name="logger">General logger</param> public ScraperConfiguration(AzureMonitorClient azureMonitorClient, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, ILogger logger) { Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient)); Guard.NotNull(logger, nameof(logger)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector)); AzureMonitorClient = azureMonitorClient; Logger = logger; MetricSinkWriter = metricSinkWriter; AzureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector; }
public async Task ReportMetricAsync_WriteToNoSinks_Succeeds() { // Arrange var metricName = BogusGenerator.Name.FirstName(); var metricDescription = BogusGenerator.Lorem.Sentence(); var metricValue = BogusGenerator.Random.Double(); var scrapeResult = ScrapeResultGenerator.Generate(metricValue); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink>(), NullLogger <MetricSinkWriter> .Instance); // Act & Assert await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult); }
public async Task ReportMetricAsync_WriteToOneSinkWithoutMetricDescription_Succeeds(string metricDescription) { // Arrange var metricName = BogusGenerator.Name.FirstName(); var metricValue = BogusGenerator.Random.Double(); var scrapeResult = ScrapeResultGenerator.Generate(metricValue); var metricSink = new Mock <IMetricSink>(); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { metricSink.Object }, NullLogger <MetricSinkWriter> .Instance); // Act & Assert await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult); }
private static void ScheduleResourcesScraping(MetricsDeclaration metricsDeclaration, MetricSinkWriter metricSinkWriter, AzureMonitorClientFactory azureMonitorClientFactory, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IScrapingMutex scrapingTaskMutex, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services) { var jobName = GenerateResourceScrapingJobName(metricsDeclaration, logger); services.AddScheduler(builder => { builder.AddJob(jobServices => new ResourcesScrapingJob(jobName, metricsDeclaration, jobServices.GetService <ResourceDiscoveryRepository>(), metricSinkWriter, jobServices.GetService <MetricScraperFactory>(), azureMonitorClientFactory, azureScrapingPrometheusMetricsCollector, resourceMetricDefinitionMemoryCache, scrapingTaskMutex, configuration, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration, loggerFactory, jobServices.GetService <ILogger <ResourcesScrapingJob> >()), schedulerOptions => { schedulerOptions.CronSchedule = metricsDeclaration.Metrics[0].Scraping.Schedule; schedulerOptions.RunImmediately = true; }, jobName: jobName); builder.AddUnobservedTaskExceptionHandler(s => { return ((_, exceptionEventArgs) => { var exceptionLogger = s.GetService <ILogger <BackgroundJobMonitor> >(); BackgroundJobMonitor.HandleException(jobName, exceptionEventArgs, exceptionLogger); }); }); }); logger.LogInformation("Scheduled scraping job {JobName}.", jobName); }
public async Task ReportMetricAsync_WriteToOneSinkWithoutMetricName_ThrowsException(string metricName) { // Arrange var metricDescription = _bogus.Lorem.Sentence(); var metricValue = _bogus.Random.Double(); var scrapeResult = ScrapeResultGenerator.Generate(metricValue); var metricSink = new Mock <IMetricSink>(); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { metricSink.Object }, NullLogger <MetricSinkWriter> .Instance); // Act & Assert // ReSharper disable once ExpressionIsAlwaysNull await Assert.ThrowsAsync <ArgumentException>(() => metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult)); }
public async Task ReportMetricAsync_WriteToOneSinkWithoutScrapeResult_ThrowsException() { // Arrange var metricName = BogusGenerator.Name.FirstName(); var metricDescription = BogusGenerator.Lorem.Sentence(); ScrapeResult scrapeResult = null; var metricSink = new Mock <IMetricSink>(); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { metricSink.Object }, NullLogger <MetricSinkWriter> .Instance); // Act & Assert // ReSharper disable once ExpressionIsAlwaysNull await Assert.ThrowsAsync <ArgumentNullException>(() => metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult)); }
/// <summary> /// Create a metrics scraping job for one or more resources, either enumerated specifically or /// identified via resource definition groups. All metrics included are expected to have /// the same scraping schedule. /// </summary> /// <param name="jobName">name of scheduled job</param> /// <param name="metricsDeclaration">declaration of which metrics to collect from which resources</param> /// <param name="resourceDiscoveryRepository">repository source for discovering resources via criteria</param> /// <param name="metricSinkWriter">destination for metric reporting output</param> /// <param name="metricScraperFactory">means of obtaining a metrics scraper for a particular type of resource</param> /// <param name="azureMonitorClientFactory">means of obtaining a Azure Monitor client</param> /// <param name="azureScrapingPrometheusMetricsCollector">metrics collector to write metrics to Prometheus</param> /// <param name="resourceMetricDefinitionMemoryCache">cache of metric definitions by resource ID</param> /// <param name="scrapingTaskMutex">semaphore used to limit concurrency of tasks if configured, or null for no limiting</param> /// <param name="configuration">Promitor configuration</param> /// <param name="azureMonitorIntegrationConfiguration">options for Azure Monitor integration</param> /// <param name="azureMonitorLoggingConfiguration">options for Azure Monitor logging</param> /// <param name="loggerFactory">means to obtain a logger</param> /// <param name="logger">logger to use for scraping detail</param> public ResourcesScrapingJob(string jobName, MetricsDeclaration metricsDeclaration, ResourceDiscoveryRepository resourceDiscoveryRepository, MetricSinkWriter metricSinkWriter, MetricScraperFactory metricScraperFactory, AzureMonitorClientFactory azureMonitorClientFactory, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IScrapingMutex scrapingTaskMutex, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <ResourcesScrapingJob> logger) : base(jobName, logger) { Guard.NotNullOrWhitespace(jobName, nameof(jobName)); Guard.NotNull(metricsDeclaration, nameof(metricsDeclaration)); Guard.NotNull(metricsDeclaration.AzureMetadata, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.AzureMetadata)}"); Guard.NotNull(metricsDeclaration.Metrics, $"{nameof(metricsDeclaration)}.{nameof(metricsDeclaration.Metrics)}"); Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory)); Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector)); Guard.NotNull(resourceMetricDefinitionMemoryCache, nameof(resourceMetricDefinitionMemoryCache)); Guard.NotNull(configuration, nameof(configuration)); Guard.NotNull(azureMonitorIntegrationConfiguration, nameof(azureMonitorIntegrationConfiguration)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); // all metrics must have the same scraping schedule or it is not possible for them to share the same job VerifyAllMetricsHaveSameScrapingSchedule(metricsDeclaration); _metricsDeclaration = metricsDeclaration; _resourceDiscoveryRepository = resourceDiscoveryRepository; _metricSinkWriter = metricSinkWriter; _metricScraperFactory = metricScraperFactory; _azureMonitorClientFactory = azureMonitorClientFactory; _azureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector; _resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache; _scrapingTaskMutex = scrapingTaskMutex; _configuration = configuration; _azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration; _azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration; _loggerFactory = loggerFactory; }
public async Task ReportMetricAsync_WriteToOneSink_Succeeds() { // Arrange var metricName = BogusGenerator.Name.FirstName(); var metricDescription = BogusGenerator.Lorem.Sentence(); var metricValue = BogusGenerator.Random.Double(); var scrapeResult = ScrapeResultGenerator.Generate(metricValue); var metricSink = new Mock <IMetricSink>(); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { metricSink.Object }, NullLogger <MetricSinkWriter> .Instance); // Act await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult); // Assert metricSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, scrapeResult), Times.Once()); }
public ResourceScrapingJob(string jobName, ScrapeDefinition <IAzureResourceDefinition> metricScrapeDefinition, MetricSinkWriter metricSinkWriter, MetricScraperFactory metricScraperFactory, AzureMonitorClient azureMonitorClient, ILogger <ResourceScrapingJob> logger) : base(jobName, logger) { Guard.NotNull(metricScrapeDefinition, nameof(metricScrapeDefinition)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); _metricScrapeDefinition = metricScrapeDefinition; _metricSinkWriter = metricSinkWriter; _metricScraperFactory = metricScraperFactory; _azureMonitorClient = azureMonitorClient; }
public async Task ReportMetricAsync_WriteToStatsDSink_Succeeds() { // Arrange var metricName = _bogus.Name.FirstName(); var metricDescription = _bogus.Lorem.Sentence(); var metricValue = _bogus.Random.Double(); var scrapeResult = ScrapeResultGenerator.Generate(metricValue); var statsDPublisherMock = new Mock <IStatsDPublisher>(); var statsdMetricSink = new StatsdMetricSink(statsDPublisherMock.Object, NullLogger <StatsdMetricSink> .Instance); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { statsdMetricSink }, NullLogger <MetricSinkWriter> .Instance); // Act await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult); // Assert statsDPublisherMock.Verify(mock => mock.Gauge(metricValue, metricName), Times.Once()); }
public async Task ReportMetricAsync_WriteToMultipleSinks_Succeeds() { // Arrange var metricName = _bogus.Name.FirstName(); var metricDescription = _bogus.Lorem.Sentence(); var metricValue = _bogus.Random.Double(); var scrapeResult = GenerateScrapeResult(metricValue); var firstSink = new Mock <IMetricSink>(); var secondSink = new Mock <IMetricSink>(); var metricSinkWriter = new MetricSinkWriter(new List <IMetricSink> { firstSink.Object, secondSink.Object }, NullLogger <MetricSinkWriter> .Instance); // Act await metricSinkWriter.ReportMetricAsync(metricName, metricDescription, scrapeResult); // Assert firstSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, It.IsAny <MeasuredMetric>()), Times.Once()); secondSink.Verify(mock => mock.ReportMetricAsync(metricName, metricDescription, It.IsAny <MeasuredMetric>()), Times.Once()); }
/// <summary> /// Constructor /// </summary> /// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param> /// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param> /// <param name="applicationId">Id of the application that is being used to interact with Azure Resource Manager</param> /// <param name="metricSinkWriter">Metrics writer to all sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="logger">Logger to write telemetry to</param> public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, string applicationId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILogger logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); Guard.NotNull(metricsCollector, nameof(metricsCollector)); Guard.NotNull(logger, nameof(logger)); _logger = logger; _metricSinkWriter = metricSinkWriter; _metricsCollector = metricsCollector; _metricLabels = new Dictionary <string, string> { { "tenant_id", tenantId }, { "subscription_id", subscriptionId }, { "app_id", applicationId } }; }
/// <summary> /// Constructor /// </summary> /// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param> /// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param> /// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param> /// <param name="metricSinkWriter">Metrics writer to all sinks</param> /// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="logger">Logger to write telemetry to</param> public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, ILogger logger) : base(azureScrapingPrometheusMetricsCollector, logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); Guard.NotNull(azureScrapingPrometheusMetricsCollector, nameof(azureScrapingPrometheusMetricsCollector)); Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo)); _metricSinkWriter = metricSinkWriter; _azureScrapingPrometheusMetricsCollector = azureScrapingPrometheusMetricsCollector; var id = DetermineApplicationId(azureAuthenticationInfo); _metricLabels = new Dictionary <string, string> { { "tenant_id", tenantId }, { "subscription_id", subscriptionId }, { "app_id", id }, }; }
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration) { var credentials = AzureAuthenticationFactory.CreateAzureAuthentication(azureCloud, tenantId, azureAuthenticationInfo, _azureCredentialsFactory); var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>(); var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, azureAuthenticationInfo, metricSinkWriter, azureScrapingPrometheusMetricsCollector, throttlingLogger); var azureClientConfiguration = Microsoft.Azure.Management.Fluent.Azure.Configure() .WithDelegatingHandler(monitorHandler); var azureMonitorLogging = azureMonitorLoggingConfiguration.Value; if (azureMonitorLogging.IsEnabled) { var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>(); ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger)); ServiceClientTracing.IsEnabled = true; azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler()) .WithLogLevel(azureMonitorLogging.InformationLevel); } return(azureClientConfiguration .Authenticate(credentials) .WithSubscription(subscriptionId)); }
/// <summary> /// Constructor /// </summary> /// <param name="azureCloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="resourceMetricDefinitionMemoryCache">Memory cache to store items in for performance optimizations</param> /// <param name="loggerFactory">Factory to create loggers with</param> /// <param name="azureMonitorIntegrationConfiguration">Options for Azure Monitor logging</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor integration</param> public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, ILoggerFactory loggerFactory, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo)); Guard.NotNull(azureMonitorIntegrationConfiguration, nameof(azureMonitorIntegrationConfiguration)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); Guard.NotNull(resourceMetricDefinitionMemoryCache, nameof(resourceMetricDefinitionMemoryCache)); _resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache; _azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration; _logger = loggerFactory.CreateLogger <AzureMonitorClient>(); _authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, azureAuthenticationInfo, loggerFactory, metricSinkWriter, azureScrapingPrometheusMetricsCollector, azureMonitorLoggingConfiguration); }
/// <summary> /// Creates a scraper that is capable of scraping a specific resource type /// </summary> /// <param name="metricDefinitionResourceType">Resource type to scrape</param> /// <param name="metricSinkWriter">Writer to send metrics to all sinks</param> /// <param name="azureScrapingPrometheusMetricsCollector">Collector to send metrics related to the runtime</param> /// <param name="azureMonitorClient">Client to interact with Azure Monitor</param> public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, AzureMonitorClient azureMonitorClient) { var scraperConfiguration = new ScraperConfiguration(azureMonitorClient, metricSinkWriter, azureScrapingPrometheusMetricsCollector, _logger); switch (metricDefinitionResourceType) { case ResourceType.ApiManagement: return(new ApiManagementScraper(scraperConfiguration)); case ResourceType.ApplicationGateway: return(new ApplicationGatewayScraper(scraperConfiguration)); case ResourceType.ApplicationInsights: return(new ApplicationInsightsScraper(scraperConfiguration)); case ResourceType.AppPlan: return(new AppPlanScraper(scraperConfiguration)); case ResourceType.AutomationAccount: return(new AutomationAccountScraper(scraperConfiguration)); case ResourceType.BlobStorage: return(new BlobStorageScraper(scraperConfiguration)); case ResourceType.Cdn: return(new CdnScraper(scraperConfiguration)); case ResourceType.ContainerInstance: return(new ContainerInstanceScraper(scraperConfiguration)); case ResourceType.ContainerRegistry: return(new ContainerRegistryScraper(scraperConfiguration)); case ResourceType.CosmosDb: return(new CosmosDbScraper(scraperConfiguration)); case ResourceType.DataFactory: return(new DataFactoryScraper(scraperConfiguration)); case ResourceType.DataShare: return(new DataShareScraper(scraperConfiguration)); case ResourceType.DeviceProvisioningService: return(new DeviceProvisioningServiceScraper(scraperConfiguration)); case ResourceType.EventHubs: return(new EventHubsScraper(scraperConfiguration)); case ResourceType.ExpressRouteCircuit: return(new ExpressRouteCircuitScraper(scraperConfiguration)); case ResourceType.FileStorage: return(new FileStorageScraper(scraperConfiguration)); case ResourceType.FrontDoor: return(new FrontDoorScraper(scraperConfiguration)); case ResourceType.FunctionApp: return(new FunctionAppScraper(scraperConfiguration)); case ResourceType.Generic: return(new GenericScraper(scraperConfiguration)); case ResourceType.IoTHub: return(new IoTHubScraper(scraperConfiguration)); case ResourceType.KeyVault: return(new KeyVaultScraper(scraperConfiguration)); case ResourceType.KubernetesService: return(new KubernetesServiceScraper(scraperConfiguration)); case ResourceType.LoadBalancer: return(new LoadBalancerScraper(scraperConfiguration)); case ResourceType.LogicApp: return(new LogicAppScraper(scraperConfiguration)); case ResourceType.MariaDb: return(new MariaDbScraper(scraperConfiguration)); case ResourceType.MonitorAutoscale: return(new MonitorAutoscaleScraper(scraperConfiguration)); case ResourceType.MySql: return(new MySqlScraper(scraperConfiguration)); case ResourceType.NetworkGateway: return(new NetworkGatewayScraper(scraperConfiguration)); case ResourceType.NetworkInterface: return(new NetworkInterfaceScraper(scraperConfiguration)); case ResourceType.PostgreSql: return(new PostgreSqlScraper(scraperConfiguration)); case ResourceType.RedisCache: return(new RedisCacheScraper(scraperConfiguration)); case ResourceType.RedisEnterpriseCache: return(new RedisEnterpriseCacheScraper(scraperConfiguration)); case ResourceType.ServiceBusNamespace: return(new ServiceBusNamespaceScraper(scraperConfiguration)); case ResourceType.SqlDatabase: return(new SqlDatabaseScraper(scraperConfiguration)); case ResourceType.SqlElasticPool: return(new SqlElasticPoolScraper(scraperConfiguration)); case ResourceType.SqlManagedInstance: return(new SqlManagedInstanceScraper(scraperConfiguration)); case ResourceType.SqlServer: return(new SqlServerScraper(scraperConfiguration)); case ResourceType.StorageAccount: return(new StorageAccountScraper(scraperConfiguration)); case ResourceType.StorageQueue: return(new StorageQueueScraper(scraperConfiguration)); case ResourceType.SynapseApacheSparkPool: return(new SynapseApacheSparkPoolScraper(scraperConfiguration)); case ResourceType.SynapseSqlPool: return(new SynapseSqlPoolScraper(scraperConfiguration)); case ResourceType.SynapseWorkspace: return(new SynapseWorkspaceScraper(scraperConfiguration)); case ResourceType.VirtualMachine: return(new VirtualMachineScraper(scraperConfiguration)); case ResourceType.VirtualMachineScaleSet: return(new VirtualMachineScaleSetScraper(scraperConfiguration)); case ResourceType.VirtualNetwork: return(new VirtualNetworkScraper(scraperConfiguration)); case ResourceType.WebApp: return(new WebAppScraper(scraperConfiguration)); default: throw new ArgumentOutOfRangeException(nameof(metricDefinitionResourceType), metricDefinitionResourceType, "Matching scraper not found"); } }
/// <summary> /// Creates a scraper that is capable of scraping a specific resource type /// </summary> /// <param name="metricDefinitionResourceType">Resource type to scrape</param> /// <param name="metricSinkWriter">Writer to send metrics to all sinks</param> /// <param name="azureMonitorClient">Client to interact with Azure Monitor</param> public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, MetricSinkWriter metricSinkWriter, AzureMonitorClient azureMonitorClient) { var scraperConfiguration = new ScraperConfiguration(azureMonitorClient, metricSinkWriter, _logger); switch (metricDefinitionResourceType) { case ResourceType.ApiManagement: return(new ApiManagementScraper(scraperConfiguration)); case ResourceType.ApplicationGateway: return(new ApplicationGatewayScraper(scraperConfiguration)); case ResourceType.AppPlan: return(new AppPlanScraper(scraperConfiguration)); case ResourceType.BlobStorage: return(new BlobStorageScraper(scraperConfiguration)); case ResourceType.ContainerInstance: return(new ContainerInstanceScraper(scraperConfiguration)); case ResourceType.ContainerRegistry: return(new ContainerRegistryScraper(scraperConfiguration)); case ResourceType.CosmosDb: return(new CosmosDbScraper(scraperConfiguration)); case ResourceType.DeviceProvisioningService: return(new DeviceProvisioningServiceScraper(scraperConfiguration)); case ResourceType.EventHubs: return(new EventHubsScraper(scraperConfiguration)); case ResourceType.ExpressRouteCircuit: return(new ExpressRouteCircuitScraper(scraperConfiguration)); case ResourceType.FileStorage: return(new FileStorageScraper(scraperConfiguration)); case ResourceType.FunctionApp: return(new FunctionAppScraper(scraperConfiguration)); case ResourceType.Generic: return(new GenericScraper(scraperConfiguration)); case ResourceType.IoTHub: return(new IoTHubScraper(scraperConfiguration)); case ResourceType.KeyVault: return(new KeyVaultScraper(scraperConfiguration)); case ResourceType.NetworkGateway: return(new NetworkGatewayScraper(scraperConfiguration)); case ResourceType.NetworkInterface: return(new NetworkInterfaceScraper(scraperConfiguration)); case ResourceType.LogicApp: return(new LogicAppScraper(scraperConfiguration)); case ResourceType.PostgreSql: return(new PostgreSqlScraper(scraperConfiguration)); case ResourceType.RedisCache: return(new RedisCacheScraper(scraperConfiguration)); case ResourceType.ServiceBusQueue: return(new ServiceBusQueueScraper(scraperConfiguration)); case ResourceType.SqlDatabase: return(new SqlDatabaseScraper(scraperConfiguration)); case ResourceType.SqlManagedInstance: return(new SqlManagedInstanceScraper(scraperConfiguration)); case ResourceType.SqlServer: return(new SqlServerScraper(scraperConfiguration)); case ResourceType.StorageAccount: return(new StorageAccountScraper(scraperConfiguration)); case ResourceType.StorageQueue: return(new StorageQueueScraper(scraperConfiguration)); case ResourceType.VirtualMachine: return(new VirtualMachineScraper(scraperConfiguration)); case ResourceType.VirtualMachineScaleSet: return(new VirtualMachineScaleSetScraper(scraperConfiguration)); case ResourceType.WebApp: return(new WebAppScraper(scraperConfiguration)); default: throw new ArgumentOutOfRangeException(); } }
private static void ScheduleResourceDiscoveryGroupScraping(AzureResourceDiscoveryGroup resourceDiscoveryGroup, AzureMetadata azureMetadata, MetricDefinition metricDefinition, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services) { var jobName = GenerateResourceDiscoveryGroupScrapingJobName(metricDefinition.PrometheusMetricDefinition?.Name, resourceDiscoveryGroup.Name); services.AddScheduler(builder => { builder.AddJob(jobServices => { return(new ResourceDiscoveryGroupScrapingJob(jobName, resourceDiscoveryGroup.Name, azureMetadata, metricDefinition, jobServices.GetService <ResourceDiscoveryRepository>(), metricSinkWriter, jobServices.GetService <MetricScraperFactory>(), azureMonitorClientFactory, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory, jobServices.GetService <ILogger <ResourceDiscoveryGroupScrapingJob> >())); }, schedulerOptions => { schedulerOptions.CronSchedule = metricDefinition.Scraping.Schedule; schedulerOptions.RunImmediately = true; }, jobName: jobName); builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services); }); logger.LogInformation("Scheduled scraping job {JobName} for resource collection {ResourceDiscoveryGroup} which will be reported as metric {MetricName}", jobName, resourceDiscoveryGroup.Name, metricDefinition.PrometheusMetricDefinition?.Name); }
private static void ScheduleResourceScraping(IAzureResourceDefinition resource, AzureMetadata azureMetadata, MetricDefinition metric, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services) { var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId; var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(azureMetadata.Cloud, azureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory); var scrapeDefinition = metric.CreateScrapeDefinition(resource, azureMetadata); var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource); services.AddScheduler(builder => { builder.AddJob(jobServices => { return(new ResourceScrapingJob(jobName, scrapeDefinition, metricSinkWriter, jobServices.GetService <MetricScraperFactory>(), azureMonitorClient, jobServices.GetService <ILogger <ResourceScrapingJob> >())); }, schedulerOptions => { schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule; schedulerOptions.RunImmediately = true; }, jobName: jobName); builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services); }); logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.UniqueName, scrapeDefinition.PrometheusMetricDefinition?.Name); }
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { var azureCredentials = AzureAuthenticationFactory.GetConfiguredAzureAuthentication(configuration); var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials, metricSinkWriter, azureScrapingPrometheusMetricsCollector, resourceMetricDefinitionMemoryCache, loggerFactory, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration); return(azureMonitorClient); }
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector) { var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud); var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>(); var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, metricSinkWriter, metricsCollector, throttlingLogger); var azureClientConfiguration = Azure.Configure() .WithDelegatingHandler(monitorHandler); var azureMonitorLogging = azureMonitorLoggingConfiguration.Value; if (azureMonitorLogging.IsEnabled) { var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>(); ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger)); ServiceClientTracing.IsEnabled = true; azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler()) .WithLogLevel(azureMonitorLogging.InformationLevel); } return(azureClientConfiguration .Authenticate(credentials) .WithSubscription(subscriptionId)); }
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { var azureCredentials = DetermineAzureCredentials(configuration); var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, metricSinkWriter, metricsCollector, loggerFactory); return(azureMonitorClient); }
/// <summary> /// Provides an Azure Monitor client /// </summary> /// <param name="cloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="configuration">Configuration of Promitor</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param> /// <param name="loggerFactory">Factory to create loggers with</param> public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { if (_azureMonitorClients.ContainsKey(subscriptionId)) { return(_azureMonitorClients[subscriptionId]); } var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, metricsCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory); _azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient); return(azureMonitorClient); }
/// <summary> /// Constructor /// </summary> /// <param name="azureCloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param> /// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="loggerFactory">Factory to create loggers with</param> public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILoggerFactory loggerFactory) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); _logger = loggerFactory.CreateLogger <AzureMonitorClient>(); _authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, applicationId, applicationSecret, azureMonitorLoggingConfiguration, loggerFactory, metricSinkWriter, metricsCollector); }
/// <summary> /// Provides an Azure Monitor client /// </summary> /// <param name="cloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="azureScrapingPrometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="resourceMetricDefinitionMemoryCache">Memory cache to store items in</param> /// <param name="configuration">Configuration of Promitor</param> /// <param name="azureMonitorIntegrationConfiguration">Options for Azure Monitor integration</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param> /// <param name="loggerFactory">Factory to create loggers with</param> public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingPrometheusMetricsCollector azureScrapingPrometheusMetricsCollector, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions <AzureMonitorIntegrationConfiguration> azureMonitorIntegrationConfiguration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { if (_azureMonitorClients.ContainsKey(subscriptionId)) { return(_azureMonitorClients[subscriptionId]); } var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, azureScrapingPrometheusMetricsCollector, resourceMetricDefinitionMemoryCache, configuration, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration, loggerFactory); _azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient); return(azureMonitorClient); }