public ResourceDiscoveryGroupScrapingJob(string jobName, string resourceDiscoveryGroupName, AzureMetadata azureMetadata, MetricDefinition metricDefinition, ResourceDiscoveryRepository resourceDiscoveryRepository, MetricSinkWriter metricSinkWriter, MetricScraperFactory metricScraperFactory, AzureMonitorClientFactory azureMonitorClientFactory, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <ResourceDiscoveryGroupScrapingJob> logger) : base(jobName, logger) { Guard.NotNullOrWhitespace(resourceDiscoveryGroupName, nameof(resourceDiscoveryGroupName)); Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository)); Guard.NotNull(metricDefinition, nameof(metricDefinition)); Guard.NotNull(azureMetadata, nameof(azureMetadata)); Guard.NotNullOrWhitespace(jobName, nameof(jobName)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory)); Guard.NotNull(runtimeMetricCollector, nameof(runtimeMetricCollector)); Guard.NotNull(configuration, nameof(configuration)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); ResourceDiscoveryGroupName = resourceDiscoveryGroupName; _azureMetadata = azureMetadata; _metricDefinition = metricDefinition; _resourceDiscoveryRepository = resourceDiscoveryRepository; _metricSinkWriter = metricSinkWriter; _runtimeMetricCollector = runtimeMetricCollector; _azureMonitorClientFactory = azureMonitorClientFactory; _configuration = configuration; _azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration; _loggerFactory = loggerFactory; _metricScraperFactory = metricScraperFactory; }
public MetricScrapingJob(ScrapeDefinition <AzureResourceDefinition> metric, IMetricsDeclarationProvider metricsDeclarationProvider, IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector, MetricScraperFactory metricScraperFactory, ILogger logger, IExceptionTracker exceptionTracker) { Guard.NotNull(metric, nameof(metric)); Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider)); Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter)); Guard.NotNull(runtimeMetricsCollector, nameof(runtimeMetricsCollector)); Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory)); Guard.NotNull(logger, nameof(logger)); Guard.NotNull(exceptionTracker, nameof(exceptionTracker)); _metric = metric; _metricsDeclarationProvider = metricsDeclarationProvider; _prometheusMetricWriter = prometheusMetricWriter; _runtimeMetricsCollector = runtimeMetricsCollector; _exceptionTracker = exceptionTracker; _logger = logger; _metricScraperFactory = metricScraperFactory; ConfigureJob(); }
public MetricScrapingJob(MetricDefinition metric, IMetricsDeclarationProvider metricsDeclarationProvider, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger, IExceptionTracker exceptionTracker) { Guard.NotNull(metric, nameof(metric)); Guard.NotNull(exceptionTracker, nameof(exceptionTracker)); Guard.NotNull(logger, nameof(logger)); _metric = metric; _metricsDeclarationProvider = metricsDeclarationProvider; _runtimeMetricsCollector = runtimeMetricsCollector; _exceptionTracker = exceptionTracker; _logger = logger; ConfigureJob(); }
/// <summary> /// Constructor /// </summary> /// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param> /// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param> /// <param name="applicationId">Id of the application that is being used to interact with Azure Resource Manager</param> /// <param name="metricsCollector">Metrics collector to write metrics to</param> /// <param name="logger">Logger to write telemetry to</param> public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, string applicationId, IRuntimeMetricsCollector metricsCollector, ILogger logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNull(metricsCollector, nameof(metricsCollector)); Guard.NotNull(logger, nameof(logger)); _logger = logger; _metricsCollector = metricsCollector; _metricLabels = new Dictionary <string, string> { { "tenant_id", tenantId }, { "subscription_id", subscriptionId }, { "app_id", applicationId } }; }
/// <summary> /// Constructor /// </summary> /// <param name="tenantId">Id of the tenant that is being interacted with via Azure Resource Manager</param> /// <param name="subscriptionId">Id of the subscription that is being interacted with via Azure Resource Manager</param> /// <param name="azureAuthenticationInfo">Information regarding authentication with Microsoft Azure</param> /// <param name="metricSinkWriter">Metrics writer to all sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="logger">Logger to write telemetry to</param> public AzureResourceManagerThrottlingRequestHandler(string tenantId, string subscriptionId, AzureAuthenticationInfo azureAuthenticationInfo, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILogger logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter)); Guard.NotNull(metricsCollector, nameof(metricsCollector)); Guard.NotNull(azureAuthenticationInfo, nameof(azureAuthenticationInfo)); Guard.NotNull(logger, nameof(logger)); _logger = logger; _metricSinkWriter = metricSinkWriter; _metricsCollector = metricsCollector; var id = DetermineApplicationId(azureAuthenticationInfo); _metricLabels = new Dictionary <string, string> { { "tenant_id", tenantId }, { "subscription_id", subscriptionId }, { "app_id", id }, }; }
/// <summary> /// Creates a scraper that is capable of scraping a specific resource type /// </summary> /// <param name="azureMetadata">Metadata concerning the Azure resources</param> /// <param name="metricDefinitionResourceType">Resource type to scrape</param> /// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param> /// <param name="logger">General logger</param> /// <param name="exceptionTracker">Tracker used to log exceptions</param> public static IScraper <MetricDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger, IExceptionTracker exceptionTracker) { var azureCredentials = DetermineAzureCredentials(); var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, azureCredentials, runtimeMetricsCollector, logger); switch (metricDefinitionResourceType) { case ResourceType.ServiceBusQueue: return(new ServiceBusQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.Generic: return(new GenericScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.StorageQueue: return(new StorageQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.ContainerInstance: return(new ContainerInstanceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.VirtualMachine: return(new VirtualMachineScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.NetworkInterface: return(new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.ContainerRegistry: return(new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); case ResourceType.CosmosDb: return(new CosmosDbScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker)); default: throw new ArgumentOutOfRangeException(); } }
private static AzureMonitorClient ConfigureAzureMonitorClient(MetricsDeclaration metricsDeclaration, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { var azureCredentials = DetermineAzureCredentials(configuration); var azureMetadata = metricsDeclaration.AzureMetadata; var azureMonitorClient = new AzureMonitorClient(azureMetadata.Cloud, azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, runtimeMetricCollector, loggerFactory); return(azureMonitorClient); }
/// <summary> /// Constructor /// </summary> /// <param name="azureCloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param> /// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param> /// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param> /// <param name="logger">Logger to use during interaction with Azure Monitor</param> public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret)); var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud); var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, runtimeMetricsCollector, logger); _authenticatedAzureSubscription = Azure.Configure().WithDelegatingHandler(monitorHandler).Authenticate(credentials).WithSubscription(subscriptionId); _logger = logger; }
/// <summary> /// Constructor /// </summary> /// <param name="azureCloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="applicationId">Id of the Azure AD application used to authenticate with Azure Monitor</param> /// <param name="applicationSecret">Secret to authenticate with Azure Monitor for the specified Azure AD application</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="loggerFactory">Factory to create loggers with</param> public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, ILoggerFactory loggerFactory) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret)); Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration)); _logger = loggerFactory.CreateLogger <AzureMonitorClient>(); _authenticatedAzureSubscription = CreateAzureClient(azureCloud, tenantId, subscriptionId, applicationId, applicationSecret, azureMonitorLoggingConfiguration, loggerFactory, metricSinkWriter, metricsCollector); }
private IAzure CreateAzureClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector) { var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud); var throttlingLogger = loggerFactory.CreateLogger <AzureResourceManagerThrottlingRequestHandler>(); var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, metricSinkWriter, metricsCollector, throttlingLogger); var azureClientConfiguration = Azure.Configure() .WithDelegatingHandler(monitorHandler); var azureMonitorLogging = azureMonitorLoggingConfiguration.Value; if (azureMonitorLogging.IsEnabled) { var integrationLogger = loggerFactory.CreateLogger <AzureMonitorIntegrationLogger>(); ServiceClientTracing.AddTracingInterceptor(new AzureMonitorIntegrationLogger(integrationLogger)); ServiceClientTracing.IsEnabled = true; azureClientConfiguration = azureClientConfiguration.WithDelegatingHandler(new HttpLoggingDelegatingHandler()) .WithLogLevel(azureMonitorLogging.InformationLevel); } return(azureClientConfiguration .Authenticate(credentials) .WithSubscription(subscriptionId)); }
private static AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger) { var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, logger); return(azureMonitorClient); }
private AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, IRuntimeMetricsCollector runtimeMetricsCollector) { var azureCredentials = DetermineAzureCredentials(); var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, _logger); return(azureMonitorClient); }
/// <summary> /// Creates a scraper that is capable of scraping a specific resource type /// </summary> /// <param name="azureMetadata">Metadata concerning the Azure resources</param> /// <param name="metricDefinitionResourceType">Resource type to scrape</param> /// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param> /// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param> public IScraper <AzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata, IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector) { var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, runtimeMetricsCollector); var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, prometheusMetricWriter, _logger, _exceptionTracker); switch (metricDefinitionResourceType) { case ResourceType.ServiceBusQueue: return(new ServiceBusQueueScraper(scraperConfiguration)); case ResourceType.Generic: return(new GenericScraper(scraperConfiguration)); case ResourceType.StorageQueue: return(new StorageQueueScraper(scraperConfiguration)); case ResourceType.ContainerInstance: return(new ContainerInstanceScraper(scraperConfiguration)); case ResourceType.VirtualMachine: return(new VirtualMachineScraper(scraperConfiguration)); case ResourceType.NetworkInterface: return(new NetworkInterfaceScraper(scraperConfiguration)); case ResourceType.ContainerRegistry: return(new ContainerRegistryScraper(scraperConfiguration)); case ResourceType.CosmosDb: return(new CosmosDbScraper(scraperConfiguration)); case ResourceType.RedisCache: return(new RedisCacheScraper(scraperConfiguration)); case ResourceType.PostgreSql: return(new PostgreSqlScraper(scraperConfiguration)); case ResourceType.AzureSqlDatabase: return(new AzureSqlDatabaseScraper(scraperConfiguration)); default: throw new ArgumentOutOfRangeException(); } }
/// <summary> /// Provides an Azure Monitor client /// </summary> /// <param name="cloud">Name of the Azure cloud to interact with</param> /// <param name="tenantId">Id of the tenant that owns the Azure subscription</param> /// <param name="subscriptionId">Id of the Azure subscription</param> /// <param name="metricSinkWriter">Writer to send metrics to all configured sinks</param> /// <param name="metricsCollector">Metrics collector to write metrics to Prometheus</param> /// <param name="configuration">Configuration of Promitor</param> /// <param name="azureMonitorLoggingConfiguration">Options for Azure Monitor logging</param> /// <param name="loggerFactory">Factory to create loggers with</param> public AzureMonitorClient CreateIfNotExists(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { if (_azureMonitorClients.ContainsKey(subscriptionId)) { return(_azureMonitorClients[subscriptionId]); } var azureMonitorClient = CreateNewAzureMonitorClient(cloud, tenantId, subscriptionId, metricSinkWriter, metricsCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory); _azureMonitorClients.TryAdd(subscriptionId, azureMonitorClient); return(azureMonitorClient); }
private static AzureMonitorClient CreateNewAzureMonitorClient(AzureEnvironment cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector metricsCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { var azureCredentials = DetermineAzureCredentials(configuration); var azureMonitorClient = new AzureMonitorClient(cloud, tenantId, subscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, azureMonitorLoggingConfiguration, metricSinkWriter, metricsCollector, loggerFactory); return(azureMonitorClient); }
private static void ScheduleResourceScraping(IAzureResourceDefinition resource, AzureMetadata azureMetadata, MetricDefinition metric, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services) { var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId; var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(azureMetadata.Cloud, azureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory); var scrapeDefinition = metric.CreateScrapeDefinition(resource, azureMetadata); var jobName = GenerateResourceScrapingJobName(scrapeDefinition, resource); services.AddScheduler(builder => { builder.AddJob(jobServices => { return(new ResourceScrapingJob(jobName, scrapeDefinition, metricSinkWriter, jobServices.GetService <MetricScraperFactory>(), azureMonitorClient, jobServices.GetService <ILogger <ResourceScrapingJob> >())); }, schedulerOptions => { schedulerOptions.CronSchedule = scrapeDefinition.Scraping.Schedule; schedulerOptions.RunImmediately = true; }, jobName: jobName); builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services); }); logger.LogInformation("Scheduled scraping job {JobName} for resource {Resource} which will be reported as metric {MetricName}", jobName, scrapeDefinition.Resource.UniqueName, scrapeDefinition.PrometheusMetricDefinition?.Name); }
private static void ScheduleResourceDiscoveryGroupScraping(AzureResourceDiscoveryGroup resourceDiscoveryGroup, AzureMetadata azureMetadata, MetricDefinition metricDefinition, AzureMonitorClientFactory azureMonitorClientFactory, MetricSinkWriter metricSinkWriter, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, ILogger <Startup> logger, IServiceCollection services) { var jobName = GenerateResourceDiscoveryGroupScrapingJobName(metricDefinition.PrometheusMetricDefinition?.Name, resourceDiscoveryGroup.Name); services.AddScheduler(builder => { builder.AddJob(jobServices => { return(new ResourceDiscoveryGroupScrapingJob(jobName, resourceDiscoveryGroup.Name, azureMetadata, metricDefinition, jobServices.GetService <ResourceDiscoveryRepository>(), metricSinkWriter, jobServices.GetService <MetricScraperFactory>(), azureMonitorClientFactory, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory, jobServices.GetService <ILogger <ResourceDiscoveryGroupScrapingJob> >())); }, schedulerOptions => { schedulerOptions.CronSchedule = metricDefinition.Scraping.Schedule; schedulerOptions.RunImmediately = true; }, jobName: jobName); builder.UnobservedTaskExceptionHandler = (sender, exceptionEventArgs) => UnobservedJobHandlerHandler(jobName, exceptionEventArgs, services); }); logger.LogInformation("Scheduled scraping job {JobName} for resource collection {ResourceDiscoveryGroup} which will be reported as metric {MetricName}", jobName, resourceDiscoveryGroup.Name, metricDefinition.PrometheusMetricDefinition?.Name); }