private List <string> ValidateAzureMetadata(AzureMetadata azureMetadata)
{
var errorMessages = new List <string>();
if (azureMetadata == null)
{
errorMessages.Add("No azure metadata is configured");
return(errorMessages);
}
if (string.IsNullOrWhiteSpace(azureMetadata.TenantId))
{
errorMessages.Add($"{azureMetadata.TenantId} is not configured");
}
if (string.IsNullOrWhiteSpace(azureMetadata.SubscriptionId))
{
errorMessages.Add($"{azureMetadata.SubscriptionId} is not configured");
}
if (string.IsNullOrWhiteSpace(azureMetadata.ResourceGroupName))
{
errorMessages.Add($"{azureMetadata.ResourceGroupName} is not configured");
}
return(errorMessages);
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, MetricDefinition metricDefinitionDefinition)
{
Console.WriteLine($"\t> Scraping {metricDefinitionDefinition.Name} of type {metricDefinitionDefinition.ResourceType}");
var scraper = MetricScraperFactory.CreateScraper(azureMetadata, metricDefinitionDefinition.ResourceType);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
private AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, IRuntimeMetricsCollector runtimeMetricsCollector)
{
var azureCredentials = DetermineAzureCredentials();
var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, _logger);
return(azureMonitorClient);
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, MetricDefinition metricDefinitionDefinition)
{
_logger.LogInformation("Scraping '{metricName}' for resource type '{resourceType}'", metricDefinitionDefinition.Name, metricDefinitionDefinition.ResourceType);
var scraper = MetricScraperFactory.CreateScraper(azureMetadata, metricDefinitionDefinition.ResourceType, _logger, _exceptionTracker);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, ScrapeDefinition <IAzureResourceDefinition> metricDefinitionDefinition)
{
_logger.LogInformation("Scraping {MetricName} for resource type {ResourceType}", metricDefinitionDefinition.PrometheusMetricDefinition.Name, metricDefinitionDefinition.Resource.ResourceType);
var scraper = _metricScraperFactory.CreateScraper(metricDefinitionDefinition.Resource.ResourceType, azureMetadata, _prometheusMetricWriter, _azureMonitorClient);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
private static IEnumerable <string> ValidateAzureMetadata(AzureMetadata azureMetadata)
{
var errorMessages = new List <string>();
if (azureMetadata == null)
{
errorMessages.Add("No azure metadata is configured");
return(errorMessages);
}
if (string.IsNullOrWhiteSpace(azureMetadata.TenantId))
{
errorMessages.Add("No tenant id is configured");
}
if (string.IsNullOrWhiteSpace(azureMetadata.SubscriptionId))
{
errorMessages.Add("No subscription id is configured");
}
if (string.IsNullOrWhiteSpace(azureMetadata.ResourceGroupName))
{
errorMessages.Add("No resource group name is not configured");
}
return(errorMessages);
}
public ResourceDiscoveryGroupScrapingJob(string jobName, string resourceDiscoveryGroupName, AzureMetadata azureMetadata, MetricDefinition metricDefinition, ResourceDiscoveryRepository resourceDiscoveryRepository,
MetricSinkWriter metricSinkWriter,
MetricScraperFactory metricScraperFactory,
AzureMonitorClientFactory azureMonitorClientFactory, IRuntimeMetricsCollector runtimeMetricCollector, IConfiguration configuration, IOptions <AzureMonitorLoggingConfiguration> azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory,
ILogger <ResourceDiscoveryGroupScrapingJob> logger)
: base(jobName, logger)
{
Guard.NotNullOrWhitespace(resourceDiscoveryGroupName, nameof(resourceDiscoveryGroupName));
Guard.NotNull(resourceDiscoveryRepository, nameof(resourceDiscoveryRepository));
Guard.NotNull(metricDefinition, nameof(metricDefinition));
Guard.NotNull(azureMetadata, nameof(azureMetadata));
Guard.NotNullOrWhitespace(jobName, nameof(jobName));
Guard.NotNull(metricScraperFactory, nameof(metricScraperFactory));
Guard.NotNull(azureMonitorClientFactory, nameof(azureMonitorClientFactory));
Guard.NotNull(runtimeMetricCollector, nameof(runtimeMetricCollector));
Guard.NotNull(configuration, nameof(configuration));
Guard.NotNull(azureMonitorLoggingConfiguration, nameof(azureMonitorLoggingConfiguration));
Guard.NotNull(loggerFactory, nameof(loggerFactory));
Guard.NotNull(metricSinkWriter, nameof(metricSinkWriter));
ResourceDiscoveryGroupName = resourceDiscoveryGroupName;
_azureMetadata = azureMetadata;
_metricDefinition = metricDefinition;
_resourceDiscoveryRepository = resourceDiscoveryRepository;
_metricSinkWriter = metricSinkWriter;
_runtimeMetricCollector = runtimeMetricCollector;
_azureMonitorClientFactory = azureMonitorClientFactory;
_configuration = configuration;
_azureMonitorLoggingConfiguration = azureMonitorLoggingConfiguration;
_loggerFactory = loggerFactory;
_metricScraperFactory = metricScraperFactory;
}
private static MetricsDeclaration InterpretYamlStream(YamlStream metricsDeclarationYamlStream)
{
var document = metricsDeclarationYamlStream.Documents.First();
var rootNode = (YamlMappingNode)document.RootNode;
AzureMetadata azureMetadata = null;
if (rootNode.Children.ContainsKey("azureMetadata"))
{
var azureMetadataNode = (YamlMappingNode)rootNode.Children[new YamlScalarNode("azureMetadata")];
var azureMetadataSerializer = new AzureMetadataDeserializer();
azureMetadata = azureMetadataSerializer.Deserialize(azureMetadataNode);
}
List <MetricDefinition> metrics = null;
if (rootNode.Children.ContainsKey("metrics"))
{
var metricsNode = (YamlSequenceNode)rootNode.Children[new YamlScalarNode("metrics")];
var metricsDeserializer = new MetricsDeserializer();
metrics = metricsDeserializer.Deserialize(metricsNode);
}
var metricsDeclaration = new MetricsDeclaration
{
AzureMetadata = azureMetadata,
Metrics = metrics
};
return(metricsDeclaration);
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, ScrapeDefinition <AzureResourceDefinition> metricDefinitionDefinition)
{
_logger.LogInformation("Scraping '{MetricName}' for resource type '{ResourceType}'", metricDefinitionDefinition.PrometheusMetricDefinition.Name, metricDefinitionDefinition.Resource.ResourceType);
var scraper = _metricScraperFactory.CreateScraper(metricDefinitionDefinition.Resource.ResourceType, azureMetadata, _prometheusMetricWriter, _runtimeMetricsCollector);
await scraper.ScrapeAsync(metricDefinitionDefinition);
}
private void PlotAzureMetadataInAsciiTable(AzureMetadata metadata)
{
var asciiTable = CreateAzureMetadataAsciiTable();
asciiTable.AddRow(metadata.TenantId, metadata.SubscriptionId, metadata.ResourceGroupName, metadata.Cloud.Name.Humanize(LetterCasing.Title));
AnsiConsole.Write(asciiTable);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="azureCredentials">Credentials used to authenticate to Microsoft Azure</param>
/// <param name="exceptionTracker">Exception tracker</param>
protected Scraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IExceptionTracker exceptionTracker)
{
Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
_exceptionTracker = exceptionTracker;
AzureMetadata = azureMetadata ?? throw new ArgumentNullException(nameof(azureMetadata));
AzureCredentials = azureCredentials ?? throw new ArgumentNullException(nameof(azureCredentials));
}
private async Task ScrapeMetric(AzureMetadata azureMetadata, MetricDefinition metricDefinitionDefinition)
{
_logger.LogInformation("Scraping '{MetricName}' for resource type '{ResourceType}'", metricDefinitionDefinition.Name, metricDefinitionDefinition.ResourceType);
var scraper = MetricScraperFactory.CreateScraper(metricDefinitionDefinition.ResourceType, azureMetadata, _logger, _exceptionTracker);
int subscriptionReadLimit = await scraper.ScrapeAsync(metricDefinitionDefinition);
HealthMonitor.Instance.setSubscriptionLimitCount(subscriptionReadLimit);
}
public static MetricsDeclarationBuilder WithMetadata(string tenantId = "tenantId", string subscriptionId = "subscriptionId", string resourceGroupName = "resourceGroupName")
{
var azureMetadata = new AzureMetadata
{
TenantId = tenantId,
SubscriptionId = subscriptionId,
ResourceGroupName = resourceGroupName
};
return(new MetricsDeclarationBuilder(azureMetadata));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param>
/// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
/// <param name="logger">General logger</param>
public ScraperConfiguration(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, IPrometheusMetricWriter prometheusMetricWriter, ILogger logger)
{
Guard.NotNull(azureMetadata, nameof(azureMetadata));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
Guard.NotNull(logger, nameof(logger));
AzureMetadata = azureMetadata;
AzureMonitorClient = azureMonitorClient;
PrometheusMetricWriter = prometheusMetricWriter;
Logger = logger;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param>
/// <param name="logger">General logger</param>
/// <param name="exceptionTracker">Exception tracker</param>
protected Scraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger,
IExceptionTracker exceptionTracker)
{
Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
Guard.NotNull(azureMetadata, nameof(azureMetadata));
Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
_logger = logger;
_exceptionTracker = exceptionTracker;
AzureMetadata = azureMetadata;
AzureMonitorClient = azureMonitorClient;
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
internal static IScraper <MetricDefinition> CreateScraper(AzureMetadata azureMetadata, ResourceType metricDefinitionResourceType)
{
var azureCredentials = DetermineAzureCredentials();
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(azureMetadata, azureCredentials));
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="logger">General logger</param>
/// <param name="exceptionTracker">Tracker used to log exceptions</param>
public static IScraper <MetricDefinition> CreateScraper(AzureMetadata azureMetadata, ResourceType metricDefinitionResourceType, ILogger logger, IExceptionTracker exceptionTracker)
{
var azureCredentials = DetermineAzureCredentials();
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(azureMetadata, azureCredentials, logger, exceptionTracker));
case ResourceType.Generic:
return(new GenericScraper(azureMetadata, azureCredentials, logger, exceptionTracker));
default:
throw new ArgumentOutOfRangeException();
}
}
public GenericScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
: base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
{
}
private static AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger)
{
var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, logger);
return(azureMonitorClient);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="azureCredentials">Credentials used to authenticate to Microsoft Azure</param>
protected Scraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials)
{
AzureMetadata = azureMetadata ?? throw new ArgumentNullException(nameof(azureMetadata));
AzureCredentials = azureCredentials ?? throw new ArgumentNullException(nameof(azureCredentials));
}
public ServiceBusQueueScraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials) : base(azureMetadata, azureCredentials)
{
}
public StorageQueueScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
: base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
{
_azureStorageQueueClient = new AzureStorageQueueClient(logger);
}
public ServiceBusQueueScraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IExceptionTracker exceptionTracker)
: base(azureMetadata, azureCredentials, exceptionTracker)
{
}
/// <summary>
/// Creates a <see cref="ScrapeDefinition{TResourceDefinition}"/> object for the specified resource.
/// </summary>
/// <param name="resource">The resource to scrape.</param>
/// <param name="azureMetadata">The Azure global metadata.</param>
/// <returns>The scrape definition.</returns>
public ScrapeDefinition <IAzureResourceDefinition> CreateScrapeDefinition(IAzureResourceDefinition resource, AzureMetadata azureMetadata)
{
return(new ScrapeDefinition <IAzureResourceDefinition>(
AzureMetricConfiguration,
PrometheusMetricDefinition,
Scraping,
resource,
string.IsNullOrEmpty(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId,
string.IsNullOrEmpty(resource.ResourceGroupName) ? azureMetadata.ResourceGroupName : resource.ResourceGroupName));
}
public ContainerRegistryScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
: base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
{
}
public VirtualMachineScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
: base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
{
}
public MetricsDeclarationBuilder(AzureMetadata azureMetadata)
{
_azureMetadata = azureMetadata;
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
public IScraper <AzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector)
{
var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, runtimeMetricsCollector);
var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, prometheusMetricWriter, _logger, _exceptionTracker);
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.AzureSqlDatabase:
return(new AzureSqlDatabaseScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException();
}
}
protected void AssertAzureMetadata(Core.Scraping.Configuration.Model.MetricsDeclaration deserializedConfiguration, AzureMetadata azureMetadata)
{
Assert.NotNull(deserializedConfiguration.AzureMetadata);
Assert.Equal(azureMetadata.TenantId, deserializedConfiguration.AzureMetadata.TenantId);
Assert.Equal(azureMetadata.ResourceGroupName, deserializedConfiguration.AzureMetadata.ResourceGroupName);
Assert.Equal(azureMetadata.SubscriptionId, deserializedConfiguration.AzureMetadata.SubscriptionId);
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
/// <param name="logger">General logger</param>
/// <param name="exceptionTracker">Tracker used to log exceptions</param>
public static IScraper <MetricDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger, IExceptionTracker exceptionTracker)
{
var azureCredentials = DetermineAzureCredentials();
var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, azureCredentials, runtimeMetricsCollector, logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.Generic:
return(new GenericScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker));
default:
throw new ArgumentOutOfRangeException();
}
}
