/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="azureMetadata">Metadata concerning the Azure resources</param>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
/// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector)
{
var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, runtimeMetricsCollector);
var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, prometheusMetricWriter, _logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.SqlDatabase:
return(new SqlDatabaseScraper(scraperConfiguration));
case ResourceType.SqlManagedInstance:
return(new SqlManagedInstanceScraper(scraperConfiguration));
case ResourceType.VirtualMachineScaleSet:
return(new VirtualMachineScaleSetScraper(scraperConfiguration));
case ResourceType.AppPlan:
return(new AppPlanScraper(scraperConfiguration));
case ResourceType.WebApp:
return(new WebAppScraper(scraperConfiguration));
case ResourceType.FunctionApp:
return(new FunctionAppScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException();
}
}
public async Task StartScrapingAsync(ScraperConfiguration configuration)
{
if (!Directory.Exists(configuration.ArchiveDirectory))
{
Directory.CreateDirectory(configuration.ArchiveDirectory);
}
using (WebClient client = new WebClient())
{
var csvFile = new StringBuilder();
foreach (DateTime day in DateHelper.EachDay(configuration.StartDate, configuration.EndDate))
{
var currentDayEndpointUri = configuration.Endpoint.Replace("{DATE}", day.ToString("yyyy/MM/dd"));
var currentDayWeatherPageHtml = await client.DownloadStringTaskAsync(currentDayEndpointUri);
HtmlDocument currentDayWeatherPage = new HtmlDocument();
currentDayWeatherPage.LoadHtml(currentDayWeatherPageHtml);
var weatherRows = currentDayWeatherPage.DocumentNode.SelectNodes("//table[starts-with(@id, 'obsTable')]//tr");
if (weatherRows.Count > 0 && day == configuration.StartDate)
{
var headerCells = weatherRows[0].SelectNodes("th");
if (headerCells != null)
{
var csvRow = GetCsvRow(headerCells, day, false);
csvFile.AppendLine(csvRow);
}
}
foreach (var row in weatherRows)
{
var cells = row.SelectNodes("td");
if (cells != null)
{
var csvRow = GetCsvRow(cells, day, true);
csvFile.AppendLine(csvRow);
}
}
Thread.Sleep(configuration.SleepTime);
}
String timeStamp = DateTime.UtcNow.ToString("yyyyMMddHHmmssffff");
var csvFileName = $"{timeStamp}-CsvWeatherDate.csv";
var csvFullName = Path.Combine(configuration.ArchiveDirectory, csvFileName);
File.WriteAllText(csvFullName, csvFile.ToString());
}
}
public async Task StartScrapingAsync(ScraperConfiguration configuration)
{
if (!Directory.Exists(configuration.ArchiveDirectory))
{
Directory.CreateDirectory(configuration.ArchiveDirectory);
}
using (WebClient client = new WebClient())
{
var endpointUri = new Uri(configuration.Endpoint);
var endpointHtml = await client.DownloadStringTaskAsync(endpointUri);
HtmlDocument document = new HtmlDocument();
document.LoadHtml(endpointHtml);
var links = document.DocumentNode.SelectNodes("//a");
foreach (DateTime day in DateHelper.EachDay(configuration.StartDate, configuration.EndDate))
{
var link = document.DocumentNode.SelectSingleNode("//a[starts-with(@href, '" + day.ToString("yyyy-MM-dd") + "')]");
if (link != null)
{
var sensorsUri = new Uri(endpointUri, link.GetAttributeValue("href", string.Empty));
var sensorsHtml = await client.DownloadStringTaskAsync(sensorsUri);
HtmlDocument sensorsPage = new HtmlDocument();
sensorsPage.LoadHtml(sensorsHtml);
var sensorsFileLink = sensorsPage.DocumentNode.SelectNodes("//a[starts-with(@href, '" + day.ToString("yyyy-MM-dd") + "')]");
foreach (var sensorFileLink in sensorsFileLink)
{
var fileName = sensorFileLink.GetAttributeValue("href", string.Empty);
var sensorFileUri = new Uri(sensorsUri, fileName);
await client.DownloadFileTaskAsync(sensorFileUri, Path.Combine(configuration.ArchiveDirectory, fileName));
Thread.Sleep(configuration.SleepTime);
}
}
Thread.Sleep(configuration.SleepTime);
}
}
}
public VirtualMachineScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public KeyVaultScraper(ScraperConfiguration scraperConfiguration) :
base(scraperConfiguration)
{
}
public PostgreSqlScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public CdnScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public IoTHubScraper(ScraperConfiguration scraperConfiguration) :
base(scraperConfiguration)
{
}
public LogicAppScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public SqlManagedInstanceScraper(ScraperConfiguration scraperConfiguration) : base(scraperConfiguration)
{
}
public RedisCacheScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public ApiManagementScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public KubernetesServiceScraper(ScraperConfiguration scraperConfiguration) :
base(scraperConfiguration)
{
}
public NetworkGatewayScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
/// <summary>
/// Creates a scraper that is capable of scraping a specific resource type
/// </summary>
/// <param name="metricDefinitionResourceType">Resource type to scrape</param>
/// <param name="metricSinkWriter">Writer to send metrics to all sinks</param>
/// <param name="azureMonitorClient">Client to interact with Azure Monitor</param>
public IScraper <IAzureResourceDefinition> CreateScraper(ResourceType metricDefinitionResourceType, MetricSinkWriter metricSinkWriter, AzureMonitorClient azureMonitorClient)
{
var scraperConfiguration = new ScraperConfiguration(azureMonitorClient, metricSinkWriter, _logger);
switch (metricDefinitionResourceType)
{
case ResourceType.ApiManagement:
return(new ApiManagementScraper(scraperConfiguration));
case ResourceType.ApplicationGateway:
return(new ApplicationGatewayScraper(scraperConfiguration));
case ResourceType.AppPlan:
return(new AppPlanScraper(scraperConfiguration));
case ResourceType.BlobStorage:
return(new BlobStorageScraper(scraperConfiguration));
case ResourceType.ContainerInstance:
return(new ContainerInstanceScraper(scraperConfiguration));
case ResourceType.ContainerRegistry:
return(new ContainerRegistryScraper(scraperConfiguration));
case ResourceType.CosmosDb:
return(new CosmosDbScraper(scraperConfiguration));
case ResourceType.DeviceProvisioningService:
return(new DeviceProvisioningServiceScraper(scraperConfiguration));
case ResourceType.EventHubs:
return(new EventHubsScraper(scraperConfiguration));
case ResourceType.ExpressRouteCircuit:
return(new ExpressRouteCircuitScraper(scraperConfiguration));
case ResourceType.FileStorage:
return(new FileStorageScraper(scraperConfiguration));
case ResourceType.FunctionApp:
return(new FunctionAppScraper(scraperConfiguration));
case ResourceType.Generic:
return(new GenericScraper(scraperConfiguration));
case ResourceType.IoTHub:
return(new IoTHubScraper(scraperConfiguration));
case ResourceType.KeyVault:
return(new KeyVaultScraper(scraperConfiguration));
case ResourceType.NetworkGateway:
return(new NetworkGatewayScraper(scraperConfiguration));
case ResourceType.NetworkInterface:
return(new NetworkInterfaceScraper(scraperConfiguration));
case ResourceType.LogicApp:
return(new LogicAppScraper(scraperConfiguration));
case ResourceType.PostgreSql:
return(new PostgreSqlScraper(scraperConfiguration));
case ResourceType.RedisCache:
return(new RedisCacheScraper(scraperConfiguration));
case ResourceType.ServiceBusQueue:
return(new ServiceBusQueueScraper(scraperConfiguration));
case ResourceType.SqlDatabase:
return(new SqlDatabaseScraper(scraperConfiguration));
case ResourceType.SqlManagedInstance:
return(new SqlManagedInstanceScraper(scraperConfiguration));
case ResourceType.SqlServer:
return(new SqlServerScraper(scraperConfiguration));
case ResourceType.StorageAccount:
return(new StorageAccountScraper(scraperConfiguration));
case ResourceType.StorageQueue:
return(new StorageQueueScraper(scraperConfiguration));
case ResourceType.VirtualMachine:
return(new VirtualMachineScraper(scraperConfiguration));
case ResourceType.VirtualMachineScaleSet:
return(new VirtualMachineScaleSetScraper(scraperConfiguration));
case ResourceType.WebApp:
return(new WebAppScraper(scraperConfiguration));
default:
throw new ArgumentOutOfRangeException();
}
}
public ServiceBusNamespaceScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public SqlServerScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public FunctionAppScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public StorageQueueScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
_azureStorageQueueClient = new AzureStorageQueueClient(scraperConfiguration.Logger);
}
public StorageAccountScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public NetworkInterfaceScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public ContainerRegistryScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public GenericScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public SynapseApacheSparkPoolScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public CosmosDbScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public AutomationAccountScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public ServiceBusQueueScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
protected AzureMessagingScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public ExpressRouteCircuitScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}
public SqlDatabaseScraper(ScraperConfiguration scraperConfiguration)
: base(scraperConfiguration)
{
}