public override void Init(IDictionary <string, string> jobArgsDictionary)
{
var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);
_targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString);
_azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
_azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
_cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));
_aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);
// construct a cloud blob client for the configured storage account
_cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
_cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
// Get the source blob container (containing compressed log files)
// and construct a log source (fetching raw logs from the source blob container)
var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
_blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory);
}
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
{
base.Init(serviceContainer, jobArgsDictionary);
_configuration = _serviceProvider.GetRequiredService <IOptionsSnapshot <ImportAzureCdnStatisticsConfiguration> >().Value;
_azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
_cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
_cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
_blobLeaseManager = new LogFileProvider(
_cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName),
LoggerFactory);
}
public override async Task <bool> Run()
{
try
{
// construct a cloud blob client for the configured storage account
var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
// Get the source blob container (containing compressed log files)
// and construct a log source (fetching raw logs from the source blob container)
var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
var blobLeaseManager = new LogFileProvider(sourceBlobContainer);
// Get the target blob container (for archiving decompressed log files)
var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
await targetBlobContainer.CreateIfNotExistsAsync();
// Get the dead-letter table (corrupted or failed blobs will end up there)
var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");
// Create a parser
var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase);
// Get the next to-be-processed raw log file using the cdn raw log file name prefix
var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);
// Get next raw log file to be processed
var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix);
foreach (var leasedLogFile in leasedLogFiles)
{
await logProcessor.ProcessLogFileAsync(leasedLogFile);
leasedLogFile.Dispose();
}
return(true);
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
return(false);
}
}
public override async Task<bool> Run()
{
try
{
// construct a cloud blob client for the configured storage account
var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
// Get the source blob container (containing compressed log files)
// and construct a log source (fetching raw logs from the source blob container)
var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
var blobLeaseManager = new LogFileProvider(sourceBlobContainer);
// Get the target blob container (for archiving decompressed log files)
var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
await targetBlobContainer.CreateIfNotExistsAsync();
// Get the dead-letter table (corrupted or failed blobs will end up there)
var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");
// Create a parser
var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase);
// Get the next to-be-processed raw log file using the cdn raw log file name prefix
var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);
// Get next raw log file to be processed
var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix);
foreach (var leasedLogFile in leasedLogFiles)
{
await logProcessor.ProcessLogFileAsync(leasedLogFile);
leasedLogFile.Dispose();
}
return true;
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
return false;
}
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
try
{
var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
ApplicationInsights.Initialize(instrumentationKey);
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(ConsoleLogOnly);
_loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration);
_logger = _loggerFactory.CreateLogger <Job>();
var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);
_targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString);
_azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
_azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
_cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));
_aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);
// construct a cloud blob client for the configured storage account
_cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
_cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
// Get the source blob container (containing compressed log files)
// and construct a log source (fetching raw logs from the source blob container)
var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
_blobLeaseManager = new LogFileProvider(sourceBlobContainer, _loggerFactory);
}
catch (Exception exception)
{
_logger.LogCritical(LogEvents.JobInitFailed, exception, "Failed to initialize job!");
return(false);
}
return(true);
}
