public override void Init(IDictionary <string, string> jobArgsDictionary) { var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString); _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber); _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform); _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName)); _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly); // construct a cloud blob client for the configured storage account _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient(); _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); // Get the source blob container (containing compressed log files) // and construct a log source (fetching raw logs from the source blob container) var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName); _blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory); }
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary) { base.Init(serviceContainer, jobArgsDictionary); _configuration = _serviceProvider.GetRequiredService <IOptionsSnapshot <ImportAzureCdnStatisticsConfiguration> >().Value; _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform); var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount); _cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); _blobLeaseManager = new LogFileProvider( _cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName), LoggerFactory); }
public override async Task <bool> Run() { try { // construct a cloud blob client for the configured storage account var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient(); cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); // Get the source blob container (containing compressed log files) // and construct a log source (fetching raw logs from the source blob container) var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName); var blobLeaseManager = new LogFileProvider(sourceBlobContainer); // Get the target blob container (for archiving decompressed log files) var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive"); await targetBlobContainer.CreateIfNotExistsAsync(); // Get the dead-letter table (corrupted or failed blobs will end up there) var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter"); // Create a parser var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase); // Get the next to-be-processed raw log file using the cdn raw log file name prefix var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber); // Get next raw log file to be processed var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix); foreach (var leasedLogFile in leasedLogFiles) { await logProcessor.ProcessLogFileAsync(leasedLogFile); leasedLogFile.Dispose(); } return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); return(false); } }
public override async Task<bool> Run() { try { // construct a cloud blob client for the configured storage account var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient(); cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); // Get the source blob container (containing compressed log files) // and construct a log source (fetching raw logs from the source blob container) var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName); var blobLeaseManager = new LogFileProvider(sourceBlobContainer); // Get the target blob container (for archiving decompressed log files) var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive"); await targetBlobContainer.CreateIfNotExistsAsync(); // Get the dead-letter table (corrupted or failed blobs will end up there) var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter"); // Create a parser var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase); // Get the next to-be-processed raw log file using the cdn raw log file name prefix var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber); // Get next raw log file to be processed var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix); foreach (var leasedLogFile in leasedLogFiles) { await logProcessor.ProcessLogFileAsync(leasedLogFile); leasedLogFile.Dispose(); } return true; } catch (Exception exception) { Trace.TraceError(exception.ToString()); return false; } }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(ConsoleLogOnly); _loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration); _logger = _loggerFactory.CreateLogger <Job>(); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString); _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber); _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform); _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName)); _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly); // construct a cloud blob client for the configured storage account _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient(); _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); // Get the source blob container (containing compressed log files) // and construct a log source (fetching raw logs from the source blob container) var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName); _blobLeaseManager = new LogFileProvider(sourceBlobContainer, _loggerFactory); } catch (Exception exception) { _logger.LogCritical(LogEvents.JobInitFailed, exception, "Failed to initialize job!"); return(false); } return(true); }