Example #1
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
            var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
            var databaseConnectionString            = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);

            _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);

            _targetDatabase            = new SqlConnectionStringBuilder(databaseConnectionString);
            _azureCdnAccountNumber     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
            _azureCdnPlatform          = ValidateAzureCdnPlatform(azureCdnPlatform);
            _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));

            _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);

            // construct a cloud blob client for the configured storage account
            _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
            _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);

            // Get the source blob container (containing compressed log files)
            // and construct a log source (fetching raw logs from the source blob container)
            var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);

            _blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory);
        }
        public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
        {
            base.Init(serviceContainer, jobArgsDictionary);

            _configuration = _serviceProvider.GetRequiredService <IOptionsSnapshot <ImportAzureCdnStatisticsConfiguration> >().Value;

            _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);

            var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);

            _cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
            _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);

            _blobLeaseManager = new LogFileProvider(
                _cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName),
                LoggerFactory);
        }
Example #3
0
        public override async Task <bool> Run()
        {
            try
            {
                // construct a cloud blob client for the configured storage account
                var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
                cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);

                // Get the source blob container (containing compressed log files)
                // and construct a log source (fetching raw logs from the source blob container)
                var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
                var blobLeaseManager    = new LogFileProvider(sourceBlobContainer);

                // Get the target blob container (for archiving decompressed log files)
                var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
                await targetBlobContainer.CreateIfNotExistsAsync();

                // Get the dead-letter table (corrupted or failed blobs will end up there)
                var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");

                // Create a parser
                var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase);

                // Get the next to-be-processed raw log file using the cdn raw log file name prefix
                var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);

                // Get next raw log file to be processed
                var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix);

                foreach (var leasedLogFile in leasedLogFiles)
                {
                    await logProcessor.ProcessLogFileAsync(leasedLogFile);

                    leasedLogFile.Dispose();
                }

                return(true);
            }
            catch (Exception exception)
            {
                Trace.TraceError(exception.ToString());
                return(false);
            }
        }
Example #4
0
        public override async Task<bool> Run()
        {
            try
            {
                // construct a cloud blob client for the configured storage account
                var cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
                cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);

                // Get the source blob container (containing compressed log files)
                // and construct a log source (fetching raw logs from the source blob container)
                var sourceBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
                var blobLeaseManager = new LogFileProvider(sourceBlobContainer);

                // Get the target blob container (for archiving decompressed log files)
                var targetBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
                await targetBlobContainer.CreateIfNotExistsAsync();

                // Get the dead-letter table (corrupted or failed blobs will end up there)
                var deadLetterBlobContainer = cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");

                // Create a parser
                var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _targetDatabase);

                // Get the next to-be-processed raw log file using the cdn raw log file name prefix
                var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);

                // Get next raw log file to be processed
                var leasedLogFiles = await blobLeaseManager.LeaseNextLogFilesToBeProcessedAsync(prefix);
                foreach (var leasedLogFile in leasedLogFiles)
                {
                    await logProcessor.ProcessLogFileAsync(leasedLogFile);
                    leasedLogFile.Dispose();
                }

                return true;
            }
            catch (Exception exception)
            {
                Trace.TraceError(exception.ToString());
                return false;
            }
        }
Example #5
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                ApplicationInsights.Initialize(instrumentationKey);

                var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(ConsoleLogOnly);
                _loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration);
                _logger        = _loggerFactory.CreateLogger <Job>();

                var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
                var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
                var databaseConnectionString            = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
                _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);

                _targetDatabase            = new SqlConnectionStringBuilder(databaseConnectionString);
                _azureCdnAccountNumber     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
                _azureCdnPlatform          = ValidateAzureCdnPlatform(azureCdnPlatform);
                _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));

                _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);

                // construct a cloud blob client for the configured storage account
                _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
                _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);

                // Get the source blob container (containing compressed log files)
                // and construct a log source (fetching raw logs from the source blob container)
                var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
                _blobLeaseManager = new LogFileProvider(sourceBlobContainer, _loggerFactory);
            }
            catch (Exception exception)
            {
                _logger.LogCritical(LogEvents.JobInitFailed, exception, "Failed to initialize job!");

                return(false);
            }

            return(true);
        }