public async Task DeleteSourceBlobAsync(ILeasedLogFile logFile) { if (await logFile.Blob.ExistsAsync()) { try { _logger.LogInformation("Beginning to delete blob {FtpBlobUri}.", logFile.Uri); var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId); await logFile.Blob.DeleteAsync( DeleteSnapshotsOption.IncludeSnapshots, accessCondition, options : null, operationContext : null); _logger.LogInformation("Finished to delete blob {FtpBlobUri}.", logFile.Uri); } catch (Exception exception) { _logger.LogError(LogEvents.FailedBlobDelete, exception, "Failed to delete blob {FtpBlobUri}", logFile.Uri); ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } } }
internal StatisticsBlobContainerUtility( CloudBlobContainer targetContainer, CloudBlobContainer deadLetterContainer, ILoggerFactory loggerFactory, ApplicationInsightsHelper applicationInsightsHelper) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } _targetContainer = targetContainer ?? throw new ArgumentNullException(nameof(targetContainer)); _deadLetterContainer = deadLetterContainer ?? throw new ArgumentNullException(nameof(deadLetterContainer)); _applicationInsightsHelper = applicationInsightsHelper ?? throw new ArgumentNullException(nameof(applicationInsightsHelper)); _logger = loggerFactory.CreateLogger <StatisticsBlobContainerUtility>(); }
public LogFileProcessor( IStatisticsBlobContainerUtility statisticsBlobContainerUtility, ILoggerFactory loggerFactory, IStatisticsWarehouse warehouse, ApplicationInsightsHelper applicationInsightsHelper) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } _applicationInsightsHelper = applicationInsightsHelper ?? throw new ArgumentNullException(nameof(applicationInsightsHelper)); _warehouse = warehouse ?? throw new ArgumentNullException(nameof(warehouse)); _statisticsBlobContainerUtility = statisticsBlobContainerUtility ?? throw new ArgumentNullException(nameof(statisticsBlobContainerUtility)); _logger = loggerFactory.CreateLogger <ImportAzureCdnStatisticsJob>(); }
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary) { base.Init(serviceContainer, jobArgsDictionary); _configuration = _serviceProvider.GetRequiredService <IOptionsSnapshot <ImportAzureCdnStatisticsConfiguration> >().Value; _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform); var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount); _cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); _blobLeaseManager = new LogFileProvider( _cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName), LoggerFactory); _applicationInsightsHelper = new ApplicationInsightsHelper(ApplicationInsightsConfiguration.TelemetryConfiguration); }
public async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile) { try { var stopwatch = Stopwatch.StartNew(); _logger.LogInformation("Beginning opening of compressed blob {FtpBlobUri}.", logFile.Uri); var memoryStream = new MemoryStream(); // decompress into memory (these are rolling log files and relatively small) using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null)) { await blobStream.CopyToAsync(memoryStream); memoryStream.Position = 0; } stopwatch.Stop(); _logger.LogInformation("Finished opening of compressed blob {FtpBlobUri}.", logFile.Uri); ApplicationInsightsHelper.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); // verify if the stream is gzipped or not if (await IsGzipCompressedAsync(memoryStream)) { return(new GZipInputStream(memoryStream)); } else { return(memoryStream); } } catch (Exception exception) { _logger.LogError(LogEvents.FailedToDecompressBlob, exception, "Failed to open compressed blob {FtpBlobUri}", logFile.Uri); ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } }
public async Task ArchiveBlobAsync(ILeasedLogFile logFile) { try { var stopwatch = Stopwatch.StartNew(); await CopyToTargetContainerAsync(logFile); _logger.LogInformation("Finished archive upload for blob {FtpBlobUri}.", logFile.Uri); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); } catch (Exception exception) { _logger.LogError(LogEvents.FailedBlobUpload, exception, "Failed archive upload for blob {FtpBlobUri}", logFile.Uri); ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName) { var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.BlobName; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri); using (var logStreamReader = new StreamReader(logStream)) { var lineNumber = 0; do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { lineNumber++; var logEntry = CdnLogEntryParser.ParseLogEntryFromLine( lineNumber, rawLogLine, (e, line) => _logger.LogError( LogEvents.FailedToParseLogFileEntry, e, LogMessages.ParseLogEntryLineFailed, fileName, line)); if (logEntry != null) { var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } } } } } while (!logStreamReader.EndOfStream); } stopwatch.Stop(); _logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count); ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri); ApplicationInsightsHelper.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } return(new CdnStatistics(packageStatistics, toolStatistics)); }