public async Task DeleteSourceBlobAsync(ILeasedLogFile logFile) { if (await logFile.Blob.ExistsAsync()) { try { _logger.LogInformation("Beginning to delete blob {FtpBlobUri}.", logFile.Uri); var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId); await logFile.Blob.DeleteAsync( DeleteSnapshotsOption.IncludeSnapshots, accessCondition, options : null, operationContext : null); _logger.LogInformation("Finished to delete blob {FtpBlobUri}.", logFile.Uri); } catch (Exception exception) { _logger.LogError(LogEvents.FailedBlobDelete, exception, "Failed to delete blob {FtpBlobUri}", logFile.Uri); _applicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } } }
private static async Task CopyToContainerAsync( ILeasedLogFile logFile, CloudBlobContainer container, Exception e = null) { var archivedBlob = container.GetBlockBlobReference(logFile.Blob.Name); if (!await archivedBlob.ExistsAsync()) { await archivedBlob.StartCopyAsync(logFile.Blob); archivedBlob = (CloudBlockBlob)await container.GetBlobReferenceFromServerAsync(logFile.Blob.Name); while (archivedBlob.CopyState.Status == CopyStatus.Pending) { Task.Delay(TimeSpan.FromSeconds(1)).Wait(); archivedBlob = (CloudBlockBlob)await container.GetBlobReferenceFromServerAsync(logFile.Blob.Name); } await archivedBlob.FetchAttributesAsync(); if (e != null) { // add the job error to the blob's metadata archivedBlob.Metadata[_jobErrorMetadataKey] = e.ToString().Replace("\r\n", string.Empty); await archivedBlob.SetMetadataAsync(); } else if (archivedBlob.Metadata.ContainsKey(_jobErrorMetadataKey)) { archivedBlob.Metadata.Remove(_jobErrorMetadataKey); await archivedBlob.SetMetadataAsync(); } } }
private async Task ArchiveBlobAsync(ILeasedLogFile logFile) { var stopwatch = Stopwatch.StartNew(); try { await EnsureCopiedToContainerAsync(logFile, _targetContainer); _jobEventSource.FinishingArchiveUpload(logFile.Uri); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedArchiveUpload(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } }
public async Task ProcessLogFileAsync(ILeasedLogFile logFile) { if (logFile == null) return; try { var packageStatistics = await ParseLogEntries(logFile); if (packageStatistics.Any()) { // replicate data to the statistics database var warehouse = new Warehouse(_jobEventSource, _targetDatabase); var downloadFacts = await warehouse.CreateAsync(packageStatistics, logFile.Blob.Name); await warehouse.InsertDownloadFactsAsync(downloadFacts, logFile.Blob.Name); } await ArchiveBlobAsync(logFile); // delete the blob from the 'to-be-processed' container await DeleteSourceBlobAsync(logFile); } catch (Exception e) { await _deadLetterContainer.CreateIfNotExistsAsync(); // copy the blob to a dead-letter container await EnsureCopiedToContainerAsync(logFile, _deadLetterContainer, e); // delete the blob from the 'to-be-processed' container await DeleteSourceBlobAsync(logFile); } }
public WhenPackageAndToolStatisticsInLogFile(ITestOutputHelper testOutputHelper) { var loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(new XunitLoggerProvider(testOutputHelper)); _loggerFactory = loggerFactory; _leasedLogFile = GetLeasedLogFileMock(_logFileName); }
public async Task ProcessLogFileAsync(ILeasedLogFile logFile) { if (logFile == null) { return; } try { var cdnStatistics = await ParseLogEntries(logFile); var hasPackageStatistics = cdnStatistics.PackageStatistics.Any(); var hasToolStatistics = cdnStatistics.ToolStatistics.Any(); var hasDnxStatistics = cdnStatistics.DnxStatistics.Any(); if (hasPackageStatistics || hasToolStatistics || hasDnxStatistics) { // replicate data to the statistics database var warehouse = new Warehouse(_jobEventSource, _targetDatabase); if (hasPackageStatistics) { var downloadFacts = await warehouse.CreateAsync(cdnStatistics.PackageStatistics, logFile.Blob.Name); await warehouse.InsertDownloadFactsAsync(downloadFacts, logFile.Blob.Name); } if (hasToolStatistics) { var downloadFacts = await warehouse.CreateAsync(cdnStatistics.ToolStatistics, logFile.Blob.Name); await warehouse.InsertDownloadFactsAsync(downloadFacts, logFile.Blob.Name); } if (hasDnxStatistics) { var downloadFacts = await warehouse.CreateAsync(cdnStatistics.DnxStatistics, logFile.Blob.Name); await warehouse.InsertDownloadFactsAsync(downloadFacts, logFile.Blob.Name); } } await ArchiveBlobAsync(logFile); // delete the blob from the 'to-be-processed' container await DeleteSourceBlobAsync(logFile); } catch (Exception e) { await _deadLetterContainer.CreateIfNotExistsAsync(); // copy the blob to a dead-letter container await EnsureCopiedToContainerAsync(logFile, _deadLetterContainer, e); // delete the blob from the 'to-be-processed' container await DeleteSourceBlobAsync(logFile); } }
public WhenPackageAndToolStatisticsInLogFile(ITestOutputHelper testOutputHelper) { var loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(new XunitLoggerProvider(testOutputHelper)); _loggerFactory = loggerFactory; _leasedLogFile = GetLeasedLogFileMock(_logFileName); _applicationInsightsHelper = new ApplicationInsightsHelper(TelemetryConfiguration.CreateDefault()); }
public async Task ProcessLogFileAsync(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, bool aggregatesOnly = false) { if (logFile == null) { return; } try { var logFileName = logFile.BlobName; var cdnStatistics = await ParseLogEntries(logFile, packageStatisticsParser, logFileName); var hasPackageStatistics = cdnStatistics.PackageStatistics.Any(); var hasToolStatistics = cdnStatistics.ToolStatistics.Any(); // replicate data to the statistics database if (hasPackageStatistics) { await ProcessPackageStatisticsInLogFileAsync(cdnStatistics, logFileName, aggregatesOnly); } if (hasToolStatistics) { await ProcessToolStatisticsInLogFileAsync(cdnStatistics, logFileName, aggregatesOnly); } if (!aggregatesOnly) { await _statisticsBlobContainerUtility.ArchiveBlobAsync(logFile); } } catch (Exception e) { _logger.LogError(LogEvents.FailedToProcessLogFile, e, "Unable to process {LogFile}", logFile.Uri); if (!aggregatesOnly) { // copy the blob to a dead-letter container await _statisticsBlobContainerUtility.CopyToDeadLetterContainerAsync(logFile, e); } } if (!aggregatesOnly) { // delete the blob from the 'to-be-processed' container await _statisticsBlobContainerUtility.DeleteSourceBlobAsync(logFile); } }
private async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile) { var stopwatch = Stopwatch.StartNew(); try { _jobEventSource.BeginningOpenCompressedBlob(logFile.Uri); var memoryStream = new MemoryStream(); // decompress into memory (these are rolling log files and relatively small) using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null)) { await blobStream.CopyToAsync(memoryStream); memoryStream.Position = 0; } stopwatch.Stop(); _jobEventSource.FinishedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); // verify if the stream is gzipped or not if (await IsGzipCompressed(memoryStream)) { return(new GZipInputStream(memoryStream)); } else { return(memoryStream); } } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } }
public async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile) { try { var stopwatch = Stopwatch.StartNew(); _logger.LogInformation("Beginning opening of compressed blob {FtpBlobUri}.", logFile.Uri); var memoryStream = new MemoryStream(); // decompress into memory (these are rolling log files and relatively small) using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null)) { await blobStream.CopyToAsync(memoryStream); memoryStream.Position = 0; } stopwatch.Stop(); _logger.LogInformation("Finished opening of compressed blob {FtpBlobUri}.", logFile.Uri); _applicationInsightsHelper.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); // verify if the stream is gzipped or not if (await IsGzipCompressedAsync(memoryStream)) { return(new GZipInputStream(memoryStream)); } else { return(memoryStream); } } catch (Exception exception) { _logger.LogError(LogEvents.FailedToDecompressBlob, exception, "Failed to open compressed blob {FtpBlobUri}", logFile.Uri); _applicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } }
private async Task DeleteSourceBlobAsync(ILeasedLogFile logFile) { if (await logFile.Blob.ExistsAsync()) { try { _jobEventSource.BeginningDelete(logFile.Uri); var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId); await logFile.Blob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, accessCondition, null, null); _jobEventSource.FinishedDelete(logFile.Uri); } catch (Exception exception) { _jobEventSource.FailedDelete(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } } }
public async Task ArchiveBlobAsync(ILeasedLogFile logFile) { try { var stopwatch = Stopwatch.StartNew(); await CopyToTargetContainerAsync(logFile); _logger.LogInformation("Finished archive upload for blob {FtpBlobUri}.", logFile.Uri); stopwatch.Stop(); _applicationInsightsHelper.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); } catch (Exception exception) { _logger.LogError(LogEvents.FailedBlobUpload, exception, "Failed archive upload for blob {FtpBlobUri}", logFile.Uri); _applicationInsightsHelper.TrackException(exception, logFile.Blob.Name); throw; } }
private static async Task EnsureCopiedToContainerAsync(ILeasedLogFile logFile, CloudBlobContainer targetContainer, Exception e = null) { var archivedBlob = targetContainer.GetBlockBlobReference(logFile.Blob.Name); if (!await archivedBlob.ExistsAsync()) { await archivedBlob.StartCopyFromBlobAsync(logFile.Blob); archivedBlob = (CloudBlockBlob)await targetContainer.GetBlobReferenceFromServerAsync(logFile.Blob.Name); while (archivedBlob.CopyState.Status == CopyStatus.Pending) { Task.Delay(TimeSpan.FromSeconds(1)).Wait(); archivedBlob = (CloudBlockBlob)await targetContainer.GetBlobReferenceFromServerAsync(logFile.Blob.Name); } await archivedBlob.FetchAttributesAsync(); if (e != null) { // add the job error to the blob's metadata if (archivedBlob.Metadata.ContainsKey("JobError")) { archivedBlob.Metadata["JobError"] = e.ToString().Replace("\r\n", string.Empty); } else { archivedBlob.Metadata.Add("JobError", e.ToString().Replace("\r\n", string.Empty)); } await archivedBlob.SetMetadataAsync(); } else if (archivedBlob.Metadata.ContainsKey("JobError")) { archivedBlob.Metadata.Remove("JobError"); await archivedBlob.SetMetadataAsync(); } } }
private async Task <IReadOnlyCollection <PackageStatistics> > ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } return(packageStatistics); }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName) { var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.BlobName; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri); using (var logStreamReader = new StreamReader(logStream)) { var lineNumber = 0; do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { lineNumber++; var logEntry = CdnLogEntryParser.ParseLogEntryFromLine( lineNumber, rawLogLine, (e, line) => _logger.LogError( LogEvents.FailedToParseLogFileEntry, e, LogMessages.ParseLogEntryLineFailed, fileName, line)); if (logEntry != null) { var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } } } } } while (!logStreamReader.EndOfStream); } stopwatch.Stop(); _logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count); ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri); ApplicationInsightsHelper.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } return(new CdnStatistics(packageStatistics, toolStatistics)); }
public async Task CopyToDeadLetterContainerAsync(ILeasedLogFile logFile, Exception e) { await CopyToContainerAsync(logFile, _deadLetterContainer, e); }
private async Task<CdnStatistics> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List<PackageStatistics>(); var toolStatistics = new List<ToolStatistics>(); var dnxStatistics = new List<DnxStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } if (logEntry.RequestUrl.Contains("dist.asp.net")) { var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry); if (dnxInfo != null) { dnxStatistics.Add(dnxInfo); } } } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics); return cdnStatistics; }
private async Task CopyToTargetContainerAsync(ILeasedLogFile logFile) { await CopyToContainerAsync(logFile, _targetContainer); }
private async Task<IReadOnlyCollection<PackageStatistics>> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List<PackageStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } return packageStatistics; }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var dnxStatistics = new List <DnxStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } if (logEntry.RequestUrl.Contains("dist.asp.net")) { var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry); if (dnxInfo != null) { dnxStatistics.Add(dnxInfo); } } } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics); return(cdnStatistics); }
private async Task<Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile) { var stopwatch = Stopwatch.StartNew(); try { _jobEventSource.BeginningOpenCompressedBlob(logFile.Uri); var memoryStream = new MemoryStream(); // decompress into memory (these are rolling log files and relatively small) using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null)) { await blobStream.CopyToAsync(memoryStream); memoryStream.Position = 0; } stopwatch.Stop(); _jobEventSource.FinishedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); return new GZipInputStream(memoryStream); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } }