public async Task DeleteSourceBlobAsync(ILeasedLogFile logFile)
{
if (await logFile.Blob.ExistsAsync())
{
try
{
_logger.LogInformation("Beginning to delete blob {FtpBlobUri}.", logFile.Uri);
var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId);
await logFile.Blob.DeleteAsync(
DeleteSnapshotsOption.IncludeSnapshots,
accessCondition,
options : null,
operationContext : null);
_logger.LogInformation("Finished to delete blob {FtpBlobUri}.", logFile.Uri);
}
catch (Exception exception)
{
_logger.LogError(LogEvents.FailedBlobDelete, exception, "Failed to delete blob {FtpBlobUri}", logFile.Uri);
ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name);
throw;
}
}
}
internal StatisticsBlobContainerUtility(
CloudBlobContainer targetContainer,
CloudBlobContainer deadLetterContainer,
ILoggerFactory loggerFactory,
ApplicationInsightsHelper applicationInsightsHelper)
{
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
_targetContainer = targetContainer ?? throw new ArgumentNullException(nameof(targetContainer));
_deadLetterContainer = deadLetterContainer ?? throw new ArgumentNullException(nameof(deadLetterContainer));
_applicationInsightsHelper = applicationInsightsHelper ?? throw new ArgumentNullException(nameof(applicationInsightsHelper));
_logger = loggerFactory.CreateLogger <StatisticsBlobContainerUtility>();
}
public LogFileProcessor(
IStatisticsBlobContainerUtility statisticsBlobContainerUtility,
ILoggerFactory loggerFactory,
IStatisticsWarehouse warehouse,
ApplicationInsightsHelper applicationInsightsHelper)
{
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
_applicationInsightsHelper = applicationInsightsHelper ?? throw new ArgumentNullException(nameof(applicationInsightsHelper));
_warehouse = warehouse ?? throw new ArgumentNullException(nameof(warehouse));
_statisticsBlobContainerUtility = statisticsBlobContainerUtility ?? throw new ArgumentNullException(nameof(statisticsBlobContainerUtility));
_logger = loggerFactory.CreateLogger <ImportAzureCdnStatisticsJob>();
}
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
{
base.Init(serviceContainer, jobArgsDictionary);
_configuration = _serviceProvider.GetRequiredService <IOptionsSnapshot <ImportAzureCdnStatisticsConfiguration> >().Value;
_azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
_cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
_cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
_blobLeaseManager = new LogFileProvider(
_cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName),
LoggerFactory);
_applicationInsightsHelper = new ApplicationInsightsHelper(ApplicationInsightsConfiguration.TelemetryConfiguration);
}
public async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile)
{
try
{
var stopwatch = Stopwatch.StartNew();
_logger.LogInformation("Beginning opening of compressed blob {FtpBlobUri}.", logFile.Uri);
var memoryStream = new MemoryStream();
// decompress into memory (these are rolling log files and relatively small)
using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null))
{
await blobStream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
stopwatch.Stop();
_logger.LogInformation("Finished opening of compressed blob {FtpBlobUri}.", logFile.Uri);
ApplicationInsightsHelper.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name);
// verify if the stream is gzipped or not
if (await IsGzipCompressedAsync(memoryStream))
{
return(new GZipInputStream(memoryStream));
}
else
{
return(memoryStream);
}
}
catch (Exception exception)
{
_logger.LogError(LogEvents.FailedToDecompressBlob, exception, "Failed to open compressed blob {FtpBlobUri}", logFile.Uri);
ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name);
throw;
}
}
public async Task ArchiveBlobAsync(ILeasedLogFile logFile)
{
try
{
var stopwatch = Stopwatch.StartNew();
await CopyToTargetContainerAsync(logFile);
_logger.LogInformation("Finished archive upload for blob {FtpBlobUri}.", logFile.Uri);
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name);
}
catch (Exception exception)
{
_logger.LogError(LogEvents.FailedBlobUpload, exception, "Failed archive upload for blob {FtpBlobUri}", logFile.Uri);
ApplicationInsightsHelper.TrackException(exception, logFile.Blob.Name);
throw;
}
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName)
{
var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.BlobName;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
var lineNumber = 0;
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
lineNumber++;
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
lineNumber,
rawLogLine,
(e, line) => _logger.LogError(
LogEvents.FailedToParseLogFileEntry,
e,
LogMessages.ParseLogEntryLineFailed,
fileName,
line));
if (logEntry != null)
{
var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
stopwatch.Stop();
_logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count);
ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri);
ApplicationInsightsHelper.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
return(new CdnStatistics(packageStatistics, toolStatistics));
}
