public async Task ProcessLogFileAsync(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, bool aggregatesOnly = false)
{
if (logFile == null)
{
return;
}
try
{
var logFileName = logFile.BlobName;
var cdnStatistics = await ParseLogEntries(logFile, packageStatisticsParser, logFileName);
var hasPackageStatistics = cdnStatistics.PackageStatistics.Any();
var hasToolStatistics = cdnStatistics.ToolStatistics.Any();
// replicate data to the statistics database
if (hasPackageStatistics)
{
await ProcessPackageStatisticsInLogFileAsync(cdnStatistics, logFileName, aggregatesOnly);
}
if (hasToolStatistics)
{
await ProcessToolStatisticsInLogFileAsync(cdnStatistics, logFileName, aggregatesOnly);
}
if (!aggregatesOnly)
{
await _statisticsBlobContainerUtility.ArchiveBlobAsync(logFile);
}
}
catch (Exception e)
{
_logger.LogError(LogEvents.FailedToProcessLogFile, e, "Unable to process {LogFile}", logFile.Uri);
if (!aggregatesOnly)
{
// copy the blob to a dead-letter container
await _statisticsBlobContainerUtility.CopyToDeadLetterContainerAsync(logFile, e);
}
}
if (!aggregatesOnly)
{
// delete the blob from the 'to-be-processed' container
await _statisticsBlobContainerUtility.DeleteSourceBlobAsync(logFile);
}
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName)
{
var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.BlobName;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
var lineNumber = 0;
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
lineNumber++;
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
lineNumber,
rawLogLine,
(e, line) => _logger.LogError(
LogEvents.FailedToParseLogFileEntry,
e,
LogMessages.ParseLogEntryLineFailed,
fileName,
line));
if (logEntry != null)
{
var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
stopwatch.Stop();
_logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count);
ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri);
ApplicationInsightsHelper.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
return(new CdnStatistics(packageStatistics, toolStatistics));
}
