private async Task ProcessPackageStatisticsInLogFileAsync(CdnStatistics cdnStatistics, string logFileName, bool aggregatesOnly)
{
_logger.LogInformation("Creating facts for package download statistics in {LogFileName}", logFileName);
var downloadFacts = await _warehouse.CreateAsync(cdnStatistics.PackageStatistics, logFileName);
if (downloadFacts != null)
{
// check if we already successfully imported package statistics for this file
if (!await _warehouse.HasImportedPackageStatisticsAsync(logFileName) && !aggregatesOnly)
{
// store facts recorded in this logfile
await _warehouse.InsertDownloadFactsAsync(downloadFacts, logFileName);
}
else
{
_logger.LogWarning("Already imported package download statistics for {LogFileName}: skipping.", logFileName);
}
// create aggregates for the logfile
var logFileAggregates = new LogFileAggregates(logFileName);
foreach (var table in downloadFacts)
{
if (string.Equals(table.TableName, "dbo.Fact_Download", StringComparison.InvariantCultureIgnoreCase))
{
// aggregate download counts by date
var downloadsByDate =
table.AsEnumerable()
.GroupBy(e => e.Field <int>("Dimension_Date_Id"))
.Select(e => new KeyValuePair <int, int>(e.Key, e.Count()));
foreach (var keyValuePair in downloadsByDate)
{
logFileAggregates.PackageDownloadsByDateDimensionId.Add(keyValuePair.Key, keyValuePair.Value);
_logger.LogInformation(
"{LogFile} contains {PackageDownloadCount} package downloads for date id {DimensionDateId}",
logFileName, keyValuePair.Value, keyValuePair.Key);
}
}
}
// store aggregates for this logfile
_logger.LogInformation(
"Storing aggregate facts for package download statistics in {LogFileName}",
logFileName);
await _warehouse.StoreLogFileAggregatesAsync(logFileAggregates);
}
}
private async Task ProcessToolStatisticsInLogFileAsync(CdnStatistics cdnStatistics, string logFileName, bool aggregatesOnly)
{
// check if we already successfully imported tool statistics for this file
if (await _warehouse.HasImportedToolStatisticsAsync(logFileName))
{
_logger.LogWarning(
"Already imported tool download statistics for {LogFileName}: skipping.", logFileName);
}
else
{
_logger.LogInformation("Creating facts for tool download statistics in {LogFileName}", logFileName);
var downloadFacts = await _warehouse.CreateAsync(cdnStatistics.ToolStatistics, logFileName);
// store facts recorded in this logfile
if (downloadFacts != null && !aggregatesOnly)
{
await _warehouse.InsertDownloadFactsAsync(downloadFacts, logFileName);
}
}
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var dnxStatistics = new List <DnxStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
if (logEntry.RequestUrl.Contains("dist.asp.net"))
{
var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
if (dnxInfo != null)
{
dnxStatistics.Add(dnxInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);
return(cdnStatistics);
}
