private async Task ProcessPackageStatisticsInLogFileAsync(CdnStatistics cdnStatistics, string logFileName, bool aggregatesOnly) { _logger.LogInformation("Creating facts for package download statistics in {LogFileName}", logFileName); var downloadFacts = await _warehouse.CreateAsync(cdnStatistics.PackageStatistics, logFileName); if (downloadFacts != null) { // check if we already successfully imported package statistics for this file if (!await _warehouse.HasImportedPackageStatisticsAsync(logFileName) && !aggregatesOnly) { // store facts recorded in this logfile await _warehouse.InsertDownloadFactsAsync(downloadFacts, logFileName); } else { _logger.LogWarning("Already imported package download statistics for {LogFileName}: skipping.", logFileName); } // create aggregates for the logfile var logFileAggregates = new LogFileAggregates(logFileName); foreach (var table in downloadFacts) { if (string.Equals(table.TableName, "dbo.Fact_Download", StringComparison.InvariantCultureIgnoreCase)) { // aggregate download counts by date var downloadsByDate = table.AsEnumerable() .GroupBy(e => e.Field <int>("Dimension_Date_Id")) .Select(e => new KeyValuePair <int, int>(e.Key, e.Count())); foreach (var keyValuePair in downloadsByDate) { logFileAggregates.PackageDownloadsByDateDimensionId.Add(keyValuePair.Key, keyValuePair.Value); _logger.LogInformation( "{LogFile} contains {PackageDownloadCount} package downloads for date id {DimensionDateId}", logFileName, keyValuePair.Value, keyValuePair.Key); } } } // store aggregates for this logfile _logger.LogInformation( "Storing aggregate facts for package download statistics in {LogFileName}", logFileName); await _warehouse.StoreLogFileAggregatesAsync(logFileAggregates); } }
private async Task ProcessToolStatisticsInLogFileAsync(CdnStatistics cdnStatistics, string logFileName, bool aggregatesOnly) { // check if we already successfully imported tool statistics for this file if (await _warehouse.HasImportedToolStatisticsAsync(logFileName)) { _logger.LogWarning( "Already imported tool download statistics for {LogFileName}: skipping.", logFileName); } else { _logger.LogInformation("Creating facts for tool download statistics in {LogFileName}", logFileName); var downloadFacts = await _warehouse.CreateAsync(cdnStatistics.ToolStatistics, logFileName); // store facts recorded in this logfile if (downloadFacts != null && !aggregatesOnly) { await _warehouse.InsertDownloadFactsAsync(downloadFacts, logFileName); } } }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var dnxStatistics = new List <DnxStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } if (logEntry.RequestUrl.Contains("dist.asp.net")) { var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry); if (dnxInfo != null) { dnxStatistics.Add(dnxInfo); } } } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics); return(cdnStatistics); }