コード例 #1
0
        private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName)
        {
            var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile);

            var blobUri  = logFile.Uri;
            var blobName = logFile.BlobName;

            var packageStatistics = new List <PackageStatistics>();
            var toolStatistics    = new List <ToolStatistics>();

            var stopwatch = Stopwatch.StartNew();

            try
            {
                // parse the log into table entities
                _logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri);

                using (var logStreamReader = new StreamReader(logStream))
                {
                    var lineNumber = 0;
                    do
                    {
                        var rawLogLine = logStreamReader.ReadLine();
                        if (rawLogLine != null)
                        {
                            lineNumber++;

                            var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
                                lineNumber,
                                rawLogLine,
                                (e, line) => _logger.LogError(
                                    LogEvents.FailedToParseLogFileEntry,
                                    e,
                                    LogMessages.ParseLogEntryLineFailed,
                                    fileName,
                                    line));

                            if (logEntry != null)
                            {
                                var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry);
                                if (statistic != null)
                                {
                                    packageStatistics.Add(statistic);
                                }
                                else
                                {
                                    // check if this is a dist.nuget.org download
                                    if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
                                    {
                                        var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
                                        if (toolInfo != null)
                                        {
                                            toolStatistics.Add(toolInfo);
                                        }
                                    }
                                }
                            }
                        }
                    } while (!logStreamReader.EndOfStream);
                }

                stopwatch.Stop();

                _logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count);
                ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
            }
            catch (Exception exception)
            {
                if (stopwatch.IsRunning)
                {
                    stopwatch.Stop();
                }

                _logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri);
                ApplicationInsightsHelper.TrackException(exception, blobName);

                throw;
            }
            finally
            {
                logStream.Dispose();
            }

            return(new CdnStatistics(packageStatistics, toolStatistics));
        }
コード例 #2
0
        private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
        {
            var logStream = await OpenCompressedBlobAsync(logFile);

            var blobUri  = logFile.Uri;
            var blobName = logFile.Blob.Name;

            var packageStatistics = new List <PackageStatistics>();
            var toolStatistics    = new List <ToolStatistics>();
            var dnxStatistics     = new List <DnxStatistics>();

            var stopwatch = Stopwatch.StartNew();

            try
            {
                // parse the log into table entities
                _jobEventSource.BeginningParseLog(blobUri);

                using (var logStreamReader = new StreamReader(logStream))
                {
                    do
                    {
                        var rawLogLine = logStreamReader.ReadLine();
                        if (rawLogLine != null)
                        {
                            var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
                            if (logEntry != null)
                            {
                                var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
                                if (statistic != null)
                                {
                                    packageStatistics.Add(statistic);
                                }
                                else
                                {
                                    // check if this is a dist.nuget.org download
                                    if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
                                    {
                                        var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
                                        if (toolInfo != null)
                                        {
                                            toolStatistics.Add(toolInfo);
                                        }
                                    }
                                    if (logEntry.RequestUrl.Contains("dist.asp.net"))
                                    {
                                        var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
                                        if (dnxInfo != null)
                                        {
                                            dnxStatistics.Add(dnxInfo);
                                        }
                                    }
                                }
                            }
                        }
                    } while (!logStreamReader.EndOfStream);
                }

                _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);

                stopwatch.Stop();
                ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
            }
            catch (Exception exception)
            {
                if (stopwatch.IsRunning)
                {
                    stopwatch.Stop();
                }

                _jobEventSource.FailedParseLog(blobUri);
                ApplicationInsights.TrackException(exception, blobName);
                throw;
            }
            finally
            {
                logStream.Dispose();
            }

            var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);

            return(cdnStatistics);
        }