private async Task <IReadOnlyCollection <PackageStatistics> > ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
return(packageStatistics);
}
private static string GetParsedModifiedLogEntry(string rawLogEntry)
{
var parsedEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogEntry);
if (parsedEntry == null)
{
return(null);
}
const string spaceCharacter = " ";
const string dashCharacter = "-";
var stringBuilder = new StringBuilder();
// timestamp
stringBuilder.Append(ToUnixTimeStamp(parsedEntry.EdgeServerTimeDelivered) + spaceCharacter);
// time-taken
stringBuilder.Append((parsedEntry.EdgeServerTimeTaken.HasValue ? parsedEntry.EdgeServerTimeTaken.Value.ToString() : dashCharacter) + spaceCharacter);
// REMOVE c-ip
stringBuilder.Append(dashCharacter + spaceCharacter);
// filesize
stringBuilder.Append((parsedEntry.FileSize.HasValue ? parsedEntry.FileSize.Value.ToString() : dashCharacter) + spaceCharacter);
// s-ip
stringBuilder.Append((parsedEntry.EdgeServerIpAddress ?? dashCharacter) + spaceCharacter);
// s-port
stringBuilder.Append((parsedEntry.EdgeServerPort.HasValue ? parsedEntry.EdgeServerPort.Value.ToString() : dashCharacter) + spaceCharacter);
// sc-status
stringBuilder.Append((parsedEntry.CacheStatusCode ?? dashCharacter) + spaceCharacter);
// sc-bytes
stringBuilder.Append((parsedEntry.EdgeServerBytesSent.HasValue ? parsedEntry.EdgeServerBytesSent.Value.ToString() : dashCharacter) + spaceCharacter);
// cs-method
stringBuilder.Append((parsedEntry.HttpMethod ?? dashCharacter) + spaceCharacter);
// cs-uri-stem
stringBuilder.Append((parsedEntry.RequestUrl ?? dashCharacter) + spaceCharacter);
// -
stringBuilder.Append(dashCharacter + spaceCharacter);
// rs-duration
stringBuilder.Append((parsedEntry.RemoteServerTimeTaken.HasValue ? parsedEntry.RemoteServerTimeTaken.Value.ToString() : dashCharacter) + spaceCharacter);
// rs-bytes
stringBuilder.Append((parsedEntry.RemoteServerBytesSent.HasValue ? parsedEntry.RemoteServerBytesSent.Value.ToString() : dashCharacter) + spaceCharacter);
// c-referrer
stringBuilder.Append((parsedEntry.Referrer ?? dashCharacter) + spaceCharacter);
// c-user-agent
stringBuilder.Append((parsedEntry.UserAgent ?? dashCharacter) + spaceCharacter);
// customer-id
stringBuilder.Append((parsedEntry.CustomerId ?? dashCharacter) + spaceCharacter);
// x-ec_custom-1
stringBuilder.AppendLine((parsedEntry.CustomField ?? dashCharacter) + spaceCharacter);
return(stringBuilder.ToString());
}
public void IgnoresLinesWithIncompleteDataWithOnErrorCallback()
{
// Act
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
IncompleteDataLine,
IndexOutOfRangeExceptionOnError);
// Assert
Assert.Null(logEntry);
}
public void ThrowsExceptionForLinesWithIncorrectFormatDataWithoutOnErrorCallback()
{
// Act/Assert
Assert.Throws <FormatException>(() =>
{
CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
IncorrectFormatDataLine,
null);
});
}
public void IgnoresLinesWithIncorrectFormatDataWithOnErrorCallback()
{
// Act
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
IncorrectFormatDataLine,
FormatExceptionOnError);
// Assert
Assert.Null(logEntry);
}
public void ThrowsExceptionForLinesWithIncompleteDataWithoutOnErrorCallback()
{
// Act/Assert
Assert.Throws <IndexOutOfRangeException>(() =>
{
CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
IncompleteDataLine,
null);
});
}
public void IgnoresNon200HttpStatusCodes(string status)
{
// Arrange
var line = string.Format(StatusLineFormat, status);
// Act
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
line,
FailOnError);
// Assert
Assert.Null(logEntry);
}
public void DoesNotIgnore200LevelAndUnrecognizedHttpStatusCodes(string status)
{
// Arrange
var line = string.Format(StatusLineFormat, status);
// Act
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
line,
(e, lineNumber) => Assert.False(true, "The error action should not be called."));
// Assert
Assert.NotNull(logEntry);
Assert.Equal(status, logEntry.CacheStatusCode);
}
public void DoesNotIgnore200LevelAndUnrecognizedHttpStatusCodes(string status)
{
// Arrange
var line = string.Format(StatusLineFormat, status);
// Act
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
LineNumber,
line,
FailOnError);
// Assert
Assert.NotNull(logEntry);
Assert.Equal(status, logEntry.CacheStatusCode);
}
public void CdnLogEntryParserIntegration(string input)
{
var collector = new ChinaStatsCollector(
Mock.Of <ILogSource>(),
Mock.Of <ILogDestination>(),
Mock.Of <ILogger <ChinaStatsCollector> >());
var tranformedInput = collector.TransformRawLogLine(input);
if (tranformedInput == null)
{
return;
}
string output = tranformedInput.ToString();
const int lineNumber = 1;
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(lineNumber, output, onErrorAction: null);
Assert.Contains(tranformedInput.XEc_Custom_1, logEntry.CustomField);
Assert.Contains(tranformedInput.CUserAgent, logEntry.UserAgent);
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, string fileName)
{
var logStream = await _statisticsBlobContainerUtility.OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.BlobName;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_logger.LogInformation("Beginning to parse blob {FtpBlobUri}.", blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
var lineNumber = 0;
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
lineNumber++;
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(
lineNumber,
rawLogLine,
(e, line) => _logger.LogError(
LogEvents.FailedToParseLogFileEntry,
e,
LogMessages.ParseLogEntryLineFailed,
fileName,
line));
if (logEntry != null)
{
var statistic = packageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
stopwatch.Stop();
_logger.LogInformation("Finished parsing blob {FtpBlobUri} ({RecordCount} records).", blobUri, packageStatistics.Count);
ApplicationInsightsHelper.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_logger.LogError(LogEvents.FailedToParseLogFile, exception, "Failed to parse blob {FtpBlobUri}.", blobUri);
ApplicationInsightsHelper.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
return(new CdnStatistics(packageStatistics, toolStatistics));
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var dnxStatistics = new List <DnxStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
if (logEntry.RequestUrl.Contains("dist.asp.net"))
{
var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
if (dnxInfo != null)
{
dnxStatistics.Add(dnxInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);
return(cdnStatistics);
}
