private async Task ArchiveBlobAsync(ILeasedLogFile logFile)
{
var stopwatch = Stopwatch.StartNew();
try
{
await EnsureCopiedToContainerAsync(logFile, _targetContainer);
_jobEventSource.FinishingArchiveUpload(logFile.Uri);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedArchiveUpload(logFile.Uri);
ApplicationInsights.TrackException(exception, logFile.Blob.Name);
throw;
}
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
try
{
var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
ApplicationInsights.Initialize(instrumentationKey);
var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);
_targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString);
_azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
_azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
_cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));
return(true);
}
catch (Exception exception)
{
ApplicationInsights.TrackException(exception);
Trace.TraceError(exception.ToString());
}
return(false);
}
private async Task <IReadOnlyCollection <PackageStatistics> > ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
return(packageStatistics);
}
public override async Task <bool> Run()
{
try
{
var stopwatch = Stopwatch.StartNew();
// build downloads.v1.json
var targets = new List <StorageContainerTarget>();
targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
foreach (var dataContainerName in _dataContainerNames)
{
targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
}
var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase);
await downloadCountReport.Run();
stopwatch.Stop();
ApplicationInsights.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsights.TrackReportProcessed(DownloadCountReport.ReportName);
stopwatch.Restart();
// build stats-totals.json
var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
await galleryTotalsReport.Run();
stopwatch.Stop();
ApplicationInsights.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsights.TrackReportProcessed(GalleryTotalsReport.ReportName);
// build tools.v1.json
var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
await toolsReport.Run();
stopwatch.Stop();
ApplicationInsights.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsights.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
stopwatch.Restart();
return(true);
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
ApplicationInsights.TrackException(exception);
return(false);
}
}
public override async Task <bool> Run()
{
try
{
var reportGenerationTime = DateTime.UtcNow;
var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName);
Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
if (string.IsNullOrEmpty(_reportName))
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{ new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
};
foreach (var reportGenerator in reportGenerators)
{
await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);
ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
}
await RebuildPackageReports(destinationContainer, reportGenerationTime);
await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
}
else
{
// generate only the specific report
var reportBuilder = new ReportBuilder(_reportName);
var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);
await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
}
Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
return(true);
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
ApplicationInsights.TrackException(exception);
return(false);
}
}
private async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile)
{
var stopwatch = Stopwatch.StartNew();
try
{
_jobEventSource.BeginningOpenCompressedBlob(logFile.Uri);
var memoryStream = new MemoryStream();
// decompress into memory (these are rolling log files and relatively small)
using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null))
{
await blobStream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
stopwatch.Stop();
_jobEventSource.FinishedOpenCompressedBlob(logFile.Uri);
ApplicationInsights.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name);
// verify if the stream is gzipped or not
if (await IsGzipCompressed(memoryStream))
{
return(new GZipInputStream(memoryStream));
}
else
{
return(memoryStream);
}
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedOpenCompressedBlob(logFile.Uri);
ApplicationInsights.TrackException(exception, logFile.Blob.Name);
throw;
}
}
private async Task <string> TryAcquireLeaseAsync(ICloudBlob blob)
{
string leaseId;
var blobUriString = blob.Uri.ToString();
try
{
var sourceBlobExists = await blob.ExistsAsync();
if (!sourceBlobExists)
{
return(null);
}
_jobEventSource.BeginningAcquireLease(blobUriString);
leaseId = await blob.AcquireLeaseAsync(_defaultLeaseTime, null);
_jobEventSource.FinishedAcquireLease(blobUriString);
}
catch (StorageException storageException)
{
// check if this is a 409 Conflict with a StatusDescription stating that "There is already a lease present."
// or 404 NotFound (might have been removed by another other instance of this job)
var webException = storageException.InnerException as WebException;
if (webException != null)
{
var httpWebResponse = webException.Response as HttpWebResponse;
if (httpWebResponse != null)
{
if ((httpWebResponse.StatusCode == HttpStatusCode.Conflict &&
httpWebResponse.StatusDescription == "There is already a lease present.") || httpWebResponse.StatusCode == HttpStatusCode.NotFound)
{
_jobEventSource.FailedAcquireLease(blobUriString); // no need to report these in Application Insights
return(null);
}
}
}
_jobEventSource.FailedAcquireLease(blobUriString);
ApplicationInsights.TrackException(storageException);
throw;
}
return(leaseId);
}
public async Task <IReadOnlyCollection <ILeasedLogFile> > LeaseNextLogFilesToBeProcessedAsync(string prefix)
{
try
{
_jobEventSource.BeginningBlobListing(prefix);
var blobResultSegments = await _container.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.None, _maxListBlobResultSegments, null, null, null);
_jobEventSource.FinishingBlobListing(prefix);
var leasedFiles = new List <ILeasedLogFile>();
foreach (var logFile in blobResultSegments.Results)
{
if (leasedFiles.Count == _maxLeasesPerJobRun)
{
break;
}
// Get the source blob
var blobName = logFile.Uri.Segments.Last();
var logFileBlob = _container.GetBlockBlobReference(blobName);
// try to acquire a lease on the blob
var leaseId = await TryAcquireLeaseAsync(logFileBlob);
if (string.IsNullOrEmpty(leaseId))
{
// the blob is already leased, ignore it and move on
continue;
}
leasedFiles.Add(new LeasedLogFile(logFileBlob, leaseId));
}
return(leasedFiles);
}
catch (Exception exception)
{
_jobEventSource.FailedBlobListing(prefix);
ApplicationInsights.TrackException(exception);
}
return(Enumerable.Empty <ILeasedLogFile>().ToList());
}
private async Task DeleteSourceBlobAsync(ILeasedLogFile logFile)
{
if (await logFile.Blob.ExistsAsync())
{
try
{
_jobEventSource.BeginningDelete(logFile.Uri);
var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId);
await logFile.Blob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, accessCondition, null, null);
_jobEventSource.FinishedDelete(logFile.Uri);
}
catch (Exception exception)
{
_jobEventSource.FailedDelete(logFile.Uri);
ApplicationInsights.TrackException(exception, logFile.Blob.Name);
throw;
}
}
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
try
{
var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
ApplicationInsights.Initialize(instrumentationKey);
var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
_statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString);
var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase);
_galleryDatabase = new SqlConnectionStringBuilder(galleryDatabaseConnectionString);
var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount);
_statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName);
var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);
_dataStorageAccount = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount);
var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName)
.Split(new [] { '|' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var containerName in containerNames)
{
ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName);
}
_dataContainerNames = containerNames;
return(true);
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
ApplicationInsights.TrackException(exception);
return(false);
}
}
internal async Task InsertDownloadFactsAsync(DataTable downloadFacts, string logFileName)
{
Trace.WriteLine("Inserting into facts table...");
var stopwatch = Stopwatch.StartNew();
using (var connection = await _targetDatabase.ConnectTo())
using (var transaction = connection.BeginTransaction(IsolationLevel.Serializable))
{
var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction);
bulkCopy.DestinationTableName = downloadFacts.TableName;
bulkCopy.BulkCopyTimeout = _defaultCommandTimeout;
try
{
await bulkCopy.WriteToServerAsync(downloadFacts);
transaction.Commit();
stopwatch.Stop();
ApplicationInsights.TrackMetric("Insert facts duration (ms)", stopwatch.ElapsedMilliseconds, logFileName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
ApplicationInsights.TrackException(exception, logFileName);
transaction.Rollback();
throw;
}
}
Trace.Write(" DONE");
}
private async Task <IReadOnlyCollection <T> > GetDimension <T>(string dimension, string logFileName, Func <SqlConnection, Task <IReadOnlyCollection <T> > > retrieve)
{
var stopwatch = Stopwatch.StartNew();
var count = _maxRetryCount;
while (count > 0)
{
try
{
_jobEventSource.BeginningRetrieveDimension(dimension);
IReadOnlyCollection <T> dimensions;
using (var connection = await _targetDatabase.ConnectTo())
{
dimensions = await retrieve(connection);
}
stopwatch.Stop();
_jobEventSource.FinishedRetrieveDimension(dimension, stopwatch.ElapsedMilliseconds);
ApplicationInsights.TrackRetrieveDimensionDuration(dimension, stopwatch.ElapsedMilliseconds, logFileName);
return(dimensions);
}
catch (SqlException e)
{
--count;
if (count <= 0)
{
throw;
}
if (e.Number == 1205)
{
Trace.TraceWarning("Deadlock, retrying...");
ApplicationInsights.TrackSqlException("SQL Deadlock", e, logFileName, dimension);
}
else if (e.Number == -2)
{
Trace.TraceWarning("Timeout, retrying...");
ApplicationInsights.TrackSqlException("SQL Timeout", e, logFileName, dimension);
}
else if (e.Number == 2601)
{
Trace.TraceWarning("Duplicate key, retrying...");
ApplicationInsights.TrackSqlException("SQL Duplicate Key", e, logFileName, dimension);
}
else
{
throw;
}
Task.Delay(_retryDelay).Wait();
}
catch (Exception exception)
{
_jobEventSource.FailedRetrieveDimension(dimension);
ApplicationInsights.TrackException(exception, logFileName);
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
throw;
}
}
return(Enumerable.Empty <T>().ToList());
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var dnxStatistics = new List <DnxStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
if (logEntry.RequestUrl.Contains("dist.asp.net"))
{
var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
if (dnxInfo != null)
{
dnxStatistics.Add(dnxInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);
return(cdnStatistics);
}
