public override async Task <bool> Run()
{
try
{
var reportGenerationTime = DateTime.UtcNow;
var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName);
Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
if (string.IsNullOrEmpty(_reportName))
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{ new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
};
foreach (var reportGenerator in reportGenerators)
{
await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);
ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
}
await RebuildPackageReports(destinationContainer, reportGenerationTime);
await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
}
else
{
// generate only the specific report
var reportBuilder = new ReportBuilder(_reportName);
var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);
await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
}
Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
return(true);
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
ApplicationInsights.TrackException(exception);
return(false);
}
}
private async Task GenerateStandardReport(
string reportName,
DateTime reportGenerationTime,
CloudBlobContainer destinationContainer,
ILogger <ReportBuilder> reportBuilderLogger,
ILogger <ReportDataCollector> reportCollectorLogger)
{
var stopwatch = Stopwatch.StartNew();
var reportBuilder = new ReportBuilder(reportBuilderLogger, reportName);
var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds);
await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
stopwatch.Stop();
var reportMetricName = reportName + " report";
_applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
_applicationInsightsHelper.TrackReportProcessed(reportMetricName);
}
public override async Task <bool> Run()
{
try
{
var reportGenerationTime = DateTime.UtcNow;
var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);
_logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
if (string.IsNullOrEmpty(_reportName))
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{ new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.Last6Weeks), new ReportDataCollector(_storedProcedures[ReportNames.Last6Weeks], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
};
foreach (var reportGenerator in reportGenerators)
{
await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);
ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
}
await RebuildPackageReports(destinationContainer, reportGenerationTime);
await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
}
else
{
// generate only the specific report
var reportBuilder = new ReportBuilder(_reportName);
var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);
await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
}
_logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
// totals reports
var stopwatch = Stopwatch.StartNew();
// build downloads.v1.json
var targets = new List <StorageContainerTarget>();
targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
foreach (var dataContainerName in _dataContainerNames)
{
targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
}
var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase);
await downloadCountReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
stopwatch.Restart();
// build stats-totals.json
var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
await galleryTotalsReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);
// build tools.v1.json
var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
await toolsReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
stopwatch.Restart();
return(true);
}
catch (Exception exception)
{
_logger.LogError("Job run failed! {Exception}", exception);
return(false);
}
}
private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple <string, int, string>[] parameters)
{
var dataTable = await reportDataCollector.CollectAsync(reportGenerationTime, parameters);
if (dataTable.Rows.Count == 0)
{
return;
}
var json = reportBuilder.CreateReport(dataTable);
var reportWriter = new ReportWriter(destinationContainer);
await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json);
}
public override async Task<bool> Run()
{
try
{
var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName);
Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
if (string.IsNullOrEmpty(_reportName))
{
// generate all reports
var reportGenerators = new Dictionary<ReportBuilder, ReportDataCollector>
{
{ new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _sourceDatabase) },
{ new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _sourceDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _sourceDatabase) },
{ new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _sourceDatabase) }
};
foreach (var reportGenerator in reportGenerators)
{
await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value);
ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
}
await RebuildPackageReports(destinationContainer);
await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer);
}
else
{
// generate only the specific report
var reportBuilder = new ReportBuilder(_reportName);
var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _sourceDatabase);
await ProcessReport(destinationContainer, reportBuilder, reportDataCollector);
}
Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
return true;
}
catch (Exception exception)
{
Trace.TraceError(exception.ToString());
ApplicationInsights.TrackException(exception);
return false;
}
}
private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, params Tuple<string, int, string>[] parameters)
{
var dataTable = await reportDataCollector.CollectAsync(parameters);
var json = reportBuilder.CreateReport(dataTable);
var reportWriter = new ReportWriter(destinationContainer);
await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json);
}
public override async Task Run()
{
var statisticsDatabase = GetDatabaseRegistration <StatisticsDbConfiguration>();
var reportGenerationTime = DateTime.UtcNow;
var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);
Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
var reportBuilderLogger = LoggerFactory.CreateLogger <ReportBuilder>();
var reportCollectorLogger = LoggerFactory.CreateLogger <ReportDataCollector>();
if (string.IsNullOrEmpty(_reportName))
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{
new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
},
{
new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
},
{
new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
},
{
new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
},
{
new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
},
{
new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail),
new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
}
};
foreach (var reportGenerator in reportGenerators)
{
await ProcessReport(LoggerFactory, destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);
ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
}
await RebuildPackageReports(destinationContainer, reportGenerationTime);
await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
}
else
{
// generate only the specific report
var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName);
var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds);
await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
}
Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);
// totals reports
var stopwatch = Stopwatch.StartNew();
// build downloads.v1.json
var targets = new List <StorageContainerTarget>();
targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
foreach (var dataContainerName in _dataContainerNames)
{
targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
}
var downloadCountReport = new DownloadCountReport(
LoggerFactory.CreateLogger <DownloadCountReport>(),
targets,
OpenSqlConnectionAsync <StatisticsDbConfiguration>,
OpenSqlConnectionAsync <GalleryDbConfiguration>);
await downloadCountReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
stopwatch.Restart();
// build stats-totals.json
var galleryTotalsReport = new GalleryTotalsReport(
LoggerFactory.CreateLogger <GalleryTotalsReport>(),
_cloudStorageAccount,
_statisticsContainerName,
OpenSqlConnectionAsync <StatisticsDbConfiguration>,
OpenSqlConnectionAsync <GalleryDbConfiguration>);
await galleryTotalsReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);
// build tools.v1.json
var toolsReport = new DownloadsPerToolVersionReport(
LoggerFactory.CreateLogger <DownloadsPerToolVersionReport>(),
_cloudStorageAccount,
_statisticsContainerName,
OpenSqlConnectionAsync <StatisticsDbConfiguration>,
OpenSqlConnectionAsync <GalleryDbConfiguration>);
await toolsReport.Run();
stopwatch.Stop();
ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
stopwatch.Restart();
}