public override async Task <bool> Run() { try { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName); Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) }, { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer, reportGenerationTime); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime); } else { // generate only the specific report var reportBuilder = new ReportBuilder(_reportName); var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase); await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
private async Task GenerateStandardReport( string reportName, DateTime reportGenerationTime, CloudBlobContainer destinationContainer, ILogger <ReportBuilder> reportBuilderLogger, ILogger <ReportDataCollector> reportCollectorLogger) { var stopwatch = Stopwatch.StartNew(); var reportBuilder = new ReportBuilder(reportBuilderLogger, reportName); var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); stopwatch.Stop(); var reportMetricName = reportName + " report"; _applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); _applicationInsightsHelper.TrackReportProcessed(reportMetricName); }
public override async Task <bool> Run() { try { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); _logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) }, { new ReportBuilder(ReportNames.Last6Weeks), new ReportDataCollector(_storedProcedures[ReportNames.Last6Weeks], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime); ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer, reportGenerationTime); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime); } else { // generate only the specific report var reportBuilder = new ReportBuilder(_reportName); var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase); await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } _logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); // build downloads.v1.json var targets = new List <StorageContainerTarget>(); targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName)); foreach (var dataContainerName in _dataContainerNames) { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase); await downloadCountReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName); stopwatch.Restart(); // build stats-totals.json var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await galleryTotalsReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName); // build tools.v1.json var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await toolsReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName); stopwatch.Restart(); return(true); } catch (Exception exception) { _logger.LogError("Job run failed! {Exception}", exception); return(false); } }
private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple <string, int, string>[] parameters) { var dataTable = await reportDataCollector.CollectAsync(reportGenerationTime, parameters); if (dataTable.Rows.Count == 0) { return; } var json = reportBuilder.CreateReport(dataTable); var reportWriter = new ReportWriter(destinationContainer); await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json); }
public override async Task<bool> Run() { try { var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName); Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary<ReportBuilder, ReportDataCollector> { { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _sourceDatabase) }, { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _sourceDatabase) }, { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _sourceDatabase) }, { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _sourceDatabase) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer); } else { // generate only the specific report var reportBuilder = new ReportBuilder(_reportName); var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _sourceDatabase); await ProcessReport(destinationContainer, reportBuilder, reportDataCollector); } Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); return true; } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return false; } }
private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, params Tuple<string, int, string>[] parameters) { var dataTable = await reportDataCollector.CollectAsync(parameters); var json = reportBuilder.CreateReport(dataTable); var reportWriter = new ReportWriter(destinationContainer); await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json); }
public override async Task Run() { var statisticsDatabase = GetDatabaseRegistration <StatisticsDbConfiguration>(); var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger <ReportBuilder>(); var reportCollectorLogger = LoggerFactory.CreateLogger <ReportDataCollector>(); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) }, { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) }, { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) }, { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) }, { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) }, { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(LoggerFactory, destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime); ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer, reportGenerationTime); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime); } else { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); // build downloads.v1.json var targets = new List <StorageContainerTarget>(); targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName)); foreach (var dataContainerName in _dataContainerNames) { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } var downloadCountReport = new DownloadCountReport( LoggerFactory.CreateLogger <DownloadCountReport>(), targets, OpenSqlConnectionAsync <StatisticsDbConfiguration>, OpenSqlConnectionAsync <GalleryDbConfiguration>); await downloadCountReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName); stopwatch.Restart(); // build stats-totals.json var galleryTotalsReport = new GalleryTotalsReport( LoggerFactory.CreateLogger <GalleryTotalsReport>(), _cloudStorageAccount, _statisticsContainerName, OpenSqlConnectionAsync <StatisticsDbConfiguration>, OpenSqlConnectionAsync <GalleryDbConfiguration>); await galleryTotalsReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName); // build tools.v1.json var toolsReport = new DownloadsPerToolVersionReport( LoggerFactory.CreateLogger <DownloadsPerToolVersionReport>(), _cloudStorageAccount, _statisticsContainerName, OpenSqlConnectionAsync <StatisticsDbConfiguration>, OpenSqlConnectionAsync <GalleryDbConfiguration>); await toolsReport.Run(); stopwatch.Stop(); ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName); stopwatch.Restart(); }