Ejemplo n.º 1
0
        private async Task RebuildPackageReports(CloudBlobContainer destinationContainer)
        {
            var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_sourceDatabase);

            if (!dirtyPackageIds.Any())
            {
                return;
            }

            var result = Parallel.ForEach(dirtyPackageIds, new ParallelOptions {
                MaxDegreeOfParallelism = 8
            }, dirtyPackageId =>
            {
                // generate all reports
                var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                {
                    { new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sourceDatabase) },
                    { new DownloadsByPackageIdReportBuilder(ReportNames.DownloadsByPackageId, "downloads/" + _downloadsByPackageIdReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.DownloadsByPackageId], _sourceDatabase) }
                };

                foreach (var reportGenerator in reportGenerators)
                {
                    ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                    ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report", dirtyPackageId.PackageId.ToLowerInvariant());
                }
            });

            if (result.IsCompleted)
            {
                var runToCursor = dirtyPackageIds.First().RunToCuror;
                await ReportDataCollector.UpdateDirtyPackageIdCursor(_sourceDatabase, runToCursor);
            }
        }
Ejemplo n.º 2
0
        private async Task GenerateStandardReport(
            string reportName,
            DateTime reportGenerationTime,
            CloudBlobContainer destinationContainer,
            ILogger <ReportBuilder> reportBuilderLogger,
            ILogger <ReportDataCollector> reportCollectorLogger)
        {
            var stopwatch = Stopwatch.StartNew();

            var reportBuilder       = new ReportBuilder(reportBuilderLogger, reportName);
            var reportDataCollector = new ReportDataCollector(
                reportCollectorLogger,
                _storedProcedures[reportName],
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                _sqlCommandTimeoutSeconds,
                _applicationInsightsHelper);

            await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);

            stopwatch.Stop();

            var reportMetricName = reportName + " report";

            _applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            _applicationInsightsHelper.TrackReportProcessed(reportMetricName);
        }
Ejemplo n.º 3
0
        public override async Task <bool> Run()
        {
            try
            {
                var reportGenerationTime = DateTime.UtcNow;
                var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName);
                Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                if (string.IsNullOrEmpty(_reportName))
                {
                    // generate all reports
                    var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                    {
                        { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);

                        ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                    }

                    await RebuildPackageReports(destinationContainer, reportGenerationTime);
                    await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
                }
                else
                {
                    // generate only the specific report
                    var reportBuilder       = new ReportBuilder(_reportName);
                    var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);

                    await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
                }

                Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                return(true);
            }
            catch (Exception exception)
            {
                Trace.TraceError(exception.ToString());
                ApplicationInsights.TrackException(exception);
                return(false);
            }
        }
Ejemplo n.º 4
0
        private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobContainer destinationContainer, ReportBuilder reportBuilder,
                                                ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple <string, int, string>[] parameters)
        {
            var dataTable = await reportDataCollector.CollectAsync(reportGenerationTime, parameters);

            if (dataTable.Rows.Count == 0)
            {
                return;
            }

            var json = reportBuilder.CreateReport(dataTable);

            var reportWriter = new ReportWriter(loggerFactory.CreateLogger <ReportWriter>(), destinationContainer);
            await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json);
        }
Ejemplo n.º 5
0
        private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
        {
            Logger.LogDebug("Getting list of inactive packages.");
            var packageIds = await ReportDataCollector.ListInactivePackageIdReports(
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                reportGenerationTime,
                _sqlCommandTimeoutSeconds);

            Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count);

            // Collect the list of reports
            var subContainer = "recentpopularity/";

            Logger.LogDebug("Collecting list of package detail reports");
            var reports = destinationContainer.ListBlobs(subContainer + _recentPopularityDetailByPackageReportBaseName)
                          .OfType <CloudBlockBlob>()
                          .Select(b => b.Name);

            var reportSet = new HashSet <string>(reports);

            Logger.LogInformation("Collected {PackageDetailReportCount} package detail reports", reportSet.Count);

            Parallel.ForEach(packageIds, new ParallelOptions {
                MaxDegreeOfParallelism = 8
            }, async id =>
            {
                string reportName = _recentPopularityDetailByPackageReportBaseName + id;
                string blobName   = subContainer + reportName + ReportNames.Extension;
                if (reportSet.Contains(blobName))
                {
                    var blob = destinationContainer.GetBlockBlobReference(blobName);
                    Logger.LogDebug("{ReportName}: Deleting empty report from {BlobUri}", reportName, blob.Uri.AbsoluteUri);

                    await blob.DeleteIfExistsAsync();

                    Logger.LogInformation("{ReportName}: Deleted empty report from {BlobUri}", reportName, blob.Uri.AbsoluteUri);
                }
            });
        }
Ejemplo n.º 6
0
        private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
        {
            Trace.TraceInformation("Getting list of inactive packages.");
            var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDatabase, reportGenerationTime);

            Trace.TraceInformation("Found {0} inactive packages.", packageIds.Count);

            // Collect the list of reports
            var subContainer = "recentpopularity/";

            Trace.TraceInformation("Collecting list of package detail reports");
            var reports = destinationContainer.ListBlobs(subContainer + _recentPopularityDetailByPackageReportBaseName)
                          .OfType <CloudBlockBlob>()
                          .Select(b => b.Name);

            var reportSet = new HashSet <string>(reports);

            Trace.TraceInformation("Collected {0} package detail reports", reportSet.Count);

            Parallel.ForEach(packageIds, new ParallelOptions {
                MaxDegreeOfParallelism = 8
            }, async id =>
            {
                string reportName = _recentPopularityDetailByPackageReportBaseName + id;
                string blobName   = subContainer + reportName + ".json";
                if (reportSet.Contains(blobName))
                {
                    var blob = destinationContainer.GetBlockBlobReference(blobName);
                    Trace.TraceInformation("{0}: Deleting empty report from {1}", reportName, blob.Uri.AbsoluteUri);

                    await blob.DeleteIfExistsAsync();

                    Trace.TraceInformation("{0}: Deleted empty report from {1}", reportName, blob.Uri.AbsoluteUri);
                }
            });
        }
Ejemplo n.º 7
0
        public override async Task <bool> Run()
        {
            try
            {
                var reportGenerationTime = DateTime.UtcNow;
                var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);

                _logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                if (string.IsNullOrEmpty(_reportName))
                {
                    // generate all reports
                    var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                    {
                        { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.Last6Weeks), new ReportDataCollector(_storedProcedures[ReportNames.Last6Weeks], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);

                        ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                    }

                    await RebuildPackageReports(destinationContainer, reportGenerationTime);
                    await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
                }
                else
                {
                    // generate only the specific report
                    var reportBuilder       = new ReportBuilder(_reportName);
                    var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);

                    await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
                }

                _logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                // totals reports
                var stopwatch = Stopwatch.StartNew();

                // build downloads.v1.json
                var targets = new List <StorageContainerTarget>();
                targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
                foreach (var dataContainerName in _dataContainerNames)
                {
                    targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
                }
                var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase);
                await downloadCountReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
                stopwatch.Restart();

                // build stats-totals.json
                var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
                await galleryTotalsReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);


                // build tools.v1.json
                var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
                await toolsReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
                stopwatch.Restart();

                return(true);
            }
            catch (Exception exception)
            {
                _logger.LogError("Job run failed! {Exception}", exception);

                return(false);
            }
        }
Ejemplo n.º 8
0
        private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
        {
            var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime);

            if (!dirtyPackageIds.Any())
            {
                return;
            }

            // first process the top 100 packages
            var top100 = dirtyPackageIds.Take(100);
            var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase);
            var top100Task          = Parallel.ForEach(top100, new ParallelOptions {
                MaxDegreeOfParallelism = 4
            }, dirtyPackageId =>
            {
                var packageId     = dirtyPackageId.PackageId.ToLowerInvariant();
                var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId);

                ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                ApplicationInsightsHelper.TrackReportProcessed(reportBuilder.ReportName + " report", packageId);
            });

            // once top 100 is processed, continue with the rest
            if (top100Task.IsCompleted)
            {
                var excludingTop100 = dirtyPackageIds.Skip(100);

                top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions {
                    MaxDegreeOfParallelism = 8
                },
                                              dirtyPackageId =>
                {
                    // generate all reports
                    var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                    {
                        {
                            new RecentPopularityDetailByPackageReportBuilder(
                                ReportNames.RecentPopularityDetailByPackageId,
                                "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName +
                                dirtyPackageId.PackageId.ToLowerInvariant()),
                            new ReportDataCollector(
                                _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId],
                                _statisticsDatabase)
                        }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value,
                                      reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                        ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report",
                                                                       dirtyPackageId.PackageId.ToLowerInvariant());
                    }
                });

                if (top100Task.IsCompleted)
                {
                    var runToCursor = dirtyPackageIds.First().RunToCuror;
                    await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor);
                }
            }
        }
Ejemplo n.º 9
0
        public override async Task<bool> Run()
        {
            try
            {
                var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName);
                Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                if (string.IsNullOrEmpty(_reportName))
                {
                    // generate all reports
                    var reportGenerators = new Dictionary<ReportBuilder, ReportDataCollector>
                    {
                        { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _sourceDatabase) },
                        { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _sourceDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _sourceDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _sourceDatabase) }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value);
                        ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                    }

                    await RebuildPackageReports(destinationContainer);
                    await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer);
                }
                else
                {
                    // generate only the specific report
                    var reportBuilder = new ReportBuilder(_reportName);
                    var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _sourceDatabase);

                    await ProcessReport(destinationContainer, reportBuilder, reportDataCollector);
                }

                Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _sourceDatabase.DataSource, _sourceDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                return true;
            }
            catch (Exception exception)
            {
                Trace.TraceError(exception.ToString());
                ApplicationInsights.TrackException(exception);
                return false;
            }
        }
Ejemplo n.º 10
0
        private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, params Tuple<string, int, string>[] parameters)
        {
            var dataTable = await reportDataCollector.CollectAsync(parameters);
            var json = reportBuilder.CreateReport(dataTable);

            var reportWriter = new ReportWriter(destinationContainer);
            await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json);
        }
Ejemplo n.º 11
0
        private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
        {
            var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime);

            if (!dirtyPackageIds.Any())
                return;

            // first process the top 100 packages
            var top100 = dirtyPackageIds.Take(100);
            var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase);
            var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = 4 }, dirtyPackageId =>
            {
                var packageId = dirtyPackageId.PackageId.ToLowerInvariant();
                var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId);

                ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                ApplicationInsights.TrackReportProcessed(reportBuilder.ReportName + " report", packageId);
            });

            // once top 100 is processed, continue with the rest
            if (top100Task.IsCompleted)
            {
                var excludingTop100 = dirtyPackageIds.Skip(100);

                top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions { MaxDegreeOfParallelism = 8 },
                    dirtyPackageId =>
                    {
                        // generate all reports
                        var reportGenerators = new Dictionary<ReportBuilder, ReportDataCollector>
                        {
                            {
                                new RecentPopularityDetailByPackageReportBuilder(
                                    ReportNames.RecentPopularityDetailByPackageId,
                                    "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName +
                                    dirtyPackageId.PackageId.ToLowerInvariant()),
                                new ReportDataCollector(
                                    _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId],
                                    _statisticsDatabase)
                            }
                        };

                        foreach (var reportGenerator in reportGenerators)
                        {
                            ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value,
                                reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                            ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report",
                                dirtyPackageId.PackageId.ToLowerInvariant());
                        }
                    });

                if (top100Task.IsCompleted)
                {
                    var runToCursor = dirtyPackageIds.First().RunToCuror;
                    await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor);
                }
            }
        }
Ejemplo n.º 12
0
        private static async Task ProcessReport(CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, params Tuple <string, int, string>[] parameters)
        {
            var dataTable = await reportDataCollector.CollectAsync(parameters);

            var json = reportBuilder.CreateReport(dataTable);

            var reportWriter = new ReportWriter(destinationContainer);
            await reportWriter.WriteReport(reportBuilder.ReportArtifactName, json);
        }
Ejemplo n.º 13
0
        public override async Task Run()
        {
            var statisticsDatabase = GetDatabaseRegistration <StatisticsDbConfiguration>();

            var reportGenerationTime = DateTime.UtcNow;
            var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);

            Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                            statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            var reportBuilderLogger   = LoggerFactory.CreateLogger <ReportBuilder>();
            var reportCollectorLogger = LoggerFactory.CreateLogger <ReportDataCollector>();

            if (string.IsNullOrEmpty(_reportName))
            {
                // generate all reports
                var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                {
                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    }
                };

                foreach (var reportGenerator in reportGenerators)
                {
                    await ProcessReport(LoggerFactory, destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);

                    ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                }

                await RebuildPackageReports(destinationContainer, reportGenerationTime);
                await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
            }
            else
            {
                // generate only the specific report
                var reportBuilder       = new ReportBuilder(reportBuilderLogger, _reportName);
                var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds);

                await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
            }

            Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                                  statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            // totals reports
            var stopwatch = Stopwatch.StartNew();

            // build downloads.v1.json
            var targets = new List <StorageContainerTarget>();

            targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
            foreach (var dataContainerName in _dataContainerNames)
            {
                targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
            }

            var downloadCountReport = new DownloadCountReport(
                LoggerFactory.CreateLogger <DownloadCountReport>(),
                targets,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await downloadCountReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
            stopwatch.Restart();

            // build stats-totals.json
            var galleryTotalsReport = new GalleryTotalsReport(
                LoggerFactory.CreateLogger <GalleryTotalsReport>(),
                _cloudStorageAccount,
                _statisticsContainerName,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await galleryTotalsReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);


            // build tools.v1.json
            var toolsReport = new DownloadsPerToolVersionReport(
                LoggerFactory.CreateLogger <DownloadsPerToolVersionReport>(),
                _cloudStorageAccount,
                _statisticsContainerName,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await toolsReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
            stopwatch.Restart();
        }