Пример #1
0
        public override async Task Run()
        {
            var statisticsDatabase = GetDatabaseRegistration <StatisticsDbConfiguration>();

            var reportGenerationTime = DateTime.UtcNow;
            var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);

            Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                            statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            var reportBuilderLogger   = LoggerFactory.CreateLogger <ReportBuilder>();
            var reportCollectorLogger = LoggerFactory.CreateLogger <ReportDataCollector>();

            if (string.IsNullOrEmpty(_reportNameConfig))
            {
                // generate all reports
                foreach (var reportName in ReportNames.StandardReports)
                {
                    await GenerateStandardReport(reportName, reportGenerationTime, destinationContainer, reportBuilderLogger, reportCollectorLogger);
                }

                await RebuildPackageReports(destinationContainer, reportGenerationTime);
                await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
            }
            else if (ReportNames.StandardReports.Contains(_reportNameConfig))
            {
                // generate only the specific standard report
                await GenerateStandardReport(_reportNameConfig, reportGenerationTime, destinationContainer, reportBuilderLogger, reportCollectorLogger);
            }
            else if (ShouldGenerateReport(ReportNames.RecentPopularityDetailByPackageId, _reportNameConfig))
            {
                await RebuildPackageReports(destinationContainer, reportGenerationTime);
                await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
            }

            Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                                  statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            // totals reports
            Stopwatch stopwatch;

            // build downloads.v1.json
            if (ShouldGenerateReport(ReportNames.DownloadCount, _reportNameConfig))
            {
                stopwatch = Stopwatch.StartNew();

                var targets = new List <StorageContainerTarget>();
                targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
                foreach (var dataContainerName in _dataContainerNames)
                {
                    targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
                }

                var downloadCountReport = new DownloadCountReport(
                    LoggerFactory.CreateLogger <DownloadCountReport>(),
                    targets,
                    OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                    OpenSqlConnectionAsync <GalleryDbConfiguration>,
                    _sqlCommandTimeoutSeconds);
                await downloadCountReport.Run();

                stopwatch.Stop();
                var reportMetricName = ReportNames.DownloadCount + ReportNames.Extension;
                _applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                _applicationInsightsHelper.TrackReportProcessed(reportMetricName);
            }

            // build stats-totals.json
            if (ShouldGenerateReport(ReportNames.GalleryTotals, _reportNameConfig))
            {
                stopwatch = Stopwatch.StartNew();

                var galleryTotalsReport = new GalleryTotalsReport(
                    LoggerFactory.CreateLogger <GalleryTotalsReport>(),
                    _cloudStorageAccount,
                    _statisticsContainerName,
                    OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                    OpenSqlConnectionAsync <GalleryDbConfiguration>,
                    commandTimeoutSeconds: _sqlCommandTimeoutSeconds);
                await galleryTotalsReport.Run();

                stopwatch.Stop();
                var reportMetricName = ReportNames.GalleryTotals + ReportNames.Extension;
                _applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                _applicationInsightsHelper.TrackReportProcessed(reportMetricName);
            }

            // build tools.v1.json
            if (ShouldGenerateReport(ReportNames.DownloadsPerToolVersion, _reportNameConfig))
            {
                stopwatch = Stopwatch.StartNew();

                var toolsReport = new DownloadsPerToolVersionReport(
                    LoggerFactory.CreateLogger <DownloadsPerToolVersionReport>(),
                    _cloudStorageAccount,
                    _statisticsContainerName,
                    OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                    OpenSqlConnectionAsync <GalleryDbConfiguration>,
                    _sqlCommandTimeoutSeconds);
                await toolsReport.Run();

                stopwatch.Stop();
                var reportMetricName = ReportNames.DownloadsPerToolVersion + ReportNames.Extension;
                _applicationInsightsHelper.TrackMetric(reportMetricName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                _applicationInsightsHelper.TrackReportProcessed(reportMetricName);
            }
        }
Пример #2
0
        public override async Task <bool> Run()
        {
            try
            {
                var reportGenerationTime = DateTime.UtcNow;
                var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);

                _logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                if (string.IsNullOrEmpty(_reportName))
                {
                    // generate all reports
                    var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                    {
                        { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.Last6Weeks), new ReportDataCollector(_storedProcedures[ReportNames.Last6Weeks], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) },
                        { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);

                        ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                    }

                    await RebuildPackageReports(destinationContainer, reportGenerationTime);
                    await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
                }
                else
                {
                    // generate only the specific report
                    var reportBuilder       = new ReportBuilder(_reportName);
                    var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase);

                    await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
                }

                _logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

                // totals reports
                var stopwatch = Stopwatch.StartNew();

                // build downloads.v1.json
                var targets = new List <StorageContainerTarget>();
                targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
                foreach (var dataContainerName in _dataContainerNames)
                {
                    targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
                }
                var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase);
                await downloadCountReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
                stopwatch.Restart();

                // build stats-totals.json
                var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
                await galleryTotalsReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);


                // build tools.v1.json
                var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase);
                await toolsReport.Run();

                stopwatch.Stop();
                ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
                ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
                stopwatch.Restart();

                return(true);
            }
            catch (Exception exception)
            {
                _logger.LogError("Job run failed! {Exception}", exception);

                return(false);
            }
        }
Пример #3
0
        public override async Task Run()
        {
            var statisticsDatabase = GetDatabaseRegistration <StatisticsDbConfiguration>();

            var reportGenerationTime = DateTime.UtcNow;
            var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName);

            Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                            statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            var reportBuilderLogger   = LoggerFactory.CreateLogger <ReportBuilder>();
            var reportCollectorLogger = LoggerFactory.CreateLogger <ReportDataCollector>();

            if (string.IsNullOrEmpty(_reportName))
            {
                // generate all reports
                var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                {
                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    },

                    {
                        new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail),
                        new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds)
                    }
                };

                foreach (var reportGenerator in reportGenerators)
                {
                    await ProcessReport(LoggerFactory, destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime);

                    ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report");
                }

                await RebuildPackageReports(destinationContainer, reportGenerationTime);
                await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime);
            }
            else
            {
                // generate only the specific report
                var reportBuilder       = new ReportBuilder(reportBuilderLogger, _reportName);
                var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync <StatisticsDbConfiguration>, _sqlCommandTimeoutSeconds);

                await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime);
            }

            Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}",
                                  statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name);

            // totals reports
            var stopwatch = Stopwatch.StartNew();

            // build downloads.v1.json
            var targets = new List <StorageContainerTarget>();

            targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName));
            foreach (var dataContainerName in _dataContainerNames)
            {
                targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName));
            }

            var downloadCountReport = new DownloadCountReport(
                LoggerFactory.CreateLogger <DownloadCountReport>(),
                targets,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await downloadCountReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(DownloadCountReport.ReportName);
            stopwatch.Restart();

            // build stats-totals.json
            var galleryTotalsReport = new GalleryTotalsReport(
                LoggerFactory.CreateLogger <GalleryTotalsReport>(),
                _cloudStorageAccount,
                _statisticsContainerName,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await galleryTotalsReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(GalleryTotalsReport.ReportName);


            // build tools.v1.json
            var toolsReport = new DownloadsPerToolVersionReport(
                LoggerFactory.CreateLogger <DownloadsPerToolVersionReport>(),
                _cloudStorageAccount,
                _statisticsContainerName,
                OpenSqlConnectionAsync <StatisticsDbConfiguration>,
                OpenSqlConnectionAsync <GalleryDbConfiguration>);
            await toolsReport.Run();

            stopwatch.Stop();
            ApplicationInsightsHelper.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds);
            ApplicationInsightsHelper.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName);
            stopwatch.Restart();
        }