private async Task RebuildPackageReports(CloudBlobContainer destinationContainer) { var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_sourceDatabase); if (!dirtyPackageIds.Any()) { return; } var result = Parallel.ForEach(dirtyPackageIds, new ParallelOptions { MaxDegreeOfParallelism = 8 }, dirtyPackageId => { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sourceDatabase) }, { new DownloadsByPackageIdReportBuilder(ReportNames.DownloadsByPackageId, "downloads/" + _downloadsByPackageIdReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.DownloadsByPackageId], _sourceDatabase) } }; foreach (var reportGenerator in reportGenerators) { ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report", dirtyPackageId.PackageId.ToLowerInvariant()); } }); if (result.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; await ReportDataCollector.UpdateDirtyPackageIdCursor(_sourceDatabase, runToCursor); } }
public override async Task <bool> Run() { try { var stopwatch = Stopwatch.StartNew(); // build downloads.v1.json var targets = new List <StorageContainerTarget>(); targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName)); foreach (var dataContainerName in _dataContainerNames) { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase); await downloadCountReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(DownloadCountReport.ReportName); stopwatch.Restart(); // build stats-totals.json var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await galleryTotalsReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(GalleryTotalsReport.ReportName); // build tools.v1.json var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await toolsReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName); stopwatch.Restart(); return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
public override async Task <bool> Run() { try { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName); Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) }, { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer, reportGenerationTime); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime); } else { // generate only the specific report var reportBuilder = new ReportBuilder(_reportName); var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase); await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime); if (!dirtyPackageIds.Any()) { return; } // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = 4 }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsights.TrackReportProcessed(reportBuilder.ReportName + " report", packageId); }); // once top 100 is processed, continue with the rest if (top100Task.IsCompleted) { var excludingTop100 = dirtyPackageIds.Skip(100); top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions { MaxDegreeOfParallelism = 8 }, dirtyPackageId => { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new RecentPopularityDetailByPackageReportBuilder( ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report", dirtyPackageId.PackageId.ToLowerInvariant()); } }); if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor); } } }