private async Task RebuildPackageReports(CloudBlobContainer destinationContainer)
{
var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_sourceDatabase);
if (!dirtyPackageIds.Any())
{
return;
}
var result = Parallel.ForEach(dirtyPackageIds, new ParallelOptions {
MaxDegreeOfParallelism = 8
}, dirtyPackageId =>
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{ new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sourceDatabase) },
{ new DownloadsByPackageIdReportBuilder(ReportNames.DownloadsByPackageId, "downloads/" + _downloadsByPackageIdReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.DownloadsByPackageId], _sourceDatabase) }
};
foreach (var reportGenerator in reportGenerators)
{
ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report", dirtyPackageId.PackageId.ToLowerInvariant());
}
});
if (result.IsCompleted)
{
var runToCursor = dirtyPackageIds.First().RunToCuror;
await ReportDataCollector.UpdateDirtyPackageIdCursor(_sourceDatabase, runToCursor);
}
}
private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
{
var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime);
if (!dirtyPackageIds.Any())
{
return;
}
// first process the top 100 packages
var top100 = dirtyPackageIds.Take(100);
var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase);
var top100Task = Parallel.ForEach(top100, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, dirtyPackageId =>
{
var packageId = dirtyPackageId.PackageId.ToLowerInvariant();
var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId);
ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
ApplicationInsightsHelper.TrackReportProcessed(reportBuilder.ReportName + " report", packageId);
});
// once top 100 is processed, continue with the rest
if (top100Task.IsCompleted)
{
var excludingTop100 = dirtyPackageIds.Skip(100);
top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions {
MaxDegreeOfParallelism = 8
},
dirtyPackageId =>
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{
new RecentPopularityDetailByPackageReportBuilder(
ReportNames.RecentPopularityDetailByPackageId,
"recentpopularity/" + _recentPopularityDetailByPackageReportBaseName +
dirtyPackageId.PackageId.ToLowerInvariant()),
new ReportDataCollector(
_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId],
_statisticsDatabase)
}
};
foreach (var reportGenerator in reportGenerators)
{
ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value,
reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
ApplicationInsightsHelper.TrackReportProcessed(reportGenerator.Key.ReportName + " report",
dirtyPackageId.PackageId.ToLowerInvariant());
}
});
if (top100Task.IsCompleted)
{
var runToCursor = dirtyPackageIds.First().RunToCuror;
await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor);
}
}
}
