public async Task AddSameTags_VariousRepos_FlattenedProperly() { //arrange var db = new StatisticsDatabase(Settings); StatisticsService service = new StatisticsService(db, new Mapper(MappingConfigurationFactory.Create())); var parameter1 = new RepositoryQueryParameter(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); await service.Update(parameter1, new [] { "FirstTag", "SecondTag", "Third" }).ConfigureAwait(false); await service.Update(parameter1, new[] { "FIRSTTAG", "", "Fourth" }).ConfigureAwait(false); await service.Update(parameter1, new[] { "FIRSTTAG", "SecondTag", "Fourth" }).ConfigureAwait(false); parameter1 = new RepositoryQueryParameter(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); await service.Update(parameter1, new[] { "FirstTag", "SecondTag", "Fourth" }).ConfigureAwait(false); await service.Update(parameter1, new[] { "Fifth", "Sixth" }).ConfigureAwait(false); //act List <SearchKeywordData> result = (await service.GetFlattened().ConfigureAwait(false)).ToList(); //assert result.Count.Should().Be(6); result.Single(x => x.Keyword == "FirstTag").SearchCount.Should().Be(4); result.Single(x => x.Keyword == "SecondTag").SearchCount.Should().Be(3); result.Single(x => x.Keyword == "Third").SearchCount.Should().Be(1); result.Single(x => x.Keyword == "Fourth").SearchCount.Should().Be(3); result.Single(x => x.Keyword == "Fifth").SearchCount.Should().Be(1); result.Single(x => x.Keyword == "Sixth").SearchCount.Should().Be(1); }
public async Task AddSameTags_ValuesIncrementedProperly() { var db = new StatisticsDatabase(Settings); StatisticsService service = new StatisticsService(db, new Mapper(MappingConfigurationFactory.Create())); string organizationOne = Guid.NewGuid().ToString(); string repoOne = Guid.NewGuid().ToString(); var parameter = new RepositoryQueryParameter(organizationOne, repoOne); string[] words = new[] { "FirstTag", "SecondTag", "Third" }; SearchStatistics stats1 = await service.Update(parameter, words).ConfigureAwait(false); SearchStatistics stats2 = await service.Update(parameter, words).ConfigureAwait(false); stats1.RepositoryName.Should().Be(stats2.RepositoryName); stats1.SearchKeywordData.Count.Should().Be(3); foreach (SearchKeywordData searchKeywordData in stats1.SearchKeywordData) { Assert.AreEqual(1, searchKeywordData.SearchCount); } stats2.SearchKeywordData.Count.Should().Be(3); foreach (SearchKeywordData searchKeywordData in stats2.SearchKeywordData) { Assert.AreEqual(2, searchKeywordData.SearchCount); } }
public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection <ToolDownloadCountData> data; _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog); using (var connection = await StatisticsDatabase.ConnectTo()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { data = (await connection.QueryWithRetryAsync <ToolDownloadCountData>( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } _logger.LogInformation("Gathered {DownloadedRowsCount} rows of data.", data.Count); if (data.Any()) { // Group based on Package Id var grouped = data.GroupBy(p => p.ToolId); var registrations = new JArray(); foreach (var group in grouped) { var details = new JArray(); details.Add(group.Key); foreach (var gv in group) { var version = new JArray(gv.ToolVersion, gv.TotalDownloadCount); details.Add(version); } registrations.Add(details); } var reportText = registrations.ToString(Formatting.None); foreach (var storageContainerTarget in Targets) { try { var targetBlobContainer = await GetBlobContainer(storageContainerTarget); var blob = targetBlobContainer.GetBlockBlobReference(ReportName); _logger.LogInformation("Writing report to {ReportUri}", blob.Uri.AbsoluteUri); blob.Properties.ContentType = "application/json"; await blob.UploadTextAsync(reportText); _logger.LogInformation("Wrote report to {ReportUri}", blob.Uri.AbsoluteUri); } catch (Exception ex) { _logger.LogError("Error writing report to storage account {StorageAccount}, container {ReportContainer}. {Exception}", storageContainerTarget.StorageAccount.Credentials.AccountName, storageContainerTarget.ContainerName, ex); } } } }
public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); using (var connection = await GalleryDatabase.ConnectTo()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { totalsData = (await connection.QueryWithRetryAsync <GalleryTotalsData>( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } _logger.LogInformation("Total packages: {TotalPackagesCount}", totalsData.TotalPackages); _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog); using (var connection = await StatisticsDatabase.ConnectTo()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync <long>( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, commandTimeout: TimeSpan.FromMinutes(5), transaction: transaction)); } _logger.LogInformation("Total downloads: {TotalDownloadsCount}", totalsData.Downloads); // write to blob totalsData.LastUpdateDateUtc = DateTime.UtcNow; var reportText = JsonConvert.SerializeObject(totalsData); foreach (var storageContainerTarget in Targets) { try { var targetBlobContainer = await GetBlobContainer(storageContainerTarget); var blob = targetBlobContainer.GetBlockBlobReference(ReportName); _logger.LogInformation("Writing report to {ReportUri}", blob.Uri.AbsoluteUri); blob.Properties.ContentType = "application/json"; await blob.UploadTextAsync(reportText); _logger.LogInformation("Wrote report to {ReportUri}", blob.Uri.AbsoluteUri); } catch (Exception ex) { _logger.LogError("Error writing report to storage account {StorageAccount}, container {ReportContainer}. {Exception}", storageContainerTarget.StorageAccount.Credentials.AccountName, storageContainerTarget.ContainerName, ex); } } }
public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection <DownloadCountData> downloadData; Trace.TraceInformation("Gathering Download Counts from {0}/{1}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog); using (var connection = await StatisticsDatabase.ConnectTo()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { downloadData = (await connection.QueryWithRetryAsync <DownloadCountData>( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } Trace.TraceInformation("Gathered {0} rows of data.", downloadData.Count); if (downloadData.Any()) { SemanticVersion semanticVersion = null; // Group based on Package Id var grouped = downloadData.GroupBy(p => p.PackageId); var registrations = new JArray(); foreach (var group in grouped) { var details = new JArray(); details.Add(group.Key); foreach (var gv in group) { // downloads.v1.json should only contain normalized versions - ignore others if (!string.IsNullOrEmpty(gv.PackageVersion) && SemanticVersion.TryParse(gv.PackageVersion, out semanticVersion) && gv.PackageVersion == semanticVersion.ToNormalizedString()) { var version = new JArray(gv.PackageVersion, gv.TotalDownloadCount); details.Add(version); } } registrations.Add(details); } var reportText = registrations.ToString(Formatting.None); foreach (var storageContainerTarget in Targets) { try { var targetBlobContainer = await GetBlobContainer(storageContainerTarget); var blob = targetBlobContainer.GetBlockBlobReference(ReportName); Trace.TraceInformation("Writing report to {0}", blob.Uri.AbsoluteUri); blob.Properties.ContentType = "application/json"; await blob.UploadTextAsync(reportText); Trace.TraceInformation("Wrote report to {0}", blob.Uri.AbsoluteUri); } catch (Exception ex) { Trace.TraceError("Error writing report to storage account {0}, container {1}. {2} {3}", storageContainerTarget.StorageAccount.Credentials.AccountName, storageContainerTarget.ContainerName, ex.Message, ex.StackTrace); } } } }
public StatisticsService(StatisticsDatabase database, IMapper mapper) { this.database = database; this.mapper = mapper; }