public async Task AddSameTags_VariousRepos_FlattenedProperly()
{
//arrange
var db = new StatisticsDatabase(Settings);
StatisticsService service = new StatisticsService(db, new Mapper(MappingConfigurationFactory.Create()));
var parameter1 = new RepositoryQueryParameter(Guid.NewGuid().ToString(), Guid.NewGuid().ToString());
await service.Update(parameter1, new [] { "FirstTag", "SecondTag", "Third" }).ConfigureAwait(false);
await service.Update(parameter1, new[] { "FIRSTTAG", "", "Fourth" }).ConfigureAwait(false);
await service.Update(parameter1, new[] { "FIRSTTAG", "SecondTag", "Fourth" }).ConfigureAwait(false);
parameter1 = new RepositoryQueryParameter(Guid.NewGuid().ToString(), Guid.NewGuid().ToString());
await service.Update(parameter1, new[] { "FirstTag", "SecondTag", "Fourth" }).ConfigureAwait(false);
await service.Update(parameter1, new[] { "Fifth", "Sixth" }).ConfigureAwait(false);
//act
List <SearchKeywordData> result = (await service.GetFlattened().ConfigureAwait(false)).ToList();
//assert
result.Count.Should().Be(6);
result.Single(x => x.Keyword == "FirstTag").SearchCount.Should().Be(4);
result.Single(x => x.Keyword == "SecondTag").SearchCount.Should().Be(3);
result.Single(x => x.Keyword == "Third").SearchCount.Should().Be(1);
result.Single(x => x.Keyword == "Fourth").SearchCount.Should().Be(3);
result.Single(x => x.Keyword == "Fifth").SearchCount.Should().Be(1);
result.Single(x => x.Keyword == "Sixth").SearchCount.Should().Be(1);
}
public async Task AddSameTags_ValuesIncrementedProperly()
{
var db = new StatisticsDatabase(Settings);
StatisticsService service = new StatisticsService(db, new Mapper(MappingConfigurationFactory.Create()));
string organizationOne = Guid.NewGuid().ToString();
string repoOne = Guid.NewGuid().ToString();
var parameter = new RepositoryQueryParameter(organizationOne, repoOne);
string[] words = new[] { "FirstTag", "SecondTag", "Third" };
SearchStatistics stats1 = await service.Update(parameter, words).ConfigureAwait(false);
SearchStatistics stats2 = await service.Update(parameter, words).ConfigureAwait(false);
stats1.RepositoryName.Should().Be(stats2.RepositoryName);
stats1.SearchKeywordData.Count.Should().Be(3);
foreach (SearchKeywordData searchKeywordData in stats1.SearchKeywordData)
{
Assert.AreEqual(1, searchKeywordData.SearchCount);
}
stats2.SearchKeywordData.Count.Should().Be(3);
foreach (SearchKeywordData searchKeywordData in stats2.SearchKeywordData)
{
Assert.AreEqual(2, searchKeywordData.SearchCount);
}
}
public async Task Run()
{
// Gather download count data from statistics warehouse
IReadOnlyCollection <ToolDownloadCountData> data;
_logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog);
using (var connection = await StatisticsDatabase.ConnectTo())
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
{
data = (await connection.QueryWithRetryAsync <ToolDownloadCountData>(
_storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
}
_logger.LogInformation("Gathered {DownloadedRowsCount} rows of data.", data.Count);
if (data.Any())
{
// Group based on Package Id
var grouped = data.GroupBy(p => p.ToolId);
var registrations = new JArray();
foreach (var group in grouped)
{
var details = new JArray();
details.Add(group.Key);
foreach (var gv in group)
{
var version = new JArray(gv.ToolVersion, gv.TotalDownloadCount);
details.Add(version);
}
registrations.Add(details);
}
var reportText = registrations.ToString(Formatting.None);
foreach (var storageContainerTarget in Targets)
{
try
{
var targetBlobContainer = await GetBlobContainer(storageContainerTarget);
var blob = targetBlobContainer.GetBlockBlobReference(ReportName);
_logger.LogInformation("Writing report to {ReportUri}", blob.Uri.AbsoluteUri);
blob.Properties.ContentType = "application/json";
await blob.UploadTextAsync(reportText);
_logger.LogInformation("Wrote report to {ReportUri}", blob.Uri.AbsoluteUri);
}
catch (Exception ex)
{
_logger.LogError("Error writing report to storage account {StorageAccount}, container {ReportContainer}. {Exception}",
storageContainerTarget.StorageAccount.Credentials.AccountName,
storageContainerTarget.ContainerName,
ex);
}
}
}
}
public async Task Run()
{
// gather package numbers from gallery database
GalleryTotalsData totalsData;
_logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
using (var connection = await GalleryDatabase.ConnectTo())
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
{
totalsData = (await connection.QueryWithRetryAsync <GalleryTotalsData>(
GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First();
}
_logger.LogInformation("Total packages: {TotalPackagesCount}", totalsData.TotalPackages);
_logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages);
// gather download count data from statistics warehouse
_logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog);
using (var connection = await StatisticsDatabase.ConnectTo())
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
{
totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync <long>(
WarehouseStoredProcedureName,
commandType: CommandType.StoredProcedure,
commandTimeout: TimeSpan.FromMinutes(5),
transaction: transaction));
}
_logger.LogInformation("Total downloads: {TotalDownloadsCount}", totalsData.Downloads);
// write to blob
totalsData.LastUpdateDateUtc = DateTime.UtcNow;
var reportText = JsonConvert.SerializeObject(totalsData);
foreach (var storageContainerTarget in Targets)
{
try
{
var targetBlobContainer = await GetBlobContainer(storageContainerTarget);
var blob = targetBlobContainer.GetBlockBlobReference(ReportName);
_logger.LogInformation("Writing report to {ReportUri}", blob.Uri.AbsoluteUri);
blob.Properties.ContentType = "application/json";
await blob.UploadTextAsync(reportText);
_logger.LogInformation("Wrote report to {ReportUri}", blob.Uri.AbsoluteUri);
}
catch (Exception ex)
{
_logger.LogError("Error writing report to storage account {StorageAccount}, container {ReportContainer}. {Exception}",
storageContainerTarget.StorageAccount.Credentials.AccountName,
storageContainerTarget.ContainerName,
ex);
}
}
}
public async Task Run()
{
// Gather download count data from statistics warehouse
IReadOnlyCollection <DownloadCountData> downloadData;
Trace.TraceInformation("Gathering Download Counts from {0}/{1}...", StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog);
using (var connection = await StatisticsDatabase.ConnectTo())
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
{
downloadData = (await connection.QueryWithRetryAsync <DownloadCountData>(
_storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
}
Trace.TraceInformation("Gathered {0} rows of data.", downloadData.Count);
if (downloadData.Any())
{
SemanticVersion semanticVersion = null;
// Group based on Package Id
var grouped = downloadData.GroupBy(p => p.PackageId);
var registrations = new JArray();
foreach (var group in grouped)
{
var details = new JArray();
details.Add(group.Key);
foreach (var gv in group)
{
// downloads.v1.json should only contain normalized versions - ignore others
if (!string.IsNullOrEmpty(gv.PackageVersion) &&
SemanticVersion.TryParse(gv.PackageVersion, out semanticVersion) &&
gv.PackageVersion == semanticVersion.ToNormalizedString())
{
var version = new JArray(gv.PackageVersion, gv.TotalDownloadCount);
details.Add(version);
}
}
registrations.Add(details);
}
var reportText = registrations.ToString(Formatting.None);
foreach (var storageContainerTarget in Targets)
{
try
{
var targetBlobContainer = await GetBlobContainer(storageContainerTarget);
var blob = targetBlobContainer.GetBlockBlobReference(ReportName);
Trace.TraceInformation("Writing report to {0}", blob.Uri.AbsoluteUri);
blob.Properties.ContentType = "application/json";
await blob.UploadTextAsync(reportText);
Trace.TraceInformation("Wrote report to {0}", blob.Uri.AbsoluteUri);
}
catch (Exception ex)
{
Trace.TraceError("Error writing report to storage account {0}, container {1}. {2} {3}",
storageContainerTarget.StorageAccount.Credentials.AccountName,
storageContainerTarget.ContainerName, ex.Message, ex.StackTrace);
}
}
}
}
public StatisticsService(StatisticsDatabase database, IMapper mapper)
{
this.database = database;
this.mapper = mapper;
}