public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync()
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName);
var builder = new PackageIdToOwnersBuilder(_logger);
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
{
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
ReadStream(stream, builder.Add);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation("The blob {BlobName} does not exist.", blobName);
}
var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >(
builder.GetResult(),
accessCondition);
stopwatch.Stop();
_telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed);
return(output);
}
/// <summary>
/// A helper to turn lines formatted like this "PackageId: OwnerA, OwnerB" into package ID to owners
/// dictionary.
/// </summary>
public SortedDictionary <string, SortedSet <string> > OwnersData(params string[] lines)
{
var builder = new PackageIdToOwnersBuilder(Logger);
ParseData(lines, builder.Add);
return(builder.GetResult());
}
/// <summary>
/// A helper to turn lines formatted like this "PackageId: OwnerA, OwnerB" into package ID to owners
/// dictionary.
/// </summary>
public SortedDictionary <string, SortedSet <string> > Data(params string[] lines)
{
var builder = new PackageIdToOwnersBuilder(Logger);
foreach (var line in lines)
{
var pieces = line.Split(new[] { ':' }, 2);
var id = pieces[0].Trim();
var usernames = pieces[1]
.Split(',')
.Select(x => x.Trim())
.Where(x => x.Length > 0)
.ToList();
builder.Add(id, usernames);
}
return(builder.GetResult());
}
public async Task <InitialAuxiliaryData> ProduceWorkAsync(
ConcurrentBag <NewPackageRegistration> allWork,
CancellationToken cancellationToken)
{
var ranges = await GetPackageRegistrationRangesAsync();
// Fetch exclude packages list from auxiliary files.
// These packages are excluded from the default search's results.
var excludedPackages = await _auxiliaryFileClient.LoadExcludedPackagesAsync();
Guard.Assert(
excludedPackages.Comparer == StringComparer.OrdinalIgnoreCase,
$"Excluded packages HashSet should be using {nameof(StringComparer.OrdinalIgnoreCase)}");
// Fetch the download data from the auxiliary file, since this is what is used for displaying download
// counts in the search service. We don't use the gallery DB values as they are different from the
// auxiliary file.
var downloads = await _auxiliaryFileClient.LoadDownloadDataAsync();
var popularityTransfers = await GetPopularityTransfersAsync();
// Apply changes from popularity transfers.
var transferredDownloads = GetTransferredDownloads(downloads, popularityTransfers);
// Build a list of the owners data and verified IDs as we collect package registrations from the database.
var ownersBuilder = new PackageIdToOwnersBuilder(_logger);
var verifiedPackages = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < ranges.Count && !cancellationToken.IsCancellationRequested; i++)
{
if (ShouldWait(allWork, log: true))
{
while (ShouldWait(allWork, log: false))
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
_logger.LogInformation("Resuming fetching package registrations from the database.");
}
var range = ranges[i];
var allPackages = await GetPackagesAsync(range);
var keyToPackages = allPackages
.GroupBy(x => x.PackageRegistrationKey)
.ToDictionary(x => x.Key, x => x.ToList());
var packageRegistrationInfo = await GetPackageRegistrationInfoAsync(range);
foreach (var pr in packageRegistrationInfo)
{
if (!transferredDownloads.TryGetValue(pr.Id, out var packageDownloads))
{
packageDownloads = 0;
}
if (!keyToPackages.TryGetValue(pr.Key, out var packages))
{
packages = new List <Package>();
}
var isExcludedByDefault = excludedPackages.Contains(pr.Id);
allWork.Add(new NewPackageRegistration(
pr.Id,
packageDownloads,
pr.Owners,
packages,
isExcludedByDefault));
ownersBuilder.Add(pr.Id, pr.Owners);
if (pr.IsVerified)
{
verifiedPackages.Add(pr.Id);
}
}
_logger.LogInformation("Done initializing batch {Number}/{Count}.", i + 1, ranges.Count);
}
return(new InitialAuxiliaryData(
ownersBuilder.GetResult(),
downloads,
excludedPackages,
verifiedPackages,
popularityTransfers));
}