private async Task <bool> UpdateVerifiedPackagesAsync()
{
// The "old" data in this case is the latest file that was copied to the region's storage container by this
// job (or initialized by Db2AzureSearch).
var oldResult = await _verifiedPackagesDataClient.ReadLatestAsync(
AccessConditionWrapper.GenerateEmptyCondition(),
_stringCache);
// The "new" data in this case is from the database.
var newData = await _databaseFetcher.GetVerifiedPackagesAsync();
var changes = new HashSet <string>(oldResult.Data, oldResult.Data.Comparer);
changes.SymmetricExceptWith(newData);
_logger.LogInformation("{Count} package IDs have verified status changes.", changes.Count);
if (changes.Count == 0)
{
return(false);
}
else
{
await _verifiedPackagesDataClient.ReplaceLatestAsync(newData, oldResult.Metadata.GetIfMatchCondition());
return(true);
}
}
public void GenerateIfNoneMatchCondition(string etag)
{
var actual = AccessConditionWrapper.GenerateIfNoneMatchCondition(etag);
Assert.Null(actual.IfMatchETag);
Assert.Equal(etag, actual.IfNoneMatchETag);
}
public async Task UpdatesSearchDocumentsWithVersionMatchingAllFilters()
{
VersionListDataResult = new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>
{
{
"1.0.0",
new VersionPropertiesData(listed: true, semVer2: false)
},
}),
AccessConditionWrapper.GenerateIfNotExistsCondition());
var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument);
Assert.Same(VersionListDataResult, indexActions.VersionListDataResult);
Assert.Empty(indexActions.Hijack);
Assert.Equal(4, indexActions.Search.Count);
Assert.All(indexActions.Search, x => Assert.IsType <SearchDocument.UpdateOwners>(x.Document));
Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Merge, x.ActionType));
Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.Default.ToString());
Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrerelease.ToString());
Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludeSemVer2.ToString());
Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrereleaseAndSemVer2.ToString());
}
public void GenerateIfNotExistsCondition()
{
var actual = AccessConditionWrapper.GenerateIfNotExistsCondition();
Assert.Null(actual.IfMatchETag);
Assert.Equal("*", actual.IfNoneMatchETag);
}
public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync()
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName);
var builder = new PackageIdToOwnersBuilder(_logger);
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
{
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
ReadStream(stream, builder.Add);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation("The blob {BlobName} does not exist.", blobName);
}
var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >(
builder.GetResult(),
accessCondition);
stopwatch.Stop();
_telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed);
return(output);
}
public void GenerateEmptyCondition()
{
var actual = AccessConditionWrapper.GenerateEmptyCondition();
Assert.Null(actual.IfMatchETag);
Assert.Null(actual.IfNoneMatchETag);
}
public async Task <ResultAndAccessCondition <VersionListData> > ReadAsync(string id)
{
var blobReference = Container.GetBlobReference(GetFileName(id));
_logger.LogInformation("Reading the version list for package ID {PackageId}.", id);
VersionListData data;
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
using (var streamReader = new StreamReader(stream))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
data = Serializer.Deserialize <VersionListData>(jsonTextReader);
}
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
data = new VersionListData(new Dictionary <string, VersionPropertiesData>());
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
}
return(new ResultAndAccessCondition <VersionListData>(data, accessCondition));
}
private async Task <AuxiliaryFileResult <T> > LoadAsync <T>(
AuxiliaryFileResult <T> previousResult,
Func <IAccessCondition, StringCache, Task <AuxiliaryFileResult <T> > > getResult) where T : class
{
await Task.Yield();
IAccessCondition accessCondition;
if (previousResult == null)
{
accessCondition = AccessConditionWrapper.GenerateEmptyCondition();
}
else
{
accessCondition = AccessConditionWrapper.GenerateIfNoneMatchCondition(previousResult.Metadata.ETag);
}
var newResult = await getResult(accessCondition, _stringCache);
if (newResult.Modified)
{
return(newResult);
}
else
{
return(previousResult);
}
}
private async Task WriteDownloadDataAsync(DownloadData downloadData)
{
_logger.LogInformation("Writing the initial download data file.");
await _downloadDataClient.ReplaceLatestIndexedAsync(
downloadData,
AccessConditionWrapper.GenerateIfNotExistsCondition());
_logger.LogInformation("Done uploading the initial download data file.");
}
private async Task WriteVerifiedPackagesDataAsync(HashSet <string> verifiedPackages)
{
_logger.LogInformation("Writing the initial verified packages data file.");
await _verifiedPackagesDataClient.ReplaceLatestAsync(
verifiedPackages,
AccessConditionWrapper.GenerateIfNotExistsCondition());
_logger.LogInformation("Done uploading the initial verified packages data file.");
}
private async Task WritePopularityTransfersDataAsync(PopularityTransferData popularityTransfers)
{
_logger.LogInformation("Writing the initial popularity transfers data file.");
await _popularityTransferDataClient.ReplaceLatestIndexedAsync(
popularityTransfers,
AccessConditionWrapper.GenerateIfNotExistsCondition());
_logger.LogInformation("Done uploading the initial popularity transfers data file.");
}
public async Task DoesNotEnqueueChangesForNoIndexActions()
{
_config.AzureSearchBatchSize = 2;
_producer
.Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>()))
.ReturnsAsync(() => _initialAuxiliaryData)
.Callback <ConcurrentBag <NewPackageRegistration>, CancellationToken>((w, _) =>
{
w.Add(new NewPackageRegistration("A", 0, new[] { "Microsoft", "EntityFramework" }, new Package[0], false));
w.Add(new NewPackageRegistration("B", 0, new[] { "nuget" }, new Package[0], false));
w.Add(new NewPackageRegistration("C", 0, new[] { "aspnet" }, new Package[0], false));
});
// Return empty index action for ID "B". This package ID will not be pushed to Azure Search but will appear
// in the initial owners data file.
_builder
.Setup(x => x.AddNewPackageRegistration(It.Is <NewPackageRegistration>(y => y.PackageId != "B")))
.Returns <NewPackageRegistration>(x => new IndexActions(
new List <IndexAction <KeyedDocument> > {
IndexAction.Upload(new KeyedDocument {
Key = x.PackageId
})
},
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition())));
_builder
.Setup(x => x.AddNewPackageRegistration(It.Is <NewPackageRegistration>(y => y.PackageId == "B")))
.Returns <NewPackageRegistration>(x => new IndexActions(
new List <IndexAction <KeyedDocument> >(),
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition())));
var enqueuedIndexActions = new List <KeyValuePair <string, IndexActions> >();
_batchPusher
.Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()))
.Callback <string, IndexActions>((id, actions) =>
{
enqueuedIndexActions.Add(KeyValuePair.Create(id, actions));
});
await _target.ExecuteAsync();
Assert.Equal(2, enqueuedIndexActions.Count);
var keys = enqueuedIndexActions
.Select(x => x.Key)
.OrderBy(x => x)
.ToArray();
Assert.Equal(
new[] { "A", "C" },
keys);
}
private async Task WriteOwnerDataAsync(SortedDictionary <string, SortedSet <string> > owners)
{
_logger.LogInformation("Writing the initial owners file.");
await _ownerDataClient.ReplaceLatestIndexedAsync(
owners,
AccessConditionWrapper.GenerateIfNotExistsCondition());
_logger.LogInformation("Done uploading the initial owners file.");
}
public Task <string> CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet)
{
var srcFileName = BuildFileName(validationSet, _fileMetadataService.FileSavePathTemplate, _fileMetadataService.FileExtension);
return(CopyFileAsync(
_fileMetadataService.FileFolderName,
srcFileName,
_fileMetadataService.ValidationFolderName,
BuildValidationSetPackageFileName(validationSet, _fileMetadataService.FileExtension),
AccessConditionWrapper.GenerateEmptyCondition()));
}
public async Task UpdatesSearchDocumentsWithNoVersions()
{
VersionListDataResult = new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateIfNotExistsCondition());
var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument);
Assert.Same(VersionListDataResult, indexActions.VersionListDataResult);
Assert.Empty(indexActions.Hijack);
Assert.Empty(indexActions.Search);
}
private async Task <bool> PushIndexChangesAsync()
{
// The "old" data in this case is the download count data that was last indexed by this job (or
// initialized by Db2AzureSearch).
_logger.LogInformation("Fetching old download count data from blob storage.");
var oldResult = await _downloadDataClient.ReadLatestIndexedAsync(
AccessConditionWrapper.GenerateEmptyCondition(),
_stringCache);
// The "new" data in this case is from the statistics pipeline.
_logger.LogInformation("Fetching new download count data from blob storage.");
var newData = await _auxiliaryFileClient.LoadDownloadDataAsync();
_logger.LogInformation("Removing invalid IDs and versions from the old data.");
CleanDownloadData(oldResult.Data);
_logger.LogInformation("Removing invalid IDs and versions from the new data.");
CleanDownloadData(newData);
// Fetch the download overrides from the auxiliary file. Note that the overriden downloads are kept
// separate from downloads data as the original data will be persisted to auxiliary data, whereas the
// overriden data will be persisted to Azure Search.
_logger.LogInformation("Overriding download count data.");
var downloadOverrides = await _auxiliaryFileClient.LoadDownloadOverridesAsync();
var overridenDownloads = newData.ApplyDownloadOverrides(downloadOverrides, _logger);
_logger.LogInformation("Detecting download count changes.");
var changes = _downloadSetComparer.Compare(oldResult.Data, overridenDownloads);
var idBag = new ConcurrentBag <string>(changes.Keys);
_logger.LogInformation("{Count} package IDs have download count changes.", idBag.Count);
if (!changes.Any())
{
return(false);
}
_logger.LogInformation(
"Starting {Count} workers pushing download count changes to Azure Search.",
_options.Value.MaxConcurrentBatches);
await ParallelAsync.Repeat(
() => WorkAsync(idBag, changes),
_options.Value.MaxConcurrentBatches);
_logger.LogInformation("All of the download count changes have been pushed to Azure Search.");
_logger.LogInformation("Uploading the new download count data to blob storage.");
await _downloadDataClient.ReplaceLatestIndexedAsync(newData, oldResult.Metadata.GetIfMatchCondition());
return(true);
}
public virtual Task CopyValidationPackageToPackageFileAsync(PackageValidationSet validationSet)
{
var fileName = BuildFileName(validationSet,
_fileMetadataService.FileSavePathTemplate,
_fileMetadataService.FileExtension);
return(CopyFileAsync(
_fileMetadataService.ValidationFolderName,
fileName,
_fileMetadataService.FileFolderName,
fileName,
AccessConditionWrapper.GenerateIfNotExistsCondition()));
}
public async Task PushesToIndexesUsingMaximumBatchSize()
{
_config.AzureSearchBatchSize = 2;
_producer
.Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>()))
.ReturnsAsync(() => _initialAuxiliaryData)
.Callback <ConcurrentBag <NewPackageRegistration>, CancellationToken>((w, _) =>
{
w.Add(new NewPackageRegistration("A", 0, new string[0], new Package[0], false));
w.Add(new NewPackageRegistration("B", 0, new string[0], new Package[0], false));
w.Add(new NewPackageRegistration("C", 0, new string[0], new Package[0], false));
w.Add(new NewPackageRegistration("D", 0, new string[0], new Package[0], false));
w.Add(new NewPackageRegistration("E", 0, new string[0], new Package[0], false));
});
_builder
.Setup(x => x.AddNewPackageRegistration(It.IsAny <NewPackageRegistration>()))
.Returns <NewPackageRegistration>(x => new IndexActions(
new List <IndexAction <KeyedDocument> > {
IndexAction.Upload(new KeyedDocument {
Key = x.PackageId
})
},
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition())));
var enqueuedIndexActions = new List <KeyValuePair <string, IndexActions> >();
_batchPusher
.Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()))
.Callback <string, IndexActions>((id, actions) =>
{
enqueuedIndexActions.Add(KeyValuePair.Create(id, actions));
});
await _target.ExecuteAsync();
Assert.Equal(5, enqueuedIndexActions.Count);
var keys = enqueuedIndexActions
.Select(x => x.Key)
.OrderBy(x => x)
.ToArray();
Assert.Equal(
new[] { "A", "B", "C", "D", "E" },
keys);
_batchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(5));
_batchPusher.Verify(x => x.TryFinishAsync(), Times.Once);
}
public Task <string> CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet)
{
var srcFileName = BuildFileName(
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
CoreConstants.PackageFileSavePathTemplate,
CoreConstants.NuGetPackageFileExtension);
return(CopyFileAsync(
CoreConstants.PackagesFolderName,
srcFileName,
CoreConstants.ValidationFolderName,
BuildValidationSetPackageFileName(validationSet),
AccessConditionWrapper.GenerateEmptyCondition()));
}
public Task CopyValidationPackageToPackageFileAsync(string id, string normalizedVersion)
{
var fileName = BuildFileName(
id,
normalizedVersion,
CoreConstants.PackageFileSavePathTemplate,
CoreConstants.NuGetPackageFileExtension);
return(CopyFileAsync(
CoreConstants.ValidationFolderName,
fileName,
CoreConstants.PackagesFolderName,
fileName,
AccessConditionWrapper.GenerateIfNotExistsCondition()));
}
public Task CopyPackageUrlForValidationSetAsync(PackageValidationSet validationSet, string srcPackageUrl)
{
var destFileName = BuildValidationSetPackageFileName(validationSet, _fileMetadataService.FileExtension);
_logger.LogInformation(
"Copying URL {SrcPackageUrl} to {DestFolderName}/{DestFileName}.",
srcPackageUrl,
_fileMetadataService.ValidationFolderName,
srcPackageUrl);
return(_fileStorageService.CopyFileAsync(
new Uri(srcPackageUrl),
_fileMetadataService.ValidationFolderName,
destFileName,
AccessConditionWrapper.GenerateEmptyCondition()));
}
public void RejectsEmptyEnqueue()
{
var emptyIndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> >(),
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition()));
var ex = Assert.Throws <ArgumentException>(
() => _target.EnqueueIndexActions(IdA, emptyIndexActions));
Assert.Contains("There must be at least one index action.", ex.Message);
Assert.Empty(_target._searchActions);
Assert.Empty(_target._hijackActions);
Assert.Empty(_target._idReferenceCount);
Assert.Empty(_target._versionListDataResults);
}
public async Task CopyValidationSetPackageToPackageFileAsync()
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(_etag);
_fileStorageService
.Setup(x => x.CopyFileAsync(
_validationContainerName,
_validationSetPackageFileName,
_packagesContainerName,
_packageFileName,
accessCondition))
.ReturnsAsync(_etag)
.Verifiable();
await _target.CopyValidationSetPackageToPackageFileAsync(_validationSet, accessCondition);
_fileStorageService.Verify();
}
public IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration)
{
var versionProperties = new Dictionary <string, VersionPropertiesData>();
var versionListData = new VersionListData(versionProperties);
var versionLists = new VersionLists(versionListData);
var changes = packageRegistration
.Packages
.Select(GetVersionListChange)
.ToList();
var indexChanges = versionLists.ApplyChanges(changes);
var versionToPackage = packageRegistration
.Packages
.ToDictionary(p => NuGetVersion.Parse(p.Version));
var search = indexChanges
.Search
.Select(p => GetSearchIndexAction(
packageRegistration,
versionToPackage,
versionLists,
p.Key,
p.Value))
.ToList();
var hijack = indexChanges
.Hijack
.Select(p => GetHijackIndexAction(
packageRegistration.PackageId,
versionToPackage[p.Key],
p.Value))
.ToList();
return(new IndexActions(
search,
hijack,
new ResultAndAccessCondition <VersionListData>(
versionLists.GetVersionListData(),
AccessConditionWrapper.GenerateEmptyCondition())));
}
public Facts(ITestOutputHelper output)
{
VersionListDataClient = new Mock <IVersionListDataClient>();
Search = new Mock <ISearchDocumentBuilder>();
Logger = output.GetLogger <SearchIndexActionBuilder>();
VersionListDataResult = new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateIfNotExistsCondition());
VersionListDataClient
.Setup(x => x.ReadAsync(It.IsAny <string>()))
.ReturnsAsync(() => VersionListDataResult);
Search
.Setup(x => x.UpdateOwners(It.IsAny <string>(), It.IsAny <SearchFilters>(), It.IsAny <string[]>()))
.Returns <string, SearchFilters, string[]>((_, sf, __) => new SearchDocument.UpdateOwners
{
Key = sf.ToString(),
});
Target = new SearchIndexActionBuilder(VersionListDataClient.Object, Logger);
}
public async Task <FeatureFlagSaveResult> TrySaveAsync(FeatureFlags flags, string contentId)
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(contentId);
try
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
using (var jsonWriter = new JsonTextWriter(writer))
{
Serializer.Serialize(jsonWriter, flags);
jsonWriter.Flush();
stream.Position = 0;
await _storage.SaveFileAsync(CoreConstants.Folders.ContentFolderName, CoreConstants.FeatureFlagsFileName, stream, accessCondition);
return(FeatureFlagSaveResult.Ok);
}
}
catch (StorageException e) when(e.IsPreconditionFailedException())
{
return(FeatureFlagSaveResult.Conflict);
}
}
private async Task <bool> UpdatePublicPackageAsync(PackageValidationSet validationSet)
{
_logger.LogInformation("Copying .nupkg to public storage for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
// If the validation set contains any processors, we must use the copy of the package that is specific to
// this validation set. We can't use the original validation package because it does not have any of the
// changes that the processors made. If the validation set package does not exist for some reason and there
// are processors in the validation set, this indicates a bug and an exception will be thrown by the copy
// operation below. This will cause the validation queue message to eventually dead-letter at which point
// the on-call person should investigate.
bool copied;
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)) ||
await _packageFileService.DoesValidationSetPackageExistAsync(validationSet))
{
IAccessCondition destAccessCondition;
// The package etag will be null if this validation set is expecting the package to not yet exist in
// the packages container.
if (validationSet.PackageETag == null)
{
// This will fail with HTTP 409 if the package already exists. This means that another validation
// set has completed and moved the package into the Available state first, with different package
// content.
destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package does not already exist.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
}
else
{
// This will fail with HTTP 412 if the package has been modified by another validation set. This
// would only happen if this validation set and another validation set are operating on a package
// already in the Available state.
destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(validationSet.PackageETag);
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package has etag {PackageETag}.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.PackageETag);
}
// Failures here should result in an unhandled exception. This means that this validation set has
// modified the package but is unable to copy the modified package into the packages container because
// another validation set completed first.
await _packageFileService.CopyValidationSetPackageToPackageFileAsync(
validationSet,
destAccessCondition);
copied = true;
}
else
{
_logger.LogInformation(
"The package specific to the validation set does not exist. Falling back to the validation " +
"container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
try
{
await _packageFileService.CopyValidationPackageToPackageFileAsync(validationSet);
copied = true;
}
catch (InvalidOperationException)
{
// The package already exists in the packages container. This can happen if the DB commit below fails
// and this flow is retried or another validation set for the package completed first. Either way, we
// will later attempt to use the hash from the package in the packages container (the destination).
// In other words, we don't care which copy wins when copying from the validation package because
// we know the package has not been modified.
_logger.LogInformation(
"Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
copied = false;
}
}
return(copied);
}
public Db2AzureSearchCommandFacts(ITestOutputHelper output)
{
_producer = new Mock <INewPackageRegistrationProducer>();
_builder = new Mock <IPackageEntityIndexActionBuilder>();
_blobContainerBuilder = new Mock <IBlobContainerBuilder>();
_indexBuilder = new Mock <IIndexBuilder>();
_batchPusher = new Mock <IBatchPusher>();
_catalogClient = new Mock <ICatalogClient>();
_storageFactory = new Mock <IStorageFactory>();
_ownerDataClient = new Mock <IOwnerDataClient>();
_downloadDataClient = new Mock <IDownloadDataClient>();
_verifiedPackagesDataClient = new Mock <IVerifiedPackagesDataClient>();
_popularityTransferDataClient = new Mock <IPopularityTransferDataClient>();
_options = new Mock <IOptionsSnapshot <Db2AzureSearchConfiguration> >();
_developmentOptions = new Mock <IOptionsSnapshot <Db2AzureSearchDevelopmentConfiguration> >();
_logger = output.GetLogger <Db2AzureSearchCommand>();
_config = new Db2AzureSearchConfiguration
{
MaxConcurrentBatches = 1,
StorageContainer = "container-name",
};
_developmentConfig = new Db2AzureSearchDevelopmentConfiguration();
_storage = new TestCursorStorage(new Uri("https://example/base/"));
_initialAuxiliaryData = new InitialAuxiliaryData(
owners: new SortedDictionary <string, SortedSet <string> >(),
downloads: new DownloadData(),
excludedPackages: new HashSet <string>(),
verifiedPackages: new HashSet <string>(),
popularityTransfers: new PopularityTransferData());
_options
.Setup(x => x.Value)
.Returns(() => _config);
_developmentOptions
.Setup(x => x.Value)
.Returns(() => _developmentConfig);
_producer
.Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>()))
.ReturnsAsync(() => _initialAuxiliaryData);
_builder
.Setup(x => x.AddNewPackageRegistration(It.IsAny <NewPackageRegistration>()))
.Returns(() => new IndexActions(
new IndexAction <KeyedDocument> [0],
new IndexAction <KeyedDocument> [0],
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition())));
_batchPusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult()));
_catalogClient
.Setup(x => x.GetIndexAsync(It.IsAny <string>()))
.ReturnsAsync(new CatalogIndex());
_storageFactory
.Setup(x => x.Create(It.IsAny <string>()))
.Returns(() => _storage);
_blobContainerBuilder
.Setup(x => x.DeleteIfExistsAsync())
.ReturnsAsync(true);
_target = new Db2AzureSearchCommand(
_producer.Object,
_builder.Object,
_blobContainerBuilder.Object,
_indexBuilder.Object,
() => _batchPusher.Object,
_catalogClient.Object,
_storageFactory.Object,
_ownerDataClient.Object,
_downloadDataClient.Object,
_verifiedPackagesDataClient.Object,
_popularityTransferDataClient.Object,
_options.Object,
_developmentOptions.Object,
_logger);
}
private async Task <bool> PushIndexChangesAsync()
{
// The "old" data in this case is the download count data that was last indexed by this job (or
// initialized by Db2AzureSearch).
_logger.LogInformation("Fetching old download count data from blob storage.");
var oldResult = await _downloadDataClient.ReadLatestIndexedAsync(
AccessConditionWrapper.GenerateEmptyCondition(),
_stringCache);
// The "new" data in this case is from the statistics pipeline.
_logger.LogInformation("Fetching new download count data from blob storage.");
var newData = await _auxiliaryFileClient.LoadDownloadDataAsync();
_logger.LogInformation("Removing invalid IDs and versions from the old downloads data.");
CleanDownloadData(oldResult.Data);
_logger.LogInformation("Removing invalid IDs and versions from the new downloads data.");
CleanDownloadData(newData);
_logger.LogInformation("Detecting download count changes.");
var changes = _downloadSetComparer.Compare(oldResult.Data, newData);
_logger.LogInformation("{Count} package IDs have download count changes.", changes.Count);
// The "old" data is the popularity transfers data that was last indexed by this job (or
// initialized by Db2AzureSearch).
_logger.LogInformation("Fetching old popularity transfer data from blob storage.");
var oldTransfers = await _popularityTransferDataClient.ReadLatestIndexedAsync(
AccessConditionWrapper.GenerateEmptyCondition(),
_stringCache);
// The "new" data is the latest popularity transfers data from the database.
_logger.LogInformation("Fetching new popularity transfer data from database.");
var newTransfers = await GetPopularityTransfersAsync();
_logger.LogInformation("Applying download transfers to download changes.");
ApplyDownloadTransfers(
newData,
oldTransfers.Data,
newTransfers,
changes);
var idBag = new ConcurrentBag <string>(changes.Keys);
_logger.LogInformation("{Count} package IDs need to be updated.", idBag.Count);
if (!changes.Any())
{
return(false);
}
_logger.LogInformation(
"Starting {Count} workers pushing download count changes to Azure Search.",
_options.Value.MaxConcurrentBatches);
await ParallelAsync.Repeat(
() => WorkAndRetryAsync(idBag, changes),
_options.Value.MaxConcurrentBatches);
_logger.LogInformation("All of the download count changes have been pushed to Azure Search.");
_logger.LogInformation("Uploading the new download count data to blob storage.");
await _downloadDataClient.ReplaceLatestIndexedAsync(newData, oldResult.Metadata.GetIfMatchCondition());
_logger.LogInformation("Uploading the new popularity transfer data to blob storage.");
await _popularityTransferDataClient.ReplaceLatestIndexedAsync(
newTransfers,
oldTransfers.Metadata.GetIfMatchCondition());
return(true);
}
public BaseFacts(ITestOutputHelper output)
{
_logger = output.GetLogger <BatchPusher>();
_searchIndexClientWrapper = new Mock <ISearchIndexClientWrapper>();
_searchDocumentsWrapper = new Mock <IDocumentsOperationsWrapper>();
_hijackIndexClientWrapper = new Mock <ISearchIndexClientWrapper>();
_hijackDocumentsWrapper = new Mock <IDocumentsOperationsWrapper>();
_versionListDataClient = new Mock <IVersionListDataClient>();
_config = new AzureSearchJobConfiguration();
_developmentConfig = new AzureSearchJobDevelopmentConfiguration();
_options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >();
_developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();
_telemetryService = new Mock <IAzureSearchTelemetryService>();
_searchIndexClientWrapper.Setup(x => x.IndexName).Returns("search");
_searchIndexClientWrapper.Setup(x => x.Documents).Returns(() => _searchDocumentsWrapper.Object);
_hijackIndexClientWrapper.Setup(x => x.IndexName).Returns("hijack");
_hijackIndexClientWrapper.Setup(x => x.Documents).Returns(() => _hijackDocumentsWrapper.Object);
_versionListDataClient
.Setup(x => x.TryReplaceAsync(It.IsAny <string>(), It.IsAny <VersionListData>(), It.IsAny <IAccessCondition>()))
.ReturnsAsync(true);
_options.Setup(x => x.Value).Returns(() => _config);
_developmentOptions.Setup(x => x.Value).Returns(() => _developmentConfig);
_searchBatches = new List <IndexBatch <KeyedDocument> >();
_hijackBatches = new List <IndexBatch <KeyedDocument> >();
_searchDocumentsWrapper
.Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
.ReturnsAsync(() => new DocumentIndexResult(new List <IndexingResult>()))
.Callback <IndexBatch <KeyedDocument> >(b => _searchBatches.Add(b));
_hijackDocumentsWrapper
.Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
.ReturnsAsync(() => new DocumentIndexResult(new List <IndexingResult>()))
.Callback <IndexBatch <KeyedDocument> >(b => _hijackBatches.Add(b));
_config.AzureSearchBatchSize = 2;
_config.MaxConcurrentVersionListWriters = 1;
_searchDocumentA = IndexAction.Upload(new KeyedDocument());
_searchDocumentB = IndexAction.Upload(new KeyedDocument());
_searchDocumentC = IndexAction.Upload(new KeyedDocument());
_searchDocuments = new List <IndexAction <KeyedDocument> >
{
_searchDocumentA,
_searchDocumentB,
_searchDocumentC,
};
_hijackDocumentA = IndexAction.Upload(new KeyedDocument());
_hijackDocumentB = IndexAction.Upload(new KeyedDocument());
_hijackDocumentC = IndexAction.Upload(new KeyedDocument());
_hijackDocumentD = IndexAction.Upload(new KeyedDocument());
_hijackDocumentE = IndexAction.Upload(new KeyedDocument());
_hijackDocuments = new List <IndexAction <KeyedDocument> >
{
_hijackDocumentA,
_hijackDocumentB,
_hijackDocumentC,
_hijackDocumentD,
_hijackDocumentE,
};
_indexActions = new IndexActions(
_searchDocuments,
_hijackDocuments,
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
AccessConditionWrapper.GenerateEmptyCondition()));
_target = new BatchPusher(
_searchIndexClientWrapper.Object,
_hijackIndexClientWrapper.Object,
_versionListDataClient.Object,
_options.Object,
_developmentOptions.Object,
_telemetryService.Object,
_logger);
}
