[InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size.
public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes)
{
var changeCount = 30;
var expectedPushes = fullPushes + partialPushes;
Config.AzureSearchBatchSize = 10;
IndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> >(
Enumerable
.Range(0, documentsPerId)
.Select(x => IndexAction.Merge(new KeyedDocument()))),
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
new Mock <IAccessCondition>().Object));
AddChanges(changeCount);
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.Success);
VerifyAllIdsAreProcessed(changeCount);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(changeCount));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(changeCount));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes));
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
SystemTime.Verify(x => x.Delay(It.IsAny <TimeSpan>()), Times.Exactly(expectedPushes - 1));
DownloadDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(
NewDownloadData,
It.Is <IAccessCondition>(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)),
Times.Once);
Assert.Equal(
fullPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize));
Assert.Equal(
partialPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize));
Assert.Empty(CurrentBatch);
}
public DocumentDatabase(InMemoryRavenConfiguration configuration, TransportState transportState = null)
{
DocumentLock = new PutSerialLock();
Name = configuration.DatabaseName;
Configuration = configuration;
this.transportState = transportState ?? new TransportState();
ExtensionsState = new AtomicDictionary <object>();
using (LogManager.OpenMappedContext("database", Name ?? Constants.SystemDatabase))
{
Log.Debug("Start loading the following database: {0}", Name ?? Constants.SystemDatabase);
initializer = new DocumentDatabaseInitializer(this, configuration);
initializer.InitializeEncryption();
initializer.ValidateLicense();
initializer.SubscribeToDomainUnloadOrProcessExit();
initializer.ExecuteAlterConfiguration();
initializer.SatisfyImportsOnce();
backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Default;
recentTouches = new SizeLimitedConcurrentDictionary <string, TouchedDocumentInfo>(configuration.MaxRecentTouchesToRemember, StringComparer.OrdinalIgnoreCase);
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
Database = this,
DatabaseName = Name,
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers,
TaskScheduler = backgroundTaskScheduler,
Configuration = configuration,
IndexReaderWarmers = IndexReaderWarmers
};
try
{
uuidGenerator = new SequentialUuidGenerator();
initializer.InitializeTransactionalStorage(uuidGenerator);
lastCollectionEtags = new LastCollectionEtags(TransactionalStorage, WorkContext);
}
catch (Exception)
{
if (TransactionalStorage != null)
{
TransactionalStorage.Dispose();
}
throw;
}
try
{
TransactionalStorage.Batch(actions => uuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
// Index codecs must be initialized before we try to read an index
InitializeIndexCodecTriggers();
initializer.InitializeIndexStorage();
Attachments = new AttachmentActions(this, recentTouches, uuidGenerator, Log);
Documents = new DocumentActions(this, recentTouches, uuidGenerator, Log);
Indexes = new IndexActions(this, recentTouches, uuidGenerator, Log);
Maintenance = new MaintenanceActions(this, recentTouches, uuidGenerator, Log);
Notifications = new NotificationActions(this, recentTouches, uuidGenerator, Log);
Patches = new PatchActions(this, recentTouches, uuidGenerator, Log);
Queries = new QueryActions(this, recentTouches, uuidGenerator, Log);
Tasks = new TaskActions(this, recentTouches, uuidGenerator, Log);
Transformers = new TransformerActions(this, recentTouches, uuidGenerator, Log);
inFlightTransactionalState = TransactionalStorage.GetInFlightTransactionalState(Documents.Put, Documents.Delete);
CompleteWorkContextSetup();
prefetcher = new Prefetcher(workContext);
indexingExecuter = new IndexingExecuter(workContext, prefetcher);
RaiseIndexingWiringComplete();
InitializeTriggersExceptIndexCodecs();
SecondStageInitialization();
ExecuteStartupTasks();
lastCollectionEtags.Initialize();
Log.Debug("Finish loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);
}
catch (Exception)
{
Dispose();
throw;
}
}
}
public Facts(ITestOutputHelper output)
{
DatabaseOwnerFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();
OwnerDataClient = new Mock <IOwnerDataClient>();
OwnerSetComparer = new Mock <IDataSetComparer>();
SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>();
SearchIndexActionBuilder = new Mock <ISearchIndexActionBuilder>();
Pusher = new Mock <IBatchPusher>();
Options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >();
TelemetryService = new Mock <IAzureSearchTelemetryService>();
Logger = output.GetLogger <UpdateOwnersCommand>();
Configuration = new AzureSearchJobConfiguration
{
MaxConcurrentBatches = 1,
};
DatabaseResult = new SortedDictionary <string, SortedSet <string> >();
StorageResult = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >(
new SortedDictionary <string, SortedSet <string> >(),
new Mock <IAccessCondition>().Object);
Changes = new SortedDictionary <string, string[]>();
IndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> > {
IndexAction.Merge(new KeyedDocument())
},
new List <IndexAction <KeyedDocument> > {
IndexAction.Merge(new KeyedDocument())
},
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
new Mock <IAccessCondition>().Object));
Pusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult()));
Options
.Setup(x => x.Value)
.Returns(() => Configuration);
DatabaseOwnerFetcher
.Setup(x => x.GetPackageIdToOwnersAsync())
.ReturnsAsync(() => DatabaseResult);
OwnerDataClient
.Setup(x => x.ReadLatestIndexedAsync())
.ReturnsAsync(() => StorageResult);
OwnerSetComparer
.Setup(x => x.CompareOwners(
It.IsAny <SortedDictionary <string, SortedSet <string> > >(),
It.IsAny <SortedDictionary <string, SortedSet <string> > >()))
.Returns(() => Changes);
SearchIndexActionBuilder
.Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()))
.ReturnsAsync(() => IndexActions);
Target = new UpdateOwnersCommand(
DatabaseOwnerFetcher.Object,
OwnerDataClient.Object,
OwnerSetComparer.Object,
SearchDocumentBuilder.Object,
SearchIndexActionBuilder.Object,
() => Pusher.Object,
Options.Object,
TelemetryService.Object,
Logger);
}
public Facts(ITestOutputHelper output)
{
AuxiliaryFileClient = new Mock <IAuxiliaryFileClient>();
DatabaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();
DownloadDataClient = new Mock <IDownloadDataClient>();
DownloadSetComparer = new Mock <IDownloadSetComparer>();
DownloadTransferrer = new Mock <IDownloadTransferrer>();
PopularityTransferDataClient = new Mock <IPopularityTransferDataClient>();
SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>();
IndexActionBuilder = new Mock <ISearchIndexActionBuilder>();
BatchPusher = new Mock <IBatchPusher>();
SystemTime = new Mock <ISystemTime>();
FeatureFlags = new Mock <IFeatureFlagService>();
Options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();
TelemetryService = new Mock <IAzureSearchTelemetryService>();
Logger = output.GetLogger <Auxiliary2AzureSearchCommand>();
Config = new Auxiliary2AzureSearchConfiguration
{
AzureSearchBatchSize = 10,
MaxConcurrentBatches = 1,
MaxConcurrentVersionListWriters = 1,
EnablePopularityTransfers = true,
MinPushPeriod = TimeSpan.FromSeconds(5),
};
Options.Setup(x => x.Value).Returns(() => Config);
OldDownloadData = new DownloadData();
OldDownloadResult = Data.GetAuxiliaryFileResult(OldDownloadData, "download-data-etag");
DownloadDataClient
.Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()))
.ReturnsAsync(() => OldDownloadResult);
NewDownloadData = new DownloadData();
AuxiliaryFileClient.Setup(x => x.LoadDownloadDataAsync()).ReturnsAsync(() => NewDownloadData);
Changes = new SortedDictionary <string, long>();
DownloadSetComparer
.Setup(x => x.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
.Returns(() => Changes);
OldTransfers = new PopularityTransferData();
OldTransferResult = new AuxiliaryFileResult <PopularityTransferData>(
modified: true,
data: OldTransfers,
metadata: new AuxiliaryFileMetadata(
DateTimeOffset.UtcNow,
TimeSpan.Zero,
fileSize: 0,
etag: "etag"));
PopularityTransferDataClient
.Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()))
.ReturnsAsync(OldTransferResult);
NewTransfers = new PopularityTransferData();
DatabaseFetcher
.Setup(x => x.GetPopularityTransfersAsync())
.ReturnsAsync(NewTransfers);
TransferChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);
DownloadTransferrer
.Setup(x => x.UpdateDownloadTransfers(
It.IsAny <DownloadData>(),
It.IsAny <SortedDictionary <string, long> >(),
It.IsAny <PopularityTransferData>(),
It.IsAny <PopularityTransferData>()))
.Returns(TransferChanges);
IndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> > {
IndexAction.Merge(new KeyedDocument())
},
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
Mock.Of <IAccessCondition>()));
ProcessedIds = new ConcurrentBag <string>();
IndexActionBuilder
.Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()))
.ReturnsAsync(() => IndexActions)
.Callback <string, Func <SearchFilters, KeyedDocument> >((id, b) =>
{
ProcessedIds.Add(id);
b(SearchFilters.IncludePrereleaseAndSemVer2);
});
// When pushing, delay for a little bit of time so the stopwatch has some measurable duration.
PushedIds = new ConcurrentBag <string>();
CurrentBatch = new ConcurrentBag <IndexActions>();
FinishedBatches = new ConcurrentBag <List <IndexActions> >();
BatchPusher
.Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()))
.Callback <string, IndexActions>((id, indexActions) =>
{
CurrentBatch.Add(indexActions);
PushedIds.Add(id);
});
BatchPusher
.Setup(x => x.TryFinishAsync())
.Returns(async() =>
{
await Task.Delay(TimeSpan.FromMilliseconds(1));
return(new BatchPusherResult());
})
.Callback(() =>
{
FinishedBatches.Add(CurrentBatch.ToList());
CurrentBatch = new ConcurrentBag <IndexActions>();
});
FeatureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);
Target = new UpdateDownloadsCommand(
AuxiliaryFileClient.Object,
DatabaseFetcher.Object,
DownloadDataClient.Object,
DownloadSetComparer.Object,
DownloadTransferrer.Object,
PopularityTransferDataClient.Object,
SearchDocumentBuilder.Object,
IndexActionBuilder.Object,
() => BatchPusher.Object,
SystemTime.Object,
FeatureFlags.Object,
Options.Object,
TelemetryService.Object,
Logger);
}
