public async Task AutoFlush_MultipleBatches()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 3.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlushInterval = null
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(10), EventDelay);
await WaitForDocumentCountAsync(resources.GetSearchClient(), 3 *BatchSize, delay : TimeSpan.FromSeconds(5));
// Check that we have the correct number of documents
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task AutoFlush_PartialBatch()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlushInterval = null
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(5), EventDelay);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Champion_ContinueAddingWhileSending()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
// Wrap in a block so we DisposeAsync before getting the Count below
{
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
AssertNoFailures(indexer);
// Trickle documents in
for (int i = 0; i < data.Length; i++)
{
await indexer.UploadDocumentsAsync(new[] { data[i] });
await DelayAsync(TimeSpan.FromMilliseconds(5));
}
}
// Check that we have the correct number of documents
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task AutoFlushInterval_FullBatch()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlushInterval = TimeSpan.FromMilliseconds(500)
});
AssertNoFailures(indexer);
ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer);
Task <object> submitted = client.BatchSubmitted;
await indexer.UploadDocumentsAsync(data);
await submitted;
await DelayAsync(EventDelay, EventDelay);
Assert.AreEqual(data.Length - BatchSize, pending.Count);
await DelayAsync(TimeSpan.FromSeconds(5), EventDelay);
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Dispose_Blocks()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlush = false
});
AssertNoFailures(indexer);
ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(EventDelay, EventDelay);
Assert.AreEqual(data.Length, pending.Count);
await((IAsyncDisposable)indexer).DisposeAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Behavior_Retry(int status)
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments(1);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
client.ResponseTransformer = (IndexingResult result) =>
{
client.ResponseTransformer = null;
return(new IndexingResult(result.Key, false, status));
};
AssertNoFailures(indexer);
int sent = 0;
indexer.ActionSentAsync += (a, c) => { sent++; return(Task.CompletedTask); };
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
Assert.Less(1, sent);
}
public async Task KeyFieldAccessor_Custom()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(10);
bool customAccessorInvoked = false;
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
KeyFieldAccessor = (SimpleDocument doc) =>
{
customAccessorInvoked = true;
return(doc.Id);
}
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
Assert.IsTrue(customAccessorInvoked);
}
public async Task Champion_BasicCheckpointing()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = true,
AutoFlushInterval = null
});
List <IndexDocumentsAction <SimpleDocument> > pending = new List <IndexDocumentsAction <SimpleDocument> >();
indexer.ActionAddedAsync +=
(IndexDocumentsAction <SimpleDocument> doc, CancellationToken cancellationToken) =>
{
pending.Add(doc);
return(Task.CompletedTask);
};
indexer.ActionCompletedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
CancellationToken cancellationToken) =>
{
pending.Remove(doc);
return(Task.CompletedTask);
};
indexer.ActionFailedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
Exception ex,
CancellationToken cancellationToken) =>
{
pending.Remove(doc);
return(Task.CompletedTask);
};
await indexer.UploadDocumentsAsync(data.Take(500));
await indexer.MergeDocumentsAsync(new[] { new SimpleDocument {
Id = "Fake"
} });
await indexer.UploadDocumentsAsync(data.Skip(500));
await DelayAsync(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(250));
Assert.AreEqual(1001 - BatchSize, pending.Count);
await indexer.FlushAsync();
Assert.AreEqual(0, pending.Count);
}
public async Task Behavior_Split()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
AssertNoFailures(indexer);
client.SplitNextBatch = true;
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Convenience_MergeOrUpload()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
AssertNoFailures(indexer);
await indexer.MergeOrUploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Convenience_Merge()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
ConcurrentQueue <IndexDocumentsAction <SimpleDocument> > failures = TrackFailures(indexer);
await indexer.MergeDocumentsAsync(data);
await indexer.FlushAsync();
Assert.AreEqual(data.Length, failures.Count);
}
public async Task Dispose_UndisposedNoCrash()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlush = false
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
}
public async Task Champion_FineGrainedErrors()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
// Don't touch the failures outside of the event handler until
// we've finished flushing
List <IndexingResult> failures = new List <IndexingResult>();
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = false
});
indexer.ActionFailedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
Exception ex,
CancellationToken cancellationToken) =>
{
failures.Add(result);
return(Task.CompletedTask);
};
await indexer.UploadDocumentsAsync(data.Take(500));
await indexer.MergeDocumentsAsync(new[] { new SimpleDocument {
Id = "Fake"
} });
await indexer.UploadDocumentsAsync(data.Skip(500));
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), 1000);
Assert.AreEqual(1, failures.Count);
Assert.AreEqual("Fake", failures[0].Key);
Assert.AreEqual(404, failures[0].Status);
}
public async Task Champion_OneShotUpload()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(50000);
// Wrap in a block so we DisposeAsync before getting the Count below
{
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
}
// Check that we have the correct number of documents
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Dispose_Flushes()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlush = false
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await((IAsyncDisposable)indexer).DisposeAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Notifications_Added()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
int adds = 0;
indexer.ActionAddedAsync += (a, c) => { adds++; return(Task.CompletedTask); };
await indexer.UploadDocumentsAsync(data);
await DelayAsync(EventDelay, EventDelay);
Assert.AreEqual(data.Length, adds);
}
public async Task AutoFlushInterval_TinyInterval()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlushInterval = TimeSpan.FromMilliseconds(10)
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(5), EventDelay);
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Convenience_None()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(3);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
AssertNoFailures(indexer);
IndexDocumentsBatch <SimpleDocument> batch = IndexDocumentsBatch.Create(
IndexDocumentsAction.Delete <SimpleDocument>(data[0]),
IndexDocumentsAction.Upload <SimpleDocument>(data[1]),
IndexDocumentsAction.MergeOrUpload <SimpleDocument>(data[2]));
await indexer.IndexDocumentsAsync(batch);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), 2);
}
public async Task AutoFlushInterval_DoesNotFire(int?interval)
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlushInterval = interval != null ?
(TimeSpan?)TimeSpan.FromMilliseconds(interval.Value) :
null
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(3), EventDelay);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
}
public async Task Champion_FlushAfterInterval()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(20);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlushInterval = TimeSpan.FromMilliseconds(100)
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
await DelayAsync(TimeSpan.FromMilliseconds(100));
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Champion_ManualFlushing()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = false
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}