public async Task Champion_BasicCheckpointing() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = true, AutoFlushInterval = null }); List <IndexDocumentsAction <SimpleDocument> > pending = new List <IndexDocumentsAction <SimpleDocument> >(); indexer.ActionAddedAsync += (IndexDocumentsAction <SimpleDocument> doc, CancellationToken cancellationToken) => { pending.Add(doc); return(Task.CompletedTask); }; indexer.ActionCompletedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, CancellationToken cancellationToken) => { pending.Remove(doc); return(Task.CompletedTask); }; indexer.ActionFailedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, Exception ex, CancellationToken cancellationToken) => { pending.Remove(doc); return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data.Take(500)); await indexer.MergeDocumentsAsync(new[] { new SimpleDocument { Id = "Fake" } }); await indexer.UploadDocumentsAsync(data.Skip(500)); await DelayAsync(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(250)); Assert.AreEqual(1001 - BatchSize, pending.Count); await indexer.FlushAsync(); Assert.AreEqual(0, pending.Count); }
public async Task KeyFieldAccessor_Custom() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(10); bool customAccessorInvoked = false; await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { KeyFieldAccessor = (SimpleDocument doc) => { customAccessorInvoked = true; return(doc.Id); } }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); Assert.IsTrue(customAccessorInvoked); }
public async Task AutoFlush_MultipleBatches() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 3.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlushInterval = null }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(10), EventDelay); await WaitForDocumentCountAsync(resources.GetSearchClient(), 3 *BatchSize, delay : TimeSpan.FromSeconds(5)); // Check that we have the correct number of documents await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Champion_ContinueAddingWhileSending() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); // Wrap in a block so we DisposeAsync before getting the Count below { await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); AssertNoFailures(indexer); // Trickle documents in for (int i = 0; i < data.Length; i++) { await indexer.UploadDocumentsAsync(new[] { data[i] }); await DelayAsync(TimeSpan.FromMilliseconds(5)); } } // Check that we have the correct number of documents await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task AutoFlushInterval_FullBatch() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlushInterval = TimeSpan.FromMilliseconds(500) }); AssertNoFailures(indexer); ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer); Task <object> submitted = client.BatchSubmitted; await indexer.UploadDocumentsAsync(data); await submitted; await DelayAsync(EventDelay, EventDelay); Assert.AreEqual(data.Length - BatchSize, pending.Count); await DelayAsync(TimeSpan.FromSeconds(5), EventDelay); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Dispose_Blocks() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlush = false }); AssertNoFailures(indexer); ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(EventDelay, EventDelay); Assert.AreEqual(data.Length, pending.Count); await((IAsyncDisposable)indexer).DisposeAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Behavior_Retry(int status) { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments(1); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); client.ResponseTransformer = (IndexingResult result) => { client.ResponseTransformer = null; return(new IndexingResult(result.Key, false, status)); }; AssertNoFailures(indexer); int sent = 0; indexer.ActionSentAsync += (a, c) => { sent++; return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); Assert.Less(1, sent); }
public async Task AutoFlush_PartialBatch() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlushInterval = null }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(5), EventDelay); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task BufferedSender() { await using SearchResources resources = SearchResources.CreateWithNoIndexes(this); SearchClient searchClient = null; try { searchClient = await CreateIndexAsync(resources); // Simple { #region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender1 await using SearchIndexingBufferedSender <Product> indexer = searchClient.CreateIndexingBufferedSender <Product>(); await indexer.UploadDocumentsAsync(GenerateCatalog(count : 100000)); #endregion } await WaitForDocumentCountAsync(searchClient, 100000); // Check #region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender2 //@@ await indexer.FlushAsync(); Assert.AreEqual(100000, (int)await searchClient.GetDocumentCountAsync()); #endregion } finally { if (searchClient != null) { await resources.GetIndexClient().DeleteIndexAsync(searchClient.IndexName); } } }
public async Task Champion_FineGrainedErrors() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); // Don't touch the failures outside of the event handler until // we've finished flushing List <IndexingResult> failures = new List <IndexingResult>(); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = false }); indexer.ActionFailedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, Exception ex, CancellationToken cancellationToken) => { failures.Add(result); return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data.Take(500)); await indexer.MergeDocumentsAsync(new[] { new SimpleDocument { Id = "Fake" } }); await indexer.UploadDocumentsAsync(data.Skip(500)); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), 1000); Assert.AreEqual(1, failures.Count); Assert.AreEqual("Fake", failures[0].Key); Assert.AreEqual(404, failures[0].Status); }
public async Task Dispose_ThrowsAfterDispose() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlush = false }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await((IAsyncDisposable)indexer).DisposeAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); Assert.ThrowsAsync <ObjectDisposedException>(async() => await indexer.UploadDocumentsAsync(data)); }
public async Task KeyFieldAccessor_FetchIndex() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); UnbuildableDocument[] data = UnbuildableDocument.GetDocuments(10); await using SearchIndexingBufferedSender <UnbuildableDocument> indexer = client.CreateIndexingBufferedSender <UnbuildableDocument>(); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Convenience_Upload() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Behavior_Split() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); AssertNoFailures(indexer); client.SplitNextBatch = true; await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Dispose_UndisposedNoCrash() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlush = false }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); }
public async Task Champion_OneShotUpload() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(50000); // Wrap in a block so we DisposeAsync before getting the Count below { await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); } // Check that we have the correct number of documents await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Notifications_Added() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); int adds = 0; indexer.ActionAddedAsync += (a, c) => { adds++; return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data); await DelayAsync(EventDelay, EventDelay); Assert.AreEqual(data.Length, adds); }
public async Task KeyFieldAccessor_Error() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); Hotel[] data = SearchResources.TestDocuments; await using SearchIndexingBufferedSender <Hotel> indexer = client.CreateIndexingBufferedSender <Hotel>(); AssertNoFailures(indexer); try { await indexer.UploadDocumentsAsync(data); } catch (InvalidOperationException ex) { StringAssert.Contains(nameof(Hotel), ex.Message); } }
public async Task AutoFlushInterval_TinyInterval() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlushInterval = TimeSpan.FromMilliseconds(10) }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(5), EventDelay); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task AutoFlushInterval_DoesNotFire(int?interval) { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlushInterval = interval != null ? (TimeSpan?)TimeSpan.FromMilliseconds(interval.Value) : null }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(3), EventDelay); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); }
public async Task Champion_FlushAfterInterval() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(20); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlushInterval = TimeSpan.FromMilliseconds(100) }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); await DelayAsync(TimeSpan.FromMilliseconds(100)); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Champion_ManualFlushing() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = false }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }