public async Task Flush_Blocks() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlush = false }); AssertNoFailures(indexer); ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(EventDelay, EventDelay); Assert.AreEqual(data.Length, pending.Count); await indexer.FlushAsync(); await DelayAsync(EventDelay, EventDelay); Assert.Zero(pending.Count); }
public async Task AutoFlush_PartialBatch() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlushInterval = null }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(5), EventDelay); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task AutoFlush_MultipleBatches() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 3.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>() { AutoFlushInterval = null }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await DelayAsync(TimeSpan.FromSeconds(10), EventDelay); await WaitForDocumentCountAsync(resources.GetSearchClient(), 3 *BatchSize, delay : TimeSpan.FromSeconds(5)); // Check that we have the correct number of documents await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task KeyFieldAccessor_Custom() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(10); bool customAccessorInvoked = false; await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { KeyFieldAccessor = (SimpleDocument doc) => { customAccessorInvoked = true; return(doc.Id); } }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); Assert.IsTrue(customAccessorInvoked); }
public async Task Behavior_Retry(int status) { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments(1); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); client.ResponseTransformer = (IndexingResult result) => { client.ResponseTransformer = null; return(new IndexingResult(result.Key, false, status)); }; AssertNoFailures(indexer); int sent = 0; indexer.ActionSentAsync += (a, c) => { sent++; return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); Assert.Less(1, sent); }
public async Task Champion_BasicCheckpointing() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = true, AutoFlushInterval = null }); List <IndexDocumentsAction <SimpleDocument> > pending = new List <IndexDocumentsAction <SimpleDocument> >(); indexer.ActionAddedAsync += (IndexDocumentsAction <SimpleDocument> doc, CancellationToken cancellationToken) => { pending.Add(doc); return(Task.CompletedTask); }; indexer.ActionCompletedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, CancellationToken cancellationToken) => { pending.Remove(doc); return(Task.CompletedTask); }; indexer.ActionFailedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, Exception ex, CancellationToken cancellationToken) => { pending.Remove(doc); return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data.Take(500)); await indexer.MergeDocumentsAsync(new[] { new SimpleDocument { Id = "Fake" } }); await indexer.UploadDocumentsAsync(data.Skip(500)); await DelayAsync(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(250)); Assert.AreEqual(1001 - BatchSize, pending.Count); await indexer.FlushAsync(); Assert.AreEqual(0, pending.Count); }
public async Task KeyFieldAccessor_FetchIndex() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); UnbuildableDocument[] data = UnbuildableDocument.GetDocuments(10); await using SearchIndexingBufferedSender <UnbuildableDocument> indexer = client.CreateIndexingBufferedSender <UnbuildableDocument>(); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Convenience_MergeOrUpload() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); AssertNoFailures(indexer); await indexer.MergeOrUploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Convenience_Merge() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); ConcurrentQueue <IndexDocumentsAction <SimpleDocument> > failures = TrackFailures(indexer); await indexer.MergeDocumentsAsync(data); await indexer.FlushAsync(); Assert.AreEqual(data.Length, failures.Count); }
public async Task Behavior_Split() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); BatchingSearchClient client = GetBatchingSearchClient(resources); SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>(); AssertNoFailures(indexer); client.SplitNextBatch = true; await indexer.UploadDocumentsAsync(data); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }
public async Task Champion_FineGrainedErrors() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); // Don't touch the failures outside of the event handler until // we've finished flushing List <IndexingResult> failures = new List <IndexingResult>(); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = false }); indexer.ActionFailedAsync += (IndexDocumentsAction <SimpleDocument> doc, IndexingResult result, Exception ex, CancellationToken cancellationToken) => { failures.Add(result); return(Task.CompletedTask); }; await indexer.UploadDocumentsAsync(data.Take(500)); await indexer.MergeDocumentsAsync(new[] { new SimpleDocument { Id = "Fake" } }); await indexer.UploadDocumentsAsync(data.Skip(500)); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), 1000); Assert.AreEqual(1, failures.Count); Assert.AreEqual("Fake", failures[0].Key); Assert.AreEqual(404, failures[0].Status); }
public async Task BufferedSender() { await using SearchResources resources = SearchResources.CreateWithNoIndexes(this); SearchClient searchClient = null; try { searchClient = await CreateIndexAsync(resources); // Simple { searchClient = GetOriginal(searchClient); #region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender1 await using SearchIndexingBufferedSender <Product> indexer = new SearchIndexingBufferedSender <Product>(searchClient); await indexer.UploadDocumentsAsync(GenerateCatalog(count : 100000)); #endregion } await WaitForDocumentCountAsync(searchClient, 100000); // Check #region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender2 #if SNIPPET await indexer.FlushAsync(); #endif Assert.AreEqual(100000, (int)await searchClient.GetDocumentCountAsync()); #endregion } finally { if (searchClient != null) { await resources.GetIndexClient().DeleteIndexAsync(searchClient.IndexName); } } }
public async Task Notifications_Failed() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5)); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); int failed = 0; indexer.ActionFailedAsync += (a, r, e, c) => { failed++; return(Task.CompletedTask); }; await indexer.MergeDocumentsAsync(data); await indexer.FlushAsync(); await DelayAsync(EventDelay, EventDelay); Assert.AreEqual(data.Length, failed); }
public async Task Convenience_None() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(3); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions <SimpleDocument>()); AssertNoFailures(indexer); IndexDocumentsBatch <SimpleDocument> batch = IndexDocumentsBatch.Create( IndexDocumentsAction.Delete <SimpleDocument>(data[0]), IndexDocumentsAction.Upload <SimpleDocument>(data[1]), IndexDocumentsAction.MergeOrUpload <SimpleDocument>(data[2])); await indexer.IndexDocumentsAsync(batch); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), 2); }
public async Task Champion_ManualFlushing() { await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this); SearchClient client = resources.GetSearchClient(); SimpleDocument[] data = SimpleDocument.GetDocuments(1000); await using SearchIndexingBufferedSender <SimpleDocument> indexer = client.CreateIndexingBufferedSender <SimpleDocument>( new SearchIndexingBufferedSenderOptions <SimpleDocument> { AutoFlush = false }); AssertNoFailures(indexer); await indexer.UploadDocumentsAsync(data); Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync()); await indexer.FlushAsync(); await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length); }