public async Task Flush_Blocks()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlush = false
});
AssertNoFailures(indexer);
ConcurrentDictionary <int, IndexDocumentsAction <SimpleDocument> > pending = TrackPending(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(EventDelay, EventDelay);
Assert.AreEqual(data.Length, pending.Count);
await indexer.FlushAsync();
await DelayAsync(EventDelay, EventDelay);
Assert.Zero(pending.Count);
}
public async Task AutoFlush_PartialBatch()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize / 2);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlushInterval = null
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(5), EventDelay);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task AutoFlush_MultipleBatches()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 3.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>()
{
AutoFlushInterval = null
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await DelayAsync(TimeSpan.FromSeconds(10), EventDelay);
await WaitForDocumentCountAsync(resources.GetSearchClient(), 3 *BatchSize, delay : TimeSpan.FromSeconds(5));
// Check that we have the correct number of documents
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task KeyFieldAccessor_Custom()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(10);
bool customAccessorInvoked = false;
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
KeyFieldAccessor = (SimpleDocument doc) =>
{
customAccessorInvoked = true;
return(doc.Id);
}
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
Assert.IsTrue(customAccessorInvoked);
}
public async Task Behavior_Retry(int status)
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments(1);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
client.ResponseTransformer = (IndexingResult result) =>
{
client.ResponseTransformer = null;
return(new IndexingResult(result.Key, false, status));
};
AssertNoFailures(indexer);
int sent = 0;
indexer.ActionSentAsync += (a, c) => { sent++; return(Task.CompletedTask); };
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
Assert.Less(1, sent);
}
public async Task Champion_BasicCheckpointing()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = true,
AutoFlushInterval = null
});
List <IndexDocumentsAction <SimpleDocument> > pending = new List <IndexDocumentsAction <SimpleDocument> >();
indexer.ActionAddedAsync +=
(IndexDocumentsAction <SimpleDocument> doc, CancellationToken cancellationToken) =>
{
pending.Add(doc);
return(Task.CompletedTask);
};
indexer.ActionCompletedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
CancellationToken cancellationToken) =>
{
pending.Remove(doc);
return(Task.CompletedTask);
};
indexer.ActionFailedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
Exception ex,
CancellationToken cancellationToken) =>
{
pending.Remove(doc);
return(Task.CompletedTask);
};
await indexer.UploadDocumentsAsync(data.Take(500));
await indexer.MergeDocumentsAsync(new[] { new SimpleDocument {
Id = "Fake"
} });
await indexer.UploadDocumentsAsync(data.Skip(500));
await DelayAsync(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(250));
Assert.AreEqual(1001 - BatchSize, pending.Count);
await indexer.FlushAsync();
Assert.AreEqual(0, pending.Count);
}
public async Task KeyFieldAccessor_FetchIndex()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
UnbuildableDocument[] data = UnbuildableDocument.GetDocuments(10);
await using SearchIndexingBufferedSender <UnbuildableDocument> indexer =
client.CreateIndexingBufferedSender <UnbuildableDocument>();
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Convenience_MergeOrUpload()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
AssertNoFailures(indexer);
await indexer.MergeOrUploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Convenience_Merge()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
ConcurrentQueue <IndexDocumentsAction <SimpleDocument> > failures = TrackFailures(indexer);
await indexer.MergeDocumentsAsync(data);
await indexer.FlushAsync();
Assert.AreEqual(data.Length, failures.Count);
}
public async Task Behavior_Split()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
BatchingSearchClient client = GetBatchingSearchClient(resources);
SimpleDocument[] data = SimpleDocument.GetDocuments(BatchSize);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>();
AssertNoFailures(indexer);
client.SplitNextBatch = true;
await indexer.UploadDocumentsAsync(data);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
public async Task Champion_FineGrainedErrors()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
// Don't touch the failures outside of the event handler until
// we've finished flushing
List <IndexingResult> failures = new List <IndexingResult>();
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = false
});
indexer.ActionFailedAsync +=
(IndexDocumentsAction <SimpleDocument> doc,
IndexingResult result,
Exception ex,
CancellationToken cancellationToken) =>
{
failures.Add(result);
return(Task.CompletedTask);
};
await indexer.UploadDocumentsAsync(data.Take(500));
await indexer.MergeDocumentsAsync(new[] { new SimpleDocument {
Id = "Fake"
} });
await indexer.UploadDocumentsAsync(data.Skip(500));
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), 1000);
Assert.AreEqual(1, failures.Count);
Assert.AreEqual("Fake", failures[0].Key);
Assert.AreEqual(404, failures[0].Status);
}
public async Task BufferedSender()
{
await using SearchResources resources = SearchResources.CreateWithNoIndexes(this);
SearchClient searchClient = null;
try
{
searchClient = await CreateIndexAsync(resources);
// Simple
{
searchClient = GetOriginal(searchClient);
#region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender1
await using SearchIndexingBufferedSender <Product> indexer =
new SearchIndexingBufferedSender <Product>(searchClient);
await indexer.UploadDocumentsAsync(GenerateCatalog(count : 100000));
#endregion
}
await WaitForDocumentCountAsync(searchClient, 100000);
// Check
#region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender2
#if SNIPPET
await indexer.FlushAsync();
#endif
Assert.AreEqual(100000, (int)await searchClient.GetDocumentCountAsync());
#endregion
}
finally
{
if (searchClient != null)
{
await resources.GetIndexClient().DeleteIndexAsync(searchClient.IndexName);
}
}
}
public async Task Notifications_Failed()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments((int)(BatchSize * 1.5));
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
int failed = 0;
indexer.ActionFailedAsync += (a, r, e, c) => { failed++; return(Task.CompletedTask); };
await indexer.MergeDocumentsAsync(data);
await indexer.FlushAsync();
await DelayAsync(EventDelay, EventDelay);
Assert.AreEqual(data.Length, failed);
}
public async Task Convenience_None()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(3);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender(
new SearchIndexingBufferedSenderOptions <SimpleDocument>());
AssertNoFailures(indexer);
IndexDocumentsBatch <SimpleDocument> batch = IndexDocumentsBatch.Create(
IndexDocumentsAction.Delete <SimpleDocument>(data[0]),
IndexDocumentsAction.Upload <SimpleDocument>(data[1]),
IndexDocumentsAction.MergeOrUpload <SimpleDocument>(data[2]));
await indexer.IndexDocumentsAsync(batch);
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), 2);
}
public async Task Champion_ManualFlushing()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync <SimpleDocument>(this);
SearchClient client = resources.GetSearchClient();
SimpleDocument[] data = SimpleDocument.GetDocuments(1000);
await using SearchIndexingBufferedSender <SimpleDocument> indexer =
client.CreateIndexingBufferedSender <SimpleDocument>(
new SearchIndexingBufferedSenderOptions <SimpleDocument>
{
AutoFlush = false
});
AssertNoFailures(indexer);
await indexer.UploadDocumentsAsync(data);
Assert.Zero((int)await resources.GetSearchClient().GetDocumentCountAsync());
await indexer.FlushAsync();
await WaitForDocumentCountAsync(resources.GetSearchClient(), data.Length);
}
