/// <summary>
/// Create a hotels index with the standard test documents and as many
/// extra empty documents needed to test.
/// </summary>
/// <param name="size">The total number of documents in the index.</param>
/// <returns>SearchResources for testing.</returns>
public async Task <SearchResources> CreateLargeHotelsIndexAsync(int size)
{
// Start with the standard test hotels
SearchResources resources = await SearchResources.CreateWithHotelsIndexAsync(this);
// Create empty hotels with just an ID for the rest
int existingDocumentCount = SearchResources.TestDocuments.Length;
IEnumerable <string> hotelIds =
Enumerable.Range(
existingDocumentCount + 1,
size - existingDocumentCount)
.Select(id => id.ToString());
List <SearchDocument> hotels = hotelIds.Select(id => new SearchDocument {
["hotelId"] = id
}).ToList();
// Upload the empty hotels in batches of 1000 until we're complete
SearchIndexClient client = resources.GetIndexClient();
for (int i = 0; i < hotels.Count; i += 1000)
{
IEnumerable <SearchDocument> nextHotels = hotels.Skip(i).Take(1000);
if (!nextHotels.Any())
{
break;
}
await client.IndexDocumentsAsync(IndexDocumentsBatch.Upload(nextHotels));
await resources.WaitForIndexingAsync();
}
return(resources);
}
private static async Task UploadDocumentsAsync(IEnumerable <Product> documents)
{
var searchClient = CreateSearchIndexClient();
var batch = IndexDocumentsBatch.Upload(documents);
await searchClient.IndexDocumentsAsync(batch);
}
/// <summary>
/// Indexes documents by calling <see cref="SearchClient.IndexDocumentsAsync{T}(IndexDocumentsBatch{T}, IndexDocumentsOptions, CancellationToken)"/>.
/// </summary>
/// <param name="cancellationToken">The token used to signal cancellation request.</param>
public override async Task RunAsync(CancellationToken cancellationToken)
{
await SearchClient.IndexDocumentsAsync(
IndexDocumentsBatch.Upload(_hotels),
new IndexDocumentsOptions()
{
ThrowOnAnyError = true
},
cancellationToken);
}
/// <summary>
/// Indexes documents by calling <see cref="SearchClient.IndexDocuments{T}(IndexDocumentsBatch{T}, IndexDocumentsOptions, CancellationToken)"/>.
/// </summary>
/// <param name="cancellationToken">The token used to signal cancellation request.</param>
public override void Run(CancellationToken cancellationToken)
{
SearchClient.IndexDocuments(
IndexDocumentsBatch.Upload(_hotels),
new IndexDocumentsOptions()
{
ThrowOnAnyError = true
},
cancellationToken);
}
public static async Task UploadDocumentsAsync(SearchClient searchClient, List <Hotel> hotels)
{
var batch = IndexDocumentsBatch.Upload(hotels);
try
{
await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
}
catch (RequestFailedException ex)
{
Console.WriteLine("Failed to index the documents: \n{0}", ex.Message);
}
}
public async Task RecentlyIndexedDynamicDocument()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this);
Hotel document = SearchResources.TestDocuments[0];
await resources.GetIndexClient().IndexDocumentsAsync(
IndexDocumentsBatch.Upload(new[] { document.AsDocument() }));
await resources.WaitForIndexingAsync();
Response <Hotel> response = await resources.GetQueryClient().GetDocumentAsync <Hotel>(document.HotelId);
Assert.AreEqual(document.HotelId, response.Value.HotelId);
}
public async Task VerifyRoundtrip <T>(
Func <T, string> getKey,
T document,
T expected = default,
GetDocumentOptions options = null)
{
await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this);
await resources.GetIndexClient().IndexDocumentsAsync <T>(
IndexDocumentsBatch.Upload <T>(new[] { document }));
await resources.WaitForIndexingAsync();
Response <T> response = await resources.GetQueryClient().GetDocumentAsync <T>(getKey(document), options);
// Only validate expected properties
AssertApproximate(expected ?? document, response.Value);
}
/// <summary>
/// Populates the Azure Search index with `<paramref name="documentCount"/>` number of documents, each of `<paramref name="documentSize"/>` size.
/// </summary>
/// <param name="documentCount">Number of documents to index.</param>
/// <param name="documentSize">Size of each document being indexed.</param>
/// <returns>Task representing the asynchronous work.</returns>
protected async Task PopulateIndexAsync(int documentCount, DocumentSize documentSize)
{
List <Hotel> hotels = DocumentGenerator.GenerateHotels(documentCount, documentSize);
await SearchClient.IndexDocumentsAsync(IndexDocumentsBatch.Upload(hotels), new IndexDocumentsOptions()
{
ThrowOnAnyError = true
});
long uploadedDocumentCount = 0;
while (uploadedDocumentCount != documentCount)
{
uploadedDocumentCount = (await SearchClient.GetDocumentCountAsync()).Value;
Thread.Sleep(1000);
}
}
public async Task Structs()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this);
SimpleStructHotel document = new SimpleStructHotel
{
HotelId = "4",
HotelName = "Value Inn"
};
await resources.GetIndexClient().IndexDocumentsAsync(
IndexDocumentsBatch.Upload(new[] { document }));
await resources.WaitForIndexingAsync();
SearchIndexClient client = resources.GetQueryClient();
Response <SimpleStructHotel> response = await client.GetDocumentAsync <SimpleStructHotel>(document.HotelId);
Assert.AreEqual(document, response.Value);
}
public async Task <int> BuildIndexAsync(IEnumerable <SearchLocationIndex> searchLocations)
{
logger.LogInformation($"Starting to build index for {searchLocations.Count()}");
try
{
var searchIndexClient = new SearchIndexClient(azureSearchIndexConfig.EndpointUri, GetAzureKeyCredential());
var searchClient = new SearchClient(azureSearchIndexConfig.EndpointUri, azureSearchIndexConfig.LocationSearchIndex, GetAzureKeyCredential());
var fieldBuilder = new FieldBuilder();
var searchFields = fieldBuilder.Build(typeof(SearchLocationIndex));
var definition = new SearchIndex(azureSearchIndexConfig.LocationSearchIndex, searchFields);
var suggester = new SearchSuggester(suggestorName, new[] { nameof(SearchLocationIndex.LocationName) });
definition.Suggesters.Add(suggester);
logger.LogInformation("created search objects and creating index");
await searchIndexClient.CreateOrUpdateIndexAsync(definition).ConfigureAwait(false);
logger.LogInformation("Created search index and uploading documents");
var batch = IndexDocumentsBatch.Upload(searchLocations);
IndexDocumentsResult result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
var failedRecords = result.Results.Where(r => !r.Succeeded);
if (failedRecords.Any())
{
var sampleFailedRecord = failedRecords.FirstOrDefault();
var sampleMessage = $"{failedRecords.Count()} have failed to upload to the index, sample failed record message {sampleFailedRecord.ErrorMessage}, Status = {sampleFailedRecord.Status}";
logger.LogError(sampleMessage);
throw new DfcIndexUploadException("sampleMessage");
}
logger.LogInformation($"Created search index and uploaded {result.Results.Count} documents");
return(result.Results.Count);
}
catch (Exception ex)
{
logger.LogError("Building index had an error", ex);
throw;
}
}
public async Task EmptyValuesDynamicDocument()
{
await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this);
SearchDocument document =
new SearchDocument
{
["hotelId"] = "1",
["hotelName"] = null,
["tags"] = new object[0],
["parkingIncluded"] = null,
["lastRenovationDate"] = null,
["rating"] = null,
["location"] = null,
["geoLocation"] = null,
["address"] = null,
["rooms"] = new[]
{
new SearchDocument
{
["baseRate"] = null,
["bedOptions"] = null,
["sleepsCount"] = null,
["smokingAllowed"] = null,
["tags"] = new object[0]
}
}
};
await resources.GetSearchClient().IndexDocumentsAsync(
IndexDocumentsBatch.Upload(new[] { document }));
await resources.WaitForIndexingAsync();
Response <SearchDocument> response = await resources.GetQueryClient().GetDocumentAsync <SearchDocument>((string)document["hotelId"]);
Assert.AreEqual(document["hotelId"], response.Value["hotelId"]);
}
private static async Task <IndexDocumentsResult> ExponentialBackoffAsync(SearchClient searchClient, List <Hotel> hotels, int id)
{
// Create batch of documents for indexing
var batch = IndexDocumentsBatch.Upload(hotels);
// Create an object to hold the result
IndexDocumentsResult result = null;
// Define parameters for exponential backoff
int attempts = 0;
TimeSpan delay = delay = TimeSpan.FromSeconds(2);
int maxRetryAttempts = 5;
// Implement exponential backoff
do
{
try
{
attempts++;
result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
var failedDocuments = result.Results.Where(r => r.Succeeded != true).ToList();
// handle partial failure
if (failedDocuments.Count > 0)
{
if (attempts == maxRetryAttempts)
{
Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
break;
}
else
{
Console.WriteLine("[Batch starting at doc {0} had partial failure]", id);
//Console.WriteLine("[Attempt: {0} of {1} Failed]", attempts, maxRetryAttempts);
Console.WriteLine("[Retrying {0} failed documents] \n", failedDocuments.Count);
// creating a batch of failed documents to retry
var failedDocumentKeys = failedDocuments.Select(doc => doc.Key).ToList();
hotels = hotels.Where(h => failedDocumentKeys.Contains(h.HotelId)).ToList();
batch = IndexDocumentsBatch.Upload(hotels);
Task.Delay(delay).Wait();
delay = delay * 2;
continue;
}
}
return(result);
}
catch (RequestFailedException ex)
{
Console.WriteLine("[Batch starting at doc {0} failed]", id);
//Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
Console.WriteLine("[Retrying entire batch] \n");
if (attempts == maxRetryAttempts)
{
Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
break;
}
Task.Delay(delay).Wait();
delay = delay * 2;
}
} while (true);
return(null);
}
public async Task AddDocumentsToIndex(List <PersonCity> personCities)
{
var batch = IndexDocumentsBatch.Upload(personCities);
await _searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
}