/// <summary> /// Create a hotels index with the standard test documents and as many /// extra empty documents needed to test. /// </summary> /// <param name="size">The total number of documents in the index.</param> /// <returns>SearchResources for testing.</returns> public async Task <SearchResources> CreateLargeHotelsIndexAsync(int size) { // Start with the standard test hotels SearchResources resources = await SearchResources.CreateWithHotelsIndexAsync(this); // Create empty hotels with just an ID for the rest int existingDocumentCount = SearchResources.TestDocuments.Length; IEnumerable <string> hotelIds = Enumerable.Range( existingDocumentCount + 1, size - existingDocumentCount) .Select(id => id.ToString()); List <SearchDocument> hotels = hotelIds.Select(id => new SearchDocument { ["hotelId"] = id }).ToList(); // Upload the empty hotels in batches of 1000 until we're complete SearchIndexClient client = resources.GetIndexClient(); for (int i = 0; i < hotels.Count; i += 1000) { IEnumerable <SearchDocument> nextHotels = hotels.Skip(i).Take(1000); if (!nextHotels.Any()) { break; } await client.IndexDocumentsAsync(IndexDocumentsBatch.Upload(nextHotels)); await resources.WaitForIndexingAsync(); } return(resources); }
private static async Task UploadDocumentsAsync(IEnumerable <Product> documents) { var searchClient = CreateSearchIndexClient(); var batch = IndexDocumentsBatch.Upload(documents); await searchClient.IndexDocumentsAsync(batch); }
/// <summary> /// Indexes documents by calling <see cref="SearchClient.IndexDocumentsAsync{T}(IndexDocumentsBatch{T}, IndexDocumentsOptions, CancellationToken)"/>. /// </summary> /// <param name="cancellationToken">The token used to signal cancellation request.</param> public override async Task RunAsync(CancellationToken cancellationToken) { await SearchClient.IndexDocumentsAsync( IndexDocumentsBatch.Upload(_hotels), new IndexDocumentsOptions() { ThrowOnAnyError = true }, cancellationToken); }
/// <summary> /// Indexes documents by calling <see cref="SearchClient.IndexDocuments{T}(IndexDocumentsBatch{T}, IndexDocumentsOptions, CancellationToken)"/>. /// </summary> /// <param name="cancellationToken">The token used to signal cancellation request.</param> public override void Run(CancellationToken cancellationToken) { SearchClient.IndexDocuments( IndexDocumentsBatch.Upload(_hotels), new IndexDocumentsOptions() { ThrowOnAnyError = true }, cancellationToken); }
public static async Task UploadDocumentsAsync(SearchClient searchClient, List <Hotel> hotels) { var batch = IndexDocumentsBatch.Upload(hotels); try { await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false); } catch (RequestFailedException ex) { Console.WriteLine("Failed to index the documents: \n{0}", ex.Message); } }
public async Task RecentlyIndexedDynamicDocument() { await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this); Hotel document = SearchResources.TestDocuments[0]; await resources.GetIndexClient().IndexDocumentsAsync( IndexDocumentsBatch.Upload(new[] { document.AsDocument() })); await resources.WaitForIndexingAsync(); Response <Hotel> response = await resources.GetQueryClient().GetDocumentAsync <Hotel>(document.HotelId); Assert.AreEqual(document.HotelId, response.Value.HotelId); }
public async Task VerifyRoundtrip <T>( Func <T, string> getKey, T document, T expected = default, GetDocumentOptions options = null) { await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this); await resources.GetIndexClient().IndexDocumentsAsync <T>( IndexDocumentsBatch.Upload <T>(new[] { document })); await resources.WaitForIndexingAsync(); Response <T> response = await resources.GetQueryClient().GetDocumentAsync <T>(getKey(document), options); // Only validate expected properties AssertApproximate(expected ?? document, response.Value); }
/// <summary> /// Populates the Azure Search index with `<paramref name="documentCount"/>` number of documents, each of `<paramref name="documentSize"/>` size. /// </summary> /// <param name="documentCount">Number of documents to index.</param> /// <param name="documentSize">Size of each document being indexed.</param> /// <returns>Task representing the asynchronous work.</returns> protected async Task PopulateIndexAsync(int documentCount, DocumentSize documentSize) { List <Hotel> hotels = DocumentGenerator.GenerateHotels(documentCount, documentSize); await SearchClient.IndexDocumentsAsync(IndexDocumentsBatch.Upload(hotels), new IndexDocumentsOptions() { ThrowOnAnyError = true }); long uploadedDocumentCount = 0; while (uploadedDocumentCount != documentCount) { uploadedDocumentCount = (await SearchClient.GetDocumentCountAsync()).Value; Thread.Sleep(1000); } }
public async Task Structs() { await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this); SimpleStructHotel document = new SimpleStructHotel { HotelId = "4", HotelName = "Value Inn" }; await resources.GetIndexClient().IndexDocumentsAsync( IndexDocumentsBatch.Upload(new[] { document })); await resources.WaitForIndexingAsync(); SearchIndexClient client = resources.GetQueryClient(); Response <SimpleStructHotel> response = await client.GetDocumentAsync <SimpleStructHotel>(document.HotelId); Assert.AreEqual(document, response.Value); }
public async Task <int> BuildIndexAsync(IEnumerable <SearchLocationIndex> searchLocations) { logger.LogInformation($"Starting to build index for {searchLocations.Count()}"); try { var searchIndexClient = new SearchIndexClient(azureSearchIndexConfig.EndpointUri, GetAzureKeyCredential()); var searchClient = new SearchClient(azureSearchIndexConfig.EndpointUri, azureSearchIndexConfig.LocationSearchIndex, GetAzureKeyCredential()); var fieldBuilder = new FieldBuilder(); var searchFields = fieldBuilder.Build(typeof(SearchLocationIndex)); var definition = new SearchIndex(azureSearchIndexConfig.LocationSearchIndex, searchFields); var suggester = new SearchSuggester(suggestorName, new[] { nameof(SearchLocationIndex.LocationName) }); definition.Suggesters.Add(suggester); logger.LogInformation("created search objects and creating index"); await searchIndexClient.CreateOrUpdateIndexAsync(definition).ConfigureAwait(false); logger.LogInformation("Created search index and uploading documents"); var batch = IndexDocumentsBatch.Upload(searchLocations); IndexDocumentsResult result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false); var failedRecords = result.Results.Where(r => !r.Succeeded); if (failedRecords.Any()) { var sampleFailedRecord = failedRecords.FirstOrDefault(); var sampleMessage = $"{failedRecords.Count()} have failed to upload to the index, sample failed record message {sampleFailedRecord.ErrorMessage}, Status = {sampleFailedRecord.Status}"; logger.LogError(sampleMessage); throw new DfcIndexUploadException("sampleMessage"); } logger.LogInformation($"Created search index and uploaded {result.Results.Count} documents"); return(result.Results.Count); } catch (Exception ex) { logger.LogError("Building index had an error", ex); throw; } }
public async Task EmptyValuesDynamicDocument() { await using SearchResources resources = await SearchResources.CreateWithEmptyHotelsIndexAsync(this); SearchDocument document = new SearchDocument { ["hotelId"] = "1", ["hotelName"] = null, ["tags"] = new object[0], ["parkingIncluded"] = null, ["lastRenovationDate"] = null, ["rating"] = null, ["location"] = null, ["geoLocation"] = null, ["address"] = null, ["rooms"] = new[] { new SearchDocument { ["baseRate"] = null, ["bedOptions"] = null, ["sleepsCount"] = null, ["smokingAllowed"] = null, ["tags"] = new object[0] } } }; await resources.GetSearchClient().IndexDocumentsAsync( IndexDocumentsBatch.Upload(new[] { document })); await resources.WaitForIndexingAsync(); Response <SearchDocument> response = await resources.GetQueryClient().GetDocumentAsync <SearchDocument>((string)document["hotelId"]); Assert.AreEqual(document["hotelId"], response.Value["hotelId"]); }
private static async Task <IndexDocumentsResult> ExponentialBackoffAsync(SearchClient searchClient, List <Hotel> hotels, int id) { // Create batch of documents for indexing var batch = IndexDocumentsBatch.Upload(hotels); // Create an object to hold the result IndexDocumentsResult result = null; // Define parameters for exponential backoff int attempts = 0; TimeSpan delay = delay = TimeSpan.FromSeconds(2); int maxRetryAttempts = 5; // Implement exponential backoff do { try { attempts++; result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false); var failedDocuments = result.Results.Where(r => r.Succeeded != true).ToList(); // handle partial failure if (failedDocuments.Count > 0) { if (attempts == maxRetryAttempts) { Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id); break; } else { Console.WriteLine("[Batch starting at doc {0} had partial failure]", id); //Console.WriteLine("[Attempt: {0} of {1} Failed]", attempts, maxRetryAttempts); Console.WriteLine("[Retrying {0} failed documents] \n", failedDocuments.Count); // creating a batch of failed documents to retry var failedDocumentKeys = failedDocuments.Select(doc => doc.Key).ToList(); hotels = hotels.Where(h => failedDocumentKeys.Contains(h.HotelId)).ToList(); batch = IndexDocumentsBatch.Upload(hotels); Task.Delay(delay).Wait(); delay = delay * 2; continue; } } return(result); } catch (RequestFailedException ex) { Console.WriteLine("[Batch starting at doc {0} failed]", id); //Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message); Console.WriteLine("[Retrying entire batch] \n"); if (attempts == maxRetryAttempts) { Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id); break; } Task.Delay(delay).Wait(); delay = delay * 2; } } while (true); return(null); }
public async Task AddDocumentsToIndex(List <PersonCity> personCities) { var batch = IndexDocumentsBatch.Upload(personCities); await _searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false); }