/// <summary>
/// Create a hotels index with the standard test documents and as many
/// extra empty documents needed to test.
/// </summary>
/// <param name="size">The total number of documents in the index.</param>
/// <returns>SearchResources for testing.</returns>
public async Task <SearchResources> CreateLargeHotelsIndexAsync(int size)
{
// Start with the standard test hotels
SearchResources resources = await SearchResources.CreateWithHotelsIndexAsync(this);
// Create empty hotels with just an ID for the rest
int existingDocumentCount = SearchResources.TestDocuments.Length;
IEnumerable <string> hotelIds =
Enumerable.Range(
existingDocumentCount + 1,
size - existingDocumentCount)
.Select(id => id.ToString());
List <SearchDocument> hotels = hotelIds.Select(id => new SearchDocument {
["hotelId"] = id
}).ToList();
// Upload the empty hotels in batches of 1000 until we're complete
SearchClient client = resources.GetSearchClient();
for (int i = 0; i < hotels.Count; i += 1000)
{
IEnumerable <SearchDocument> nextHotels = hotels.Skip(i).Take(1000);
if (!nextHotels.Any())
{
break;
}
await client.IndexDocumentsAsync(IndexDocumentsBatch.Upload(nextHotels));
await resources.WaitForIndexingAsync();
}
return(resources);
}
/// <summary>
/// Indexes documents by calling <see cref="SearchClient.IndexDocumentsAsync{T}(IndexDocumentsBatch{T}, IndexDocumentsOptions, CancellationToken)"/>.
/// </summary>
/// <param name="cancellationToken">The token used to signal cancellation request.</param>
public override async Task RunAsync(CancellationToken cancellationToken)
{
await SearchClient.IndexDocumentsAsync(
IndexDocumentsBatch.Upload(_hotels),
new IndexDocumentsOptions()
{
ThrowOnAnyError = true
},
cancellationToken);
}
static async Task Main(string[] args)
{
var reviewText = "The quality of the pictures are good, but the body is not durable";
var productId = "1"; // ID of the product reviewed
var reviewId = "1"; // ID of the review used as the search document ID
var indexName = "sample-index";
// Cognitive Search credentials
var searchKey = Environment.GetEnvironmentVariable("COGNITIVE_SEARCH_KEY");
var searchEndpoint = Environment.GetEnvironmentVariable("COGNITIVE_SEARCH_ENDPOINT");
Uri searchUri = new Uri(searchEndpoint);
AzureKeyCredential searchCredential = new AzureKeyCredential(searchKey);
// Initialize Search Index client
var searchIndexClient = new SearchIndexClient(searchUri, searchCredential);
// Create Index
await CreateIndex(indexName, searchIndexClient);
// Initialize Search client
var searchClient = new SearchClient(searchUri, indexName, searchCredential);
// TA credentials
var textAnalyticsKey = Environment.GetEnvironmentVariable("TEXT_ANALYTICS_KEY");
var textAnalyticsEndpoint = Environment.GetEnvironmentVariable("TEXT_ANALYTICS_ENDPOINT");
var textAnalyticsCredentials = new AzureKeyCredential(textAnalyticsKey);
var textAnalyticsUri = new Uri(textAnalyticsEndpoint);
// Initialize TA client
var textAnalyticsClient = new TextAnalyticsClient(textAnalyticsUri, textAnalyticsCredentials);
// Enable opinion mining
var options = new AnalyzeSentimentOptions()
{
IncludeOpinionMining = true
};
// Call TA analyze sentiment api
var sentimentResponse = await textAnalyticsClient.AnalyzeSentimentAsync(reviewText, language : "en", options : options);
// Map to review search document
Review review = CreateReviewDocument(productId, reviewId, sentimentResponse);
// Upload document
var batch = IndexDocumentsBatch.Create(IndexDocumentsAction.Upload(review));
try
{
IndexDocumentsResult result = await searchClient.IndexDocumentsAsync(batch);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
private static async Task AddDataAsync(SearchClient searchClient)
{
var translator = new Translator();
string modelName1 = "external styles are defined withing the element, inside the section of an HTML page";
string polishModelName1 = await translator.GetTranslatedTextAsync(modelName1);
string[] keyWords1 = { "CSS", "styles", "metadata", "element", "EF01" };
string modelName2 = "To include an external JavaScript file, use the script tag with the attribute src";
string polishModelName2 = await translator.GetTranslatedTextAsync(modelName2);
string[] keyWords2 = { "javascript", "EF02", "script", "tag", "src" };
string modelName3 = "Move to the home position";
string polishModelName3 = await translator.GetTranslatedTextAsync(modelName3);
string[] keyWords3 = { "storage", "container", "EF03", "home" };
string modelName4 = "removal of stuck magnets restores machine cycle";
string polishModelName4 = await translator.GetTranslatedTextAsync(modelName4);
string[] keyWords4 = { "magnets", "machine", "cycle", "EF04" };
var batch = IndexDocumentsBatch.Create(
IndexDocumentsAction.Upload(new SearchModel
{
Id = Guid.NewGuid().ToString(), Name = modelName1, PolishName = polishModelName1,
KeyPhrases = keyWords1, Updated = new DateTime(2020, 10, 1, 7, 0, 0)
})
, IndexDocumentsAction.Upload(new SearchModel
{
Id = Guid.NewGuid().ToString(),
Name = modelName2,
KeyPhrases = keyWords2,
PolishName = polishModelName2,
Updated = new DateTime(2020, 9, 2, 8, 54, 0)
}), IndexDocumentsAction.Upload(new SearchModel
{
Id = Guid.NewGuid().ToString(),
Name = modelName4,
KeyPhrases = keyWords4,
PolishName = polishModelName4,
Updated = new DateTime(2020, 8, 2, 12, 11, 0)
}), IndexDocumentsAction.Upload(new SearchModel
{
Id = Guid.NewGuid().ToString(),
Name = modelName3,
KeyPhrases = keyWords3,
PolishName = polishModelName3,
Updated = new DateTime(2020, 7, 2, 21, 21, 0)
}));
await searchClient.IndexDocumentsAsync(batch, new IndexDocumentsOptions { ThrowOnAnyError = true });
}
public static async Task UploadDocumentsAsync(SearchClient searchClient, List <Hotel> hotels)
{
var batch = IndexDocumentsBatch.Upload(hotels);
try
{
await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
}
catch (RequestFailedException ex)
{
Console.WriteLine("Failed to index the documents: \n{0}", ex.Message);
}
}
public async Task UploadDocument(AlbumInfoSearchObject document)
{
IndexDocumentsBatch <AlbumInfoSearchObject> batch = IndexDocumentsBatch.Create(IndexDocumentsAction.MergeOrUpload(document));
try
{
IndexDocumentsResult result = await searchClient.IndexDocumentsAsync(batch);
}
catch (Exception ex)
{
throw;
}
}
public async Task ExecuteAsync(IndexCommand[] commands,
CancellationToken ct = default)
{
var batch = IndexDocumentsBatch.Create <SearchDocument>();
commands.Foreach(x => CommandFactory.CreateCommands(x, batch.Actions));
if (batch.Actions.Count == 0)
{
return;
}
await searchClient.IndexDocumentsAsync(batch, cancellationToken : ct);
}
public async Task <int> BuildIndexAsync(IEnumerable <SearchLocationIndex> searchLocations)
{
logger.LogInformation($"Starting to build index for {searchLocations.Count()}");
try
{
var searchIndexClient = new SearchIndexClient(azureSearchIndexConfig.EndpointUri, GetAzureKeyCredential());
var searchClient = new SearchClient(azureSearchIndexConfig.EndpointUri, azureSearchIndexConfig.LocationSearchIndex, GetAzureKeyCredential());
var fieldBuilder = new FieldBuilder();
var searchFields = fieldBuilder.Build(typeof(SearchLocationIndex));
var definition = new SearchIndex(azureSearchIndexConfig.LocationSearchIndex, searchFields);
var suggester = new SearchSuggester(suggestorName, new[] { nameof(SearchLocationIndex.LocationName) });
definition.Suggesters.Add(suggester);
logger.LogInformation("created search objects and creating index");
await searchIndexClient.CreateOrUpdateIndexAsync(definition).ConfigureAwait(false);
logger.LogInformation("Created search index and uploading documents");
var batch = IndexDocumentsBatch.Upload(searchLocations);
IndexDocumentsResult result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
var failedRecords = result.Results.Where(r => !r.Succeeded);
if (failedRecords.Any())
{
var sampleFailedRecord = failedRecords.FirstOrDefault();
var sampleMessage = $"{failedRecords.Count()} have failed to upload to the index, sample failed record message {sampleFailedRecord.ErrorMessage}, Status = {sampleFailedRecord.Status}";
logger.LogError(sampleMessage);
throw new DfcIndexUploadException("sampleMessage");
}
logger.LogInformation($"Created search index and uploaded {result.Results.Count} documents");
return(result.Results.Count);
}
catch (Exception ex)
{
logger.LogError("Building index had an error", ex);
throw;
}
}
private async Task <Response <IndexDocumentsResult> > IndexProduct(SearchClient searchClient, ProductDocument product)
{
var batch = IndexDocumentsBatch.Create(IndexDocumentsAction.Upload(product));
return(await searchClient.IndexDocumentsAsync(batch));
}
private static async Task <IndexDocumentsResult> ExponentialBackoffAsync(SearchClient searchClient, List <Hotel> hotels, int id)
{
// Create batch of documents for indexing
var batch = IndexDocumentsBatch.Upload(hotels);
// Create an object to hold the result
IndexDocumentsResult result = null;
// Define parameters for exponential backoff
int attempts = 0;
TimeSpan delay = delay = TimeSpan.FromSeconds(2);
int maxRetryAttempts = 5;
// Implement exponential backoff
do
{
try
{
attempts++;
result = await searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
var failedDocuments = result.Results.Where(r => r.Succeeded != true).ToList();
// handle partial failure
if (failedDocuments.Count > 0)
{
if (attempts == maxRetryAttempts)
{
Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
break;
}
else
{
Console.WriteLine("[Batch starting at doc {0} had partial failure]", id);
//Console.WriteLine("[Attempt: {0} of {1} Failed]", attempts, maxRetryAttempts);
Console.WriteLine("[Retrying {0} failed documents] \n", failedDocuments.Count);
// creating a batch of failed documents to retry
var failedDocumentKeys = failedDocuments.Select(doc => doc.Key).ToList();
hotels = hotels.Where(h => failedDocumentKeys.Contains(h.HotelId)).ToList();
batch = IndexDocumentsBatch.Upload(hotels);
Task.Delay(delay).Wait();
delay = delay * 2;
continue;
}
}
return(result);
}
catch (RequestFailedException ex)
{
Console.WriteLine("[Batch starting at doc {0} failed]", id);
//Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
Console.WriteLine("[Retrying entire batch] \n");
if (attempts == maxRetryAttempts)
{
Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
break;
}
Task.Delay(delay).Wait();
delay = delay * 2;
}
} while (true);
return(null);
}
public async Task AddDocumentsToIndex(List <PersonCity> personCities)
{
var batch = IndexDocumentsBatch.Upload(personCities);
await _searchClient.IndexDocumentsAsync(batch).ConfigureAwait(false);
}
public Task IndexAccommodationAsync(Accommodation accommodation)
{
var action = IndexDocumentsAction.MergeOrUpload(accommodation);
return(client.IndexDocumentsAsync(IndexDocumentsBatch.Create(new[] { action })));
}
// Upload documents in a single Upload request.
public async Task UploadDocumentsAsync()
{
IndexDocumentsBatch <Hotel> batch = IndexDocumentsBatch.Create(
IndexDocumentsAction.Upload(
new Hotel()
{
HotelId = "1",
HotelName = "Secret Point Motel",
Description = "The hotel is ideally located on the main commercial artery of the city in the heart of New York. A few minutes away is Time's Square and the historic centre of the city, as well as other places of interest that make New York one of America's most attractive and cosmopolitan cities.",
DescriptionFr = "L'hôtel est idéalement situé sur la principale artère commerciale de la ville en plein cœur de New York. A quelques minutes se trouve la place du temps et le centre historique de la ville, ainsi que d'autres lieux d'intérêt qui font de New York l'une des villes les plus attractives et cosmopolites de l'Amérique.",
Category = "Boutique",
Tags = new[] { "pool", "air conditioning", "concierge" },
ParkingIncluded = false,
LastRenovationDate = new DateTimeOffset(1970, 1, 18, 0, 0, 0, TimeSpan.Zero),
Rating = 3.6,
Location = GeographyPoint.Create(22, 22),
Rooms = new Room[]
{
new Room()
{
Description = "Room Des", BaseRate = 100, DescriptionFr = "Le Room", BedOptions = "Any", Type = "Single", SmokingAllowed = false, SleepsCount = 3, Tags = new string[] { "single", "front", "near pool" }
}
},
cheapest = 100,
expensive = 300,
Address = new Address()
{
StreetAddress = "677 5th Ave",
City = "New York",
StateProvince = "NY",
PostalCode = "10022",
Country = "USA"
}
}),
IndexDocumentsAction.Upload(
new Hotel()
{
HotelId = "2",
HotelName = "Twin Dome Motel",
Description = "The hotel is situated in a nineteenth century plaza, which has been expanded and renovated to the highest architectural standards to create a modern, functional and first-class hotel in which art and unique historical elements coexist with the most modern comforts.",
DescriptionFr = "L'hôtel est situé dans une place du XIXe siècle, qui a été agrandie et rénovée aux plus hautes normes architecturales pour créer un hôtel moderne, fonctionnel et de première classe dans lequel l'art et les éléments historiques uniques coexistent avec le confort le plus moderne.",
Category = "Boutique",
Tags = new[] { "pool", "free wifi", "concierge" },
ParkingIncluded = false,
LastRenovationDate = new DateTimeOffset(1979, 2, 18, 0, 0, 0, TimeSpan.Zero),
Rating = 3.60,
Address = new Address()
{
StreetAddress = "140 University Town Center Dr",
City = "Sarasota",
StateProvince = "FL",
PostalCode = "34243",
Country = "USA"
},
Location = GeographyPoint.Create(22, 22),
Rooms = new Room[]
{
new Room()
{
Description = "Room Des", BaseRate = 100, DescriptionFr = "Le Room", BedOptions = "Any", Type = "Single", SmokingAllowed = false, SleepsCount = 3, Tags = new string[] { "single", "front", "near pool" }
}
},
cheapest = 100,
expensive = 300,
}),
IndexDocumentsAction.Upload(
new Hotel()
{
HotelId = "3",
HotelName = "Triple Landscape Hotel",
Description = "The Hotel stands out for its gastronomic excellence under the management of William Dough, who advises on and oversees all of the Hotel’s restaurant services.",
DescriptionFr = "L'hôtel est situé dans une place du XIXe siècle, qui a été agrandie et rénovée aux plus hautes normes architecturales pour créer un hôtel moderne, fonctionnel et de première classe dans lequel l'art et les éléments historiques uniques coexistent avec le confort le plus moderne.",
Category = "Resort and Spa",
Tags = new[] { "air conditioning", "bar", "continental breakfast" },
ParkingIncluded = true,
LastRenovationDate = new DateTimeOffset(2015, 9, 20, 0, 0, 0, TimeSpan.Zero),
Rating = 4.80,
Address = new Address()
{
StreetAddress = "3393 Peachtree Rd",
City = "Atlanta",
StateProvince = "GA",
PostalCode = "30326",
Country = "USA"
},
Location = GeographyPoint.Create(22, 22),
Rooms = new Room[]
{
new Room()
{
Description = "Room Des", BaseRate = 100, DescriptionFr = "Le Room", BedOptions = "Any", Type = "Single", SmokingAllowed = false, SleepsCount = 3, Tags = new string[] { "single", "front", "near pool" }
}
},
cheapest = 100,
expensive = 300,
}),
IndexDocumentsAction.Upload(
new Hotel()
{
HotelId = "4",
HotelName = "Sublime Cliff Hotel",
Description = "Sublime Cliff Hotel is located in the heart of the historic center of Sublime in an extremely vibrant and lively area within short walking distance to the sites and landmarks of the city and is surrounded by the extraordinary beauty of churches, buildings, shops and monuments. Sublime Cliff is part of a lovingly restored 1800 palace.",
DescriptionFr = "Le sublime Cliff Hotel est situé au coeur du centre historique de sublime dans un quartier extrêmement animé et vivant, à courte distance de marche des sites et monuments de la ville et est entouré par l'extraordinaire beauté des églises, des bâtiments, des commerces et Monuments. Sublime Cliff fait partie d'un Palace 1800 restauré avec amour.",
Category = "Boutique",
Tags = new[] { "concierge", "view", "24-hour front desk service" },
ParkingIncluded = true,
LastRenovationDate = new DateTimeOffset(1960, 2, 06, 0, 0, 0, TimeSpan.Zero),
Rating = 4.60,
Address = new Address()
{
StreetAddress = "7400 San Pedro Ave",
City = "San Antonio",
StateProvince = "TX",
PostalCode = "78216",
Country = "USA"
},
Location = GeographyPoint.Create(22, 22),
Rooms = new Room[]
{
new Room()
{
Description = "Room Des", BaseRate = 100, DescriptionFr = "Le Room", BedOptions = "Any", Type = "Single", SmokingAllowed = false, SleepsCount = 3, Tags = new string[] { "single", "front", "near pool" }
}
},
cheapest = 100,
expensive = 300,
})
);
IndexDocumentsResult result = await _searchClient.IndexDocumentsAsync(batch);
}
