private static void UploadData()
{
var actions = new IndexAction <Customer>[]
{
IndexAction.Upload(new Customer()
{
customerid = "1", customername = "John", customeremail = "*****@*****.**"
}),
IndexAction.Upload(new Customer()
{
customerid = "2", customername = "Mark", customeremail = "*****@*****.**"
})
};
var batch = IndexBatch.New(actions);
SearchServiceClient serviceClient = new SearchServiceClient(searchServiceName, new SearchCredentials(adminApiKey));
ISearchIndexClient indexClient = serviceClient.Indexes.GetClient("customer");
indexClient.Documents.Index(batch);
Console.WriteLine("Added Data");
Console.ReadKey();
}
private static void UploadDocuments(ISearchIndexClient indexClient)
{
var actions = new List <IndexAction <Team> >();
foreach (var team in Teams.GetTeams())
{
Console.WriteLine($"Uploading team {team.TeamName}");
actions.Add(IndexAction.Upload(team));
}
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
Console.WriteLine("Waiting for indexing...\n");
Thread.Sleep(2000);
}
private IEnumerable <string> IndexThousandsOfDocuments(SearchIndexClient client)
{
int existingDocumentCount = Data.TestDocuments.Length;
IEnumerable <string> hotelIds =
Enumerable.Range(existingDocumentCount + 1, 2001 - existingDocumentCount).Select(id => id.ToString());
IEnumerable <Hotel> hotels = hotelIds.Select(id => new Hotel()
{
HotelId = id
});
IEnumerable <IndexAction <Hotel> > actions = hotels.Select(h => IndexAction.Create(h));
var batch = IndexBatch.Create(actions.Take(1000));
client.Documents.Index(batch);
SearchTestUtilities.WaitForIndexing();
batch = IndexBatch.Create(actions.Skip(1000));
client.Documents.Index(batch);
SearchTestUtilities.WaitForIndexing();
return(hotelIds);
}
private static void ImportDocument(ISearchIndexClient indexClient)
{
var actions = new List <IndexAction <Account> >();
string line = string.Empty;
using (var file = new StreamReader("accounts.json"))
{
while ((line = file.ReadLine()) != null)
{
JObject json = JObject.Parse(line);
Account account = json.ToObject <Account>();
actions.Add(IndexAction.Upload(account));
}
file.Close();
}
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (Exception)
{
throw;
}
}
public void StaticallyTypedDateTimesRoundTripAsUtc()
{
Run(() =>
{
SearchServiceClient serviceClient = Data.GetSearchServiceClient();
Index index =
new Index()
{
Name = TestUtilities.GenerateName(),
Fields = new[]
{
new Field("ISBN", DataType.String)
{
IsKey = true
},
new Field("PublishDate", DataType.DateTimeOffset)
}
};
IndexDefinitionResponse createIndexResponse = serviceClient.Indexes.Create(index);
Assert.Equal(HttpStatusCode.Created, createIndexResponse.StatusCode);
SearchIndexClient indexClient = Data.GetSearchIndexClient(createIndexResponse.Index.Name);
DateTime localDateTime = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Local);
DateTime utcDateTime = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc);
DateTime unspecifiedDateTime = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Unspecified);
var batch =
IndexBatch.Create(
new[]
{
IndexAction.Create(new Book()
{
ISBN = "1", PublishDate = localDateTime
}),
IndexAction.Create(new Book()
{
ISBN = "2", PublishDate = utcDateTime
}),
IndexAction.Create(new Book()
{
ISBN = "3", PublishDate = unspecifiedDateTime
})
});
indexClient.Documents.Index(batch);
SearchTestUtilities.WaitForIndexing();
DocumentGetResponse <Book> getResponse = indexClient.Documents.Get <Book>("1");
Assert.Equal(localDateTime.ToUniversalTime(), getResponse.Document.PublishDate);
getResponse = indexClient.Documents.Get <Book>("2");
Assert.Equal(utcDateTime, getResponse.Document.PublishDate);
getResponse = indexClient.Documents.Get <Book>("3");
Assert.Equal(utcDateTime, getResponse.Document.PublishDate);
});
}
public void GetDocumentThrowsWhenRequestIsMalformed()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var indexedDoc =
new Hotel()
{
HotelId = "3",
BaseRate = 279.99,
Description = "Surprisingly expensive"
};
var batch = IndexBatch.Create(IndexAction.Create(indexedDoc));
client.Documents.Index(batch);
string[] selectedFields = new[] { "hotelId", "ThisFieldDoesNotExist" };
CloudException e = Assert.Throws <CloudException>(() => client.Documents.Get("3", selectedFields));
Assert.Equal(HttpStatusCode.BadRequest, e.Response.StatusCode);
Assert.Contains(
"Invalid expression: Could not find a property named 'ThisFieldDoesNotExist' on type 'search.document'.",
e.Message);
});
}
public async Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken))
{
try
{
if (_messages.Count == 0)
{
return;
}
var uploadAction = _messages.Where(w => w.Insert).Select(s => IndexAction.MergeOrUpload(s.Item));
var deleteAction = _messages.Where(w => !w.Insert).Select(s => IndexAction.Delete(s.Item));
var batch = IndexBatch.New(uploadAction.Union(deleteAction));
var result = await _indexClient.Documents.IndexAsync(batch, cancellationToken : cancellationToken);
_messages.Clear();
#if DEBUG
foreach (var errorResult in result.Results.Where(w => !w.Succeeded))
{
Debug.WriteLine(
$"Failed to process id {errorResult.Key} error {errorResult.ErrorMessage} on index {_indexClient.IndexName} ");
}
#endif
}
catch (Exception ex)
{
Debug.WriteLine(ex);
throw;
}
}
public void CanGetStaticallyTypedDocumentWithNullOrEmptyValues()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var expectedDoc =
new Hotel()
{
HotelId = "1",
BaseRate = null,
HotelName = null,
Tags = new string[0],
ParkingIncluded = null,
LastRenovationDate = null,
Rating = null,
Location = null
};
var batch = IndexBatch.Create(IndexAction.Create(expectedDoc));
client.Documents.Index(batch);
SearchTestUtilities.WaitForIndexing();
DocumentGetResponse <Hotel> getResponse = client.Documents.Get <Hotel>("1");
Assert.Equal(HttpStatusCode.OK, getResponse.StatusCode);
Assert.Equal(expectedDoc, getResponse.Document);
});
}
public void RoundTrippingDateTimeOffsetNormalizesToUtc()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var indexedDoc =
new Hotel()
{
HotelId = "1",
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.FromHours(-8))
};
var expectedDoc =
new Hotel()
{
HotelId = "1",
Tags = new string[0], // null arrays become empty arrays during indexing.
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 8, 0, 0, TimeSpan.Zero)
};
var batch = IndexBatch.Create(IndexAction.Create(indexedDoc));
client.Documents.Index(batch);
DocumentGetResponse <Hotel> response = client.Documents.Get <Hotel>("1");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(expectedDoc, response.Document);
});
}
public void CanGetStaticallyTypedDocument()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var expectedDoc =
new Hotel()
{
HotelId = "1",
BaseRate = 199.0,
Description = "Best hotel in town",
DescriptionFr = "Meilleur hôtel en ville",
HotelName = "Fancy Stay",
Category = "Luxury",
Tags = new[] { "pool", "view", "wifi", "concierge" },
ParkingIncluded = false,
SmokingAllowed = false,
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.Zero),
Rating = 5,
Location = GeographyPoint.Create(47.678581, -122.131577)
};
var batch = IndexBatch.Create(IndexAction.Create(expectedDoc));
client.Documents.Index(batch);
DocumentGetResponse <Hotel> response = client.Documents.Get <Hotel>("1");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(expectedDoc, response.Document);
});
}
private void ImportData(ISearchIndexClient indexClient)
{
var actions = new List <IndexAction <Account> >();
string line;
using (System.IO.StreamReader file = new System.IO.StreamReader("accounts.json"))
{
while ((line = file.ReadLine()) != null)
{
JObject json = JObject.Parse(line);
Account account = json.ToObject <Account>();
actions.Add(IndexAction.Upload(account));
}
file.Close();
}
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException ex)
{
_logger.Error(ex, ex.Message);
throw ex;
}
}
public ActionResult Create(Player player)
{
if (string.IsNullOrEmpty(player.Name))
{
log.Error("Player name shouldn't be empty");
throw new ArgumentException("Player name shouldn't be empty");
}
player.Id = Guid.NewGuid();
db.Players.Add(player);
db.SaveChanges();
var actions = new IndexAction <Player>[]
{
IndexAction.Upload(player)
};
var batch = IndexBatch.New(actions);
try
{
_soccerIndex.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
return(RedirectToAction("Index"));
}
static void AddSentencesToIndex(string[] sentences)
{
var indexClient = _searchClient.Indexes.GetClient(_indexName);
var docActions = new List <IndexAction <AliceDocument> >();
for (int i = 0; i < sentences.Length; i++)
{
var doc = new AliceDocument()
{
documentId = i.ToString(),
standardText = sentences[i],
englishText = sentences[i],
microsoftText = sentences[i]
};
docActions.Add(IndexAction.MergeOrUpload(doc));
}
var docBatch = IndexBatch.New(docActions);
try
{
var result = indexClient.Documents.Index(docBatch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine("Failed to index some of the documents: " + string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
}
private static void ImportData(SearchServiceClient serviceClient)
{
List <RentingStats> rentingStats = new List <RentingStats>();
using (var streamReader = new StreamReader(dataFileName))
using (var csvReader = new CsvReader(streamReader))
{
rentingStats = csvReader.GetRecords <RentingStats>().ToList();
}
var actions = new List <IndexAction <RentingStats> >();
foreach (var rentingStat in rentingStats)
{
actions.Add(IndexAction.Upload(rentingStat));
}
var batch = IndexBatch.New(actions);
try
{
ISearchIndexClient indexClient = serviceClient.Indexes.GetClient(indexName);
indexClient.Documents.Index(batch);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
public static AzureSearchIndexResult IndexContentBatch(this SearchServiceClient serviceClient, string indexName, IEnumerable <Document> contents)
{
var result = new AzureSearchIndexResult();
var actions = new List <IndexAction>();
foreach (var content in contents)
{
actions.Add(IndexAction.Upload(content));
}
var batch = IndexBatch.New(actions);
var indexClient = serviceClient.Indexes.GetClient(indexName);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
var error =
"Failed to index some of the documents: {0}" + String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key));
result.Success = false;
result.Message = error;
return(result);
}
result.Success = true;
return(result);
}
private IEnumerable <string> IndexDocuments(SearchIndexClient client, int totalDocCount)
{
int existingDocumentCount = Data.TestDocuments.Length;
IEnumerable <string> hotelIds =
Enumerable.Range(existingDocumentCount + 1, totalDocCount - existingDocumentCount)
.Select(id => id.ToString());
IEnumerable <Hotel> hotels = hotelIds.Select(id => new Hotel()
{
HotelId = id
});
List <IndexAction <Hotel> > actions = hotels.Select(h => IndexAction.Create(h)).ToList();
for (int i = 0; i < actions.Count; i += 1000)
{
IEnumerable <IndexAction <Hotel> > nextActions = actions.Skip(i).Take(1000);
if (!nextActions.Any())
{
break;
}
var batch = IndexBatch.Create(nextActions);
client.Documents.Index(batch);
SearchTestUtilities.WaitForIndexing();
}
return(hotelIds);
}
public static void UploadDocuments()
{
// This will open the JSON file, parse it and upload the documents in a batch
List <IndexAction> indexOperations = new List <IndexAction>();
JArray json = JArray.Parse(File.ReadAllText(@"contacts.json"));
foreach (var contact in json)
{
//Parse the JSON object (contact)
var doc = new Document();
doc.Add("id", contact["id"]);
doc.Add("name", contact["name"]);
doc.Add("company", contact["company"]);
doc.Add("locationsId", contact["locations"].Select(item => item["id"]).ToList());
doc.Add("locationsDescription", contact["locations"].Select(item => item["description"]).ToList());
doc.Add("locationsCombined", contact["locations"].Select(item => item["id"] + "||" + item["description"]).ToList());
indexOperations.Add(IndexAction.Upload(doc));
}
try
{
indexClient.Documents.Index(new IndexBatch(indexOperations));
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
}
public static void UploadDocuments(ISearchIndexClient indexClient, string fileId, string summary, List <string> keyPhrases)
{
// This is really inefficient as I should be batching the uploads
List <IndexAction> indexOperations = new List <IndexAction>();
var doc = new Document();
doc.Add("fileId", fileId);
doc.Add("summary", summary);
doc.Add("keyPhrases", keyPhrases);
indexOperations.Add(IndexAction.Upload(doc));
try
{
indexClient.Documents.Index(new IndexBatch(indexOperations));
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
}
private async Task ImportDataAsync()
{
var data = await File.ReadAllLinesAsync(options.HotelFileName).ConfigureAwait(false);
var hotels = data.Skip(1).Select(r =>
{
var columns = r.Split("\t");
return(new Hotel
{
HotelId = columns[0],
HotelName = columns[1],
Description = columns[2],
DescriptionFr = columns[3],
Category = columns[4],
Tags = columns[5].Split(",").ToImmutableArray(),
ParkingIncluded = columns[6] == "0" ? false : true,
SmokingAllowed = columns[7] == "0" ? false : true,
LastRenovationDate = Convert.ToDateTime(columns[8], CultureInfo.InvariantCulture),
BaseRate = Convert.ToDouble(columns[9], CultureInfo.InvariantCulture),
Rating = Convert.ToInt32(Convert.ToDouble(columns[10], CultureInfo.InvariantCulture))
});
}).ToList();
var actions = hotels.Select(h => IndexAction.Upload(h)).ToList();
var batch = IndexBatch.New(actions);
using (var client = CreateSearchServiceClient())
{
await client.Indexes.GetClient(options.IndexName).Documents.IndexAsync(batch).ConfigureAwait(false);
}
logger.LogInformation("Imported {@Count} hotels", hotels.Count);
}
/// <summary>
/// Adds all new records to index
/// </summary>
/// <param name="newRecords">The new or existing records which should be indexed.</param>
public async Task AddOrUpdateIndexDataAsync(IEnumerable <EmployerSearchModel> newRecords)
{
if (Disabled)
{
throw new Exception($"{nameof(AzureEmployerSearchRepository)} is disabled");
}
if (newRecords == null || !newRecords.Any())
{
throw new ArgumentNullException(nameof(newRecords), "You must supply at least one record to index");
}
//Remove all test organisations
if (!string.IsNullOrWhiteSpace(SharedOptions.TestPrefix))
{
newRecords = newRecords.Where(e => !e.Name.StartsWithI(SharedOptions.TestPrefix));
}
//Ensure the records are ordered by name
newRecords = newRecords.OrderBy(o => o.Name);
//Set the records to add or update
var actions = newRecords.Select(r => IndexAction.MergeOrUpload(_autoMapper.Map <AzureEmployerSearchModel>(r)))
.ToList();
var batches = new ConcurrentBag <IndexBatch <AzureEmployerSearchModel> >();
while (actions.Any())
{
var batchSize = actions.Count > 1000 ? 1000 : actions.Count;
var batch = IndexBatch.New(actions.Take(batchSize).ToList());
batches.Add(batch);
actions.RemoveRange(0, batchSize);
}
var indexClient = await _indexClient.Value;
Parallel.ForEach(
batches,
batch =>
{
var retries = 0;
retry:
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException)
{
if (retries < 30)
{
retries++;
Thread.Sleep(1000);
goto retry;
}
throw;
}
});
}
public void UploadSceneDocuments <T>(IEnumerable <T> sceneDocuments)
{
if (sceneDocuments == null)
{
throw new NullReferenceException();
}
SearchIndexClient indexClient = new SearchIndexClient(_config.Name, _config.SceneIndexName, _client.SearchCredentials);
var actions = new List <IndexAction <T> >();
foreach (var sceneDocument in sceneDocuments)
{
actions.Add(IndexAction.MergeOrUpload(sceneDocument));
}
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException ex)
{
_logger.Error(ex.Message);
throw;
}
finally
{
indexClient.Dispose();
}
}
public void RemoveFromIndex <T>(IEnumerable <T> documents)
{
var actions = documents.Select(a => IndexAction.Delete(a));
var batch = IndexBatch.New(actions);
searchIndexClient.Documents.Index(batch);
}
public async Task IndexAsync(params string[] packageIds)
{
if (packageIds == null)
{
throw new ArgumentNullException(nameof(packageIds));
}
var actions = new List <IndexAction <PackageDocument> >();
var packageIdSet = new HashSet <string>(packageIds, StringComparer.OrdinalIgnoreCase);
if (packageIdSet.Count > MaxBatchSize)
{
throw new ArgumentException($"Cannot index more than {MaxBatchSize} packages at once");
}
_logger.LogInformation("Indexing {PackageCount} packages...", packageIdSet.Count);
foreach (var packageId in packageIdSet)
{
foreach (var document in await BuildDocumentsAsync(packageId))
{
actions.Add(IndexAction.Upload(document));
}
}
var batch = IndexBatch.New(actions);
// TODO: Add retry on IndexBatchException
// See: https://docs.microsoft.com/en-us/azure/search/search-import-data-dotnet#import-data-to-the-index
await _indexClient.Documents.IndexAsync(batch);
_logger.LogInformation("Indexed {PackageCount} packages", packageIdSet.Count);
}
public void Delete(string id)
{
var result = new AzureSearchIndexResult();
var serviceClient = GetClient();
var actions = new List <IndexAction>();
var d = new Document();
d.Add("Id", id);
actions.Add(IndexAction.Delete(d));
var batch = IndexBatch.New(actions);
var indexClient = serviceClient.Indexes.GetClient(_config.IndexName);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
var error =
"Failed to index some of the documents: {0}" + String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key));
result.Success = false;
result.Message = error;
}
result.Success = true;
}
private static async Task AddOrUpdateIndex(SearchServiceClient searchClient, IEnumerable <Project> projects, string indexName, ILogger log)
{
var indexClient = searchClient.Indexes.GetClient(indexName);
var indexActions = projects
.Select(project =>
{
var projectNameMatch = Regex.Match(project.MarkdownDescription, @"^\s*#(?!#)\s*(.*?)\s*$", RegexOptions.Multiline);
var projectName = projectNameMatch.Success ? projectNameMatch.Groups[1].Value : project.Name;
var contributor = project.ContributorInfo;
var document = new Document
{
{ "id", project.Id },
{ "name", projectName },
{ "contributorName", contributor.Name },
{ "contributorUrl", contributor.Web },
{ "contributorLogo", contributor.Logo },
{ "descriptionMarkdownFilename", project.Name },
{ "descriptionMarkdown", project.MarkdownDescription }
};
log.LogInformation("{name}, {contributorName}", projectName, contributor.Name);
return(IndexAction.MergeOrUpload(document));
})
.OfType <IndexAction>();
var result = await indexClient.Documents.IndexAsync(new IndexBatch(indexActions));
}
public static IndexAction ElitUaApiModelToIndexAction(
ElitPriceListRecord apiProduct,
long updatedOnUtcTimestamp)
{
if (apiProduct == null)
{
return(null);
}
var source = EkProductSourceEnum.ElitUa;
var productKey = new EkProductKey(source, ReplaceInvalidAzureSearchKeySymbolsWithDash(apiProduct.ActiveItemNo))
.ToKey();
var nameRu = GetValueOrFallback(apiProduct.EcatDescription, apiProduct.ItemDescription);
var product = new Document()
{
["key"] = productKey,
["updatedOnUtcTimestamp"] = updatedOnUtcTimestamp,
["source"] = (int)source,
["sourceId"] = apiProduct.ActiveItemNo,
["partNumber"] = apiProduct.PartNumber,
["cleanedPartNumber"] = PartNumberCleaner.GetCleanedPartNumber(apiProduct.PartNumber),
["brandName"] = apiProduct.Brand,
["cleanedBrandPartNumber"] = PartNumberCleaner.GetCleanedBrandPartNumber(
brandName: apiProduct.Brand,
partNumber: apiProduct.PartNumber),
["name_ru"] = SearchTextHelpers.TrimNameAndAddBrandIfMissed(
productName: nameRu,
brandName: apiProduct.Brand),
["price"] = (double)apiProduct.CustomerPrice,
};
return(IndexAction.MergeOrUpload(product));
}
private static async Task WriteToSiAsync(IEnumerable <Book> listBook)
{
SearchServiceClient serviceClient = new SearchServiceClient("librarysearchservice", new SearchCredentials("<api-key>"));
var indexClient = serviceClient.Indexes.GetClient("booksindex");
var act = listBook.Select(x => IndexAction.Upload <Book>(x));
await indexClient.Documents.IndexAsync(IndexBatch.New(act));
}
private static async Task UploadToAzureSeearch(ISearchIndexClient indexClient, string documentId, List <string> keyPhrases, string summary, TraceWriter log)
{
var document = new Document();
document.Add(KeyField, documentId);
document.Add(SummaryField, summary);
document.Add(KeyPhrasesField, string.Join(", ", keyPhrases));
var indexOperations = new List <IndexAction>()
{
IndexAction.MergeOrUpload(document)
};
try
{
await indexClient.Documents.IndexAsync(new IndexBatch(indexOperations));
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
log.Info("Failed to index some of the documents: " + string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
}
public static async Task Run([BlobTrigger("properties/{name}", Connection = "StorageConnection")] Stream myBlob, string name, TraceWriter log)
{
var searchServiceName = ConfigurationManager.AppSettings["SearchServiceName"];
var adminApiKey = ConfigurationManager.AppSettings["SearchServiceAdminApiKey"];
var serviceClient = new SearchServiceClient(searchServiceName, new SearchCredentials(adminApiKey));
var blogPost = GetBlogPost(myBlob);
blogPost.filePath = $"{ConfigurationManager.AppSettings["FilePathBeginning"]}{name}";
var action = new[]
{
IndexAction.Upload(blogPost)
};
var batch = IndexBatch.New(action);
var indexClient = serviceClient.Indexes.GetClient(ConfigurationManager.AppSettings["SearchIndexName"]);
try
{
await indexClient.Documents.IndexAsync(batch);
}
catch (IndexBatchException e)
{
log.Error("Error: ", e);
throw;
}
}
public void AddToIndex <T>(IEnumerable <T> documents)
{
var actions = documents.Select(a => IndexAction.Upload(a));
var batch = IndexBatch.New(actions);
searchIndexClient.Documents.Index(batch);
}
private void IndexContent(SearchContent searchContent, IndexAction action)
{
switch (action)
{
case IndexAction.Create:
this._searchService.AddContent(searchContent);
break;
case IndexAction.Update:
this._searchService.UpdateContent(searchContent);
break;
case IndexAction.Delete:
this._searchService.DeleteContent(searchContent);
break;
}
}
private static void UploadDocuments(ISearchIndexClient indexClient)
{
var actions =
new IndexAction<Hotel>[]
{
IndexAction.Upload(
new Hotel()
{
HotelId = "1",
BaseRate = 199.0,
Description = "Best hotel in town",
DescriptionFr = "Meilleur hôtel en ville",
HotelName = "Fancy Stay",
Category = "Luxury",
Tags = new[] { "pool", "view", "wifi", "concierge" },
ParkingIncluded = false,
SmokingAllowed = false,
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.Zero),
Rating = 5,
Location = GeographyPoint.Create(47.678581, -122.131577)
}),
IndexAction.Upload(
new Hotel()
{
HotelId = "2",
BaseRate = 79.99,
Description = "Cheapest hotel in town",
DescriptionFr = "Hôtel le moins cher en ville",
HotelName = "Roach Motel",
Category = "Budget",
Tags = new[] { "motel", "budget" },
ParkingIncluded = true,
SmokingAllowed = true,
LastRenovationDate = new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero),
Rating = 1,
Location = GeographyPoint.Create(49.678581, -122.131577)
}),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "3",
BaseRate = 129.99,
Description = "Close to town hall and the river"
}),
IndexAction.Delete(new Hotel() { HotelId = "6" })
};
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
Console.WriteLine("Waiting for documents to be indexed...\n");
Thread.Sleep(2000);
}
public async void Commit(IndexAction action, int retry = 0)
{
try
{
await AzureIndex.AzureIndexClient.Documents.IndexWithHttpMessagesAsync(IndexBatch.New(new List<IndexAction> { action }));
//var response = AzureIndex.AzureIndexClient.Documents.IndexWithHttpMessagesAsync(IndexBatch.New(actions));
//response.Wait();
}
catch (Exception ex)
{
if (retry < 6)
{
Thread.Sleep(50);
Commit(action, retry++);
}
else
{
CrawlingLog.Log.Warn("Error indexing on Item for " + Index.Name, ex);
}
}
}
private void IndexContent(SearchContent searchContent, IndexAction action)
{
// Index
string indexDir = Context.Server.MapPath(Config.GetConfiguration()["SearchIndexDir"]);
IndexBuilder ib = new IndexBuilder(indexDir, false);
switch (action)
{
case IndexAction.Create:
ib.AddContent(searchContent);
break;
case IndexAction.Update:
ib.UpdateContent(searchContent);
break;
case IndexAction.Delete:
ib.DeleteContent(searchContent);
break;
}
ib.Close();
}
private static void UploadDocuments(string fileName)
{
List<IndexAction> indexOperations = new List<IndexAction>();
DirectoryInfo Folder = new DirectoryInfo(IndexName); ;
IndexAction ia = new IndexAction();
object session = Newtonsoft.Json.JsonConvert.DeserializeObject(System.IO.File.ReadAllText(Path.Combine("data", fileName)));
JArray docArray = (JArray)(session);
foreach (var document in docArray)
{
Document doc = new Document();
doc.Add("product_id", Convert.ToString(document["product_id"]));
doc.Add("product_class_id", Convert.ToInt32(document["product_class_id"]));
doc.Add("brand_name", Convert.ToString(document["brand_name"]));
doc.Add("product_name", Convert.ToString(document["product_name"]));
doc.Add("sku", Convert.ToString(document["SKU"]));
doc.Add("srp", Convert.ToInt32(document["srp"]));
doc.Add("gross_weight", Convert.ToDouble(document["gross_weight"]));
doc.Add("net_weight", Convert.ToDouble(document["net_weight"]));
doc.Add("recyclable_package", Convert.ToString(document["recyclable_package"]));
doc.Add("low_fat", Convert.ToString(document["low_fat"]));
doc.Add("units_per_case", Convert.ToInt32(document["units_per_case"]));
doc.Add("cases_per_pallet", Convert.ToInt32(document["cases_per_pallet"]));
doc.Add("shelf_width", Convert.ToDouble(document["shelf_width"]));
doc.Add("shelf_height", Convert.ToDouble(document["shelf_height"]));
doc.Add("shelf_depth", Convert.ToDouble(document["shelf_depth"]));
doc.Add("product_subcategory", Convert.ToString(document["product_subcategory"]));
doc.Add("product_category", Convert.ToString(document["product_category"]));
doc.Add("product_department", Convert.ToString(document["product_department"]));
doc.Add("product_family", Convert.ToString(document["product_family"]));
doc.Add("url", Convert.ToString(document["url"]));
indexOperations.Add(new IndexAction(IndexActionType.Upload, doc));
}
IndexBatch(indexOperations);
}
/// <summary>
/// Internal load method called from the constructor. Loads underlying values
/// based on Xml configuration.
/// </summary>
/// <param name="node">XmlNode definition for a given IndexSet</param>
internal void LoadValues(XmlNode node)
{
XmlAttributeCollection attributeCollection = node.Attributes;
try
{
this._intId = Convert.ToInt32(attributeCollection["id"].Value);
}
catch (Exception)
{
throw new ConfigurationErrorsException("IndexSet id invalid: " + Environment.NewLine + node.OuterXml);
}
try
{
this._eIndexAction = (IndexAction)Enum.Parse(typeof(IndexAction), attributeCollection["action"].Value);
}
catch (Exception)
{
throw new ConfigurationErrorsException("IndexSet "+this._intId.ToString()+" IndexAction invalid: " + Environment.NewLine + node.OuterXml);
}
try
{
if (attributeCollection["analyzer"] != null)
this._eAnalyzerType = (AnalyzerType)Enum.Parse(typeof(AnalyzerType), attributeCollection["analyzer"].Value);
}
catch (Exception)
{
throw new ConfigurationErrorsException("IndexSet " + this._intId.ToString() + " analyzer invalid: " + Environment.NewLine + node.OuterXml);
}
if (node.ChildNodes.Count==0)
throw new ConfigurationErrorsException("IndexSet " + this._intId.ToString() + " configuration missing " + Environment.NewLine + node.OuterXml);
foreach (XmlNode c in node.ChildNodes)
{
if (!c.HasChildNodes)
{
switch (c.Attributes["key"].Value.ToLower())
{
case "localpath":
this._strLocalPath = c.Attributes["value"].Value;
break;
case "idcolumn":
this._strIdColumn = c.Attributes["value"].Value;
break;
case "bottomid":
try
{
this._intBottomId = Convert.ToInt32(c.Attributes["value"].Value);
}
catch (Exception)
{
throw new ConfigurationErrorsException("IndexSet " + this._intId.ToString() + " bottomid invalid: " + Environment.NewLine + node.OuterXml);
}
break;
case "topid":
try
{
this._intTopId = Convert.ToInt32(c.Attributes["value"].Value);
}
catch (Exception)
{
throw new ConfigurationErrorsException("IndexSet " + this._intId.ToString() + " topid invalid: " + Environment.NewLine + node.OuterXml);
}
break;
}
}
else
{
switch(c.Name.ToLower())
{
case "copy":
if (this._strLocalPath!=null)
LoadCopy(c,this._strLocalPath);
else
LoadCopy(c,node);
break;
}
}
}
this.CheckValidSet(node);
}
public Task UpdateAsync(ProductInfo document)
{
var action = new IndexAction<ProductInfo>(IndexActionType.MergeOrUpload, document);
return indexClient.Documents.IndexAsync(IndexBatch.Create(action));
}
