public void Delete(string id)
{
var result = new AzureSearchIndexResult();
var serviceClient = GetClient();
var actions = new List <IndexAction>();
var d = new Document();
d.Add("Id", id);
actions.Add(IndexAction.Delete(d));
var batch = IndexBatch.New(actions);
var indexClient = serviceClient.Indexes.GetClient(_config.IndexName);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
var error =
"Failed to index some of the documents: {0}" + String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key));
result.Success = false;
result.Message = error;
}
result.Success = true;
}
public void RemoveFromIndex <T>(IEnumerable <T> documents)
{
var actions = documents.Select(a => IndexAction.Delete(a));
var batch = IndexBatch.New(actions);
searchIndexClient.Documents.Index(batch);
}
public async Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken))
{
try
{
if (_messages.Count == 0)
{
return;
}
var uploadAction = _messages.Where(w => w.Insert).Select(s => IndexAction.MergeOrUpload(s.Item));
var deleteAction = _messages.Where(w => !w.Insert).Select(s => IndexAction.Delete(s.Item));
var batch = IndexBatch.New(uploadAction.Union(deleteAction));
var result = await _indexClient.Documents.IndexAsync(batch, cancellationToken : cancellationToken);
_messages.Clear();
#if DEBUG
foreach (var errorResult in result.Results.Where(w => !w.Succeeded))
{
Debug.WriteLine(
$"Failed to process id {errorResult.Key} error {errorResult.ErrorMessage} on index {_indexClient.IndexName} ");
}
#endif
}
catch (Exception ex)
{
Debug.WriteLine(ex);
throw;
}
}
public Task DeleteAccommodationIndexAsync(Guid accommodationId)
{
var action = IndexAction.Delete(new Document {
{ "Id", accommodationId }
});
return(client.Documents.IndexAsync(new IndexBatch(new[] { action })));
}
/// <summary>
/// Delete documents from index document by company's user
/// </summary>
/// <param name="lstBlobs"></param>
/// <param name="usrUser"></param>
/// <returns></returns>
public List <BlobStructure> DeleteDocumentsFromIndex(List <BlobStructure> lstBlobs, User usrUser)
{
var idxSearch = CreateIfNotExistIndex(usrUser, cnsIndexTypes.Documents);
// Initialize client search
var client = new SearchIndexClient(ConfigurationManager.AppSettings["SearchServiceName"],
idxSearch.IndexName,
new SearchCredentials(ConfigurationManager.AppSettings["SearchApiKey"]));
// Only uploaded files
foreach (var blbBlob in lstBlobs.Where(x => x.IsUploaded))
{
var dcmMetadata = blbBlob.dctMetadata.ToObject <clsDocumentPdfIndex>();
var strFileNameWithOutExt = blbBlob.fntNameStr.Split('.')[0];
// If files not are valid documents
if (!strExtensions.Equals(dcmMetadata.Extension))
{
continue;
}
try
{
var lstActions = new List <IndexAction <clsDocumentPdfIndex> >();
// download blob like stream var
var streamFile = new Blobs.BlobStorage().fntGetStreamBlob(blbBlob);
streamFile.Position = 0;
// Create a index action by each page
using (PdfReader reader = new PdfReader(streamFile))
{
for (int indexPage = 1; indexPage <= reader.NumberOfPages; indexPage++)
{
// add index page before extension file
dcmMetadata.Id = MakeSafeId(strFileNameWithOutExt + "_" + indexPage + "." + dcmMetadata.Extension);
lstActions.Add(IndexAction.Delete(dcmMetadata));
}
}
// Index batch process
for (int i = 0; i < (int)Math.Ceiling(lstActions.Count / (double)intRegistersSkip); i++)
{
client.Documents.Index(new IndexBatch <clsDocumentPdfIndex>(lstActions.Skip(i * intRegistersSkip).Take(intRegistersSkip)));
}
lstBlobs.First(l => l.fntNameStr == blbBlob.fntNameStr).IsIndexed = true;
}
catch (Exception)
{
lstBlobs.First(l => l.fntNameStr == blbBlob.fntNameStr).IsIndexed = false;
}
}
return(lstBlobs);
}
public async Task DeleteAsync <T>(T item, IGraphRequestContext graphRequestContext) where T : class
{
var client = Get <T>(graphRequestContext);
var idx = IndexAction.Delete(item);
await client.Documents.IndexAsync(new IndexBatch <T>(
new List <IndexAction <T> > {
idx
}));
}
public void Delete(Player player)
{
var actions = new List <IndexAction <PlayerSearchEntry> >
{
IndexAction.Delete(player.ToSearchEntry())
};
var batch = IndexBatch.New(actions);
_playersIndexClient.Documents.Index(batch);
}
public static IndexAction OmegaAutoBizApiModelToIndexAction(
ProductSearchRecord apiProduct,
long updatedOnUtcTimestamp)
{
if (apiProduct == null)
{
return(null);
}
var source = EkProductSourceEnum.OmegaAutoBiz;
var productKey = new EkProductKey(source, apiProduct.ProductId.ToString())
.ToKey();
var price = GetValueOrFallback(apiProduct.CustomerPrice, apiProduct.Price);
if (price == 0)
{
return(IndexAction.Delete(new Document()
{
["key"] = productKey,
}));
}
var partNumber = apiProduct.Number?.Trim();
var brandName = apiProduct.BrandDescription?.Trim();
var nameRu = apiProduct.Description?.Trim();
var nameUk = GetValueOrFallback(apiProduct.DescriptionUkr?.Trim(), nameRu);
var description = apiProduct.Info?.Trim();
var product = new Document()
{
["key"] = productKey,
["updatedOnUtcTimestamp"] = updatedOnUtcTimestamp,
["source"] = (int)source,
["sourceId"] = apiProduct.ProductId.ToString(),
["partNumber"] = partNumber,
["cleanedPartNumber"] = PartNumberCleaner.GetCleanedPartNumber(partNumber),
["brandName"] = brandName,
["cleanedBrandPartNumber"] = PartNumberCleaner.GetCleanedBrandPartNumber(
brandName: brandName,
partNumber: partNumber),
["name_ru"] = SearchTextHelpers.TrimNameAndAddBrandIfMissed(
productName: nameRu,
brandName: brandName),
["name_uk"] = SearchTextHelpers.TrimNameAndAddBrandIfMissed(
productName: nameUk,
brandName: brandName),
["description_ru"] = description,
["description_uk"] = description,
["price"] = (double)price,
};
return(IndexAction.MergeOrUpload(product));
}
public async Task Remove(Guid planId)
{
using (var searchClient = CreateAzureSearchClient())
{
using (var indexClient = searchClient.Indexes.GetClient(GetPlanTemplateIndexName()))
{
var doc = ConvertToSearchDocument(planId);
var batch = IndexBatch.New(new[] { IndexAction.Delete(doc) });
await indexClient.Documents.IndexAsync(batch);
}
}
}
private static async Task RunDeletePackageAsync(PackageOperation packageOperation, CloudBlockBlob packageBlob, ISearchIndexClient indexClient, List <IndexAction <PackageDocument> > indexActions)
{
// Delete from index blob
await packageBlob.DeleteIfExistsAsync();
// Add delete from index actions
await indexClient.Documents.ForEachAsync <PackageDocument>(
$"Id eq '{packageOperation.Id}' and Version eq '{packageOperation.Version}'",
result =>
{
indexActions.Add(IndexAction.Delete(result.Document));
return(true);
});
}
public async Task DeleteDocumentsByIdsAsync(List <string> ids)
{
int maxBatchSize = 1000;
int pagesCount = (ids.Count / maxBatchSize) + (ids.Count % maxBatchSize > 0 ? 1 : 0);
for (var i = 0; i < pagesCount; i++)
{
var indexActions = ids
.Skip(i * maxBatchSize)
.Take(maxBatchSize)
.Select(x => IndexAction.Delete(nameof(IndexDocument.Id), x));
await ServiceClient.Indexes.GetClient(ConfigurationReader.SearchIndexName).Documents.IndexAsync(new IndexBatch(indexActions));
}
}
public void Delete(IIndexableId id)
{
var searchParams = new SearchParameters();
searchParams.SearchFields = new List <string>();
searchParams.SearchFields.Add("s_group");
searchParams.Select = new List <string>();
searchParams.Select.Add("s_key");
var resultTask = AzureIndex.AzureIndexClient.Documents.SearchWithHttpMessagesAsync(id.ToString(), searchParams);
resultTask.Wait();
var results = resultTask.Result.Body.Results;
foreach (var result in results)
{
Commit(IndexAction.Delete(result.Document));
}
}
private IndexAction <KeyedDocument> GetSearchIndexAction(
NewPackageRegistration packageRegistration,
IReadOnlyDictionary <NuGetVersion, Package> versionToPackage,
VersionLists versionLists,
SearchFilters searchFilters,
SearchIndexChangeType changeType)
{
if (changeType == SearchIndexChangeType.Delete)
{
return(IndexAction.Delete(_search.Keyed(
packageRegistration.PackageId,
searchFilters)));
}
if (changeType != SearchIndexChangeType.AddFirst)
{
throw new ArgumentException(
$"The only change types supported are {nameof(SearchIndexChangeType.AddFirst)} and " +
$"{nameof(SearchIndexChangeType.Delete)}.",
nameof(changeType));
}
var latestFlags = _search.LatestFlagsOrNull(versionLists, searchFilters);
var package = versionToPackage[latestFlags.LatestVersionInfo.ParsedVersion];
var owners = packageRegistration
.Owners
.OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase)
.ToArray();
VerifyConsistency(packageRegistration.PackageId, package);
return(IndexAction.Upload <KeyedDocument>(_search.FullFromDb(
packageRegistration.PackageId,
searchFilters,
latestFlags.LatestVersionInfo.ListedFullVersions,
latestFlags.IsLatestStable,
latestFlags.IsLatest,
latestFlags.LatestVersionInfo.FullVersion,
package,
owners,
packageRegistration.TotalDownloadCount,
packageRegistration.IsExcludedByDefault)));
}
private IndexAction <KeyedDocument> GetHijackIndexAction(
Context context,
NuGetVersion version,
HijackDocumentChanges changes)
{
IndexAction <KeyedDocument> indexAction;
if (changes.Delete)
{
indexAction = IndexAction.Delete(_hijack.Keyed(
context.PackageId,
version.ToNormalizedString()));
}
else if (!changes.UpdateMetadata)
{
indexAction = IndexAction.Merge <KeyedDocument>(_hijack.LatestFromCatalog(
context.PackageId,
version.ToNormalizedString(),
lastCommitTimestamp: context.LastCommitTimestamp,
lastCommitId: context.LastCommitId,
changes: changes));
}
else
{
var leaf = context.GetLeaf(version);
var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf);
indexAction = IndexAction.MergeOrUpload <KeyedDocument>(_hijack.FullFromCatalog(
normalizedVersion,
changes,
leaf));
}
_logger.LogInformation(
"Hijack index action prepared for {PackageId} {PackageVersion}: {IndexAction} with a {DocumentType} document.",
context.PackageId,
version.ToNormalizedString(),
indexAction.ActionType,
indexAction.Document.GetType().FullName);
return(indexAction);
}
public void RemoveIndex(int topicID, string tenantID)
{
var key = $"{tenantID}-{topicID}";
try
{
var actions = new[]
{
IndexAction.Delete("key", key)
};
var serviceClient = new SearchServiceClient(_config.SearchUrl, new SearchCredentials(_config.SearchKey));
var serviceIndexClient = serviceClient.Indexes.GetClient(IndexName);
var batch = IndexBatch.New(actions);
serviceIndexClient.Documents.Index(batch);
}
catch (Exception exc)
{
_errorLog.Log(exc, ErrorSeverity.Error);
}
}
/// <summary>
/// Deletes the asynchronous.
/// </summary>
/// <typeparam name="T">The type T</typeparam>
/// <param name="indexName">Name of the index.</param>
/// <param name="documents">The documents.</param>
/// <returns>
/// Returns Task of bool
/// </returns>
/// <exception cref="Exception">Failed to delete some documents</exception>
public async Task <bool> DeleteAsync <T>(string indexName, Dictionary <string, string> documents)
where T : class
{
try
{
using (var serviceClient = this.GetAdminSearchServiceClient())
{
var client = serviceClient.Indexes.GetClient(indexName);
var deleteIndexActions = new List <IndexAction>();
documents.ToList().ForEach(d => deleteIndexActions.Add(IndexAction.Delete(d.Value, d.Key)));
var btch = IndexBatch.New(deleteIndexActions);
await client.Documents.IndexAsync(btch).ConfigureAwait(false);
return(true);
}
}
catch (IndexBatchException ex)
{
return(false);
}
}
public async Task Search404NonMergeFailureIsNotApplicable()
{
IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404));
AllIndexActions.Add(new IdAndValue <IndexActions>(
"NuGet.Versioning",
new IndexActions(
search: new List <IndexAction <KeyedDocument> >
{
IndexAction.Delete(new KeyedDocument {
Key = "search-doc"
}),
},
hijack: new List <IndexAction <KeyedDocument> >(),
versionListDataResult: new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
Mock.Of <IAccessCondition>()))));
var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception);
Assert.False(result.Applicable);
}
public async Task BatchUpdateAsync <T>(IEnumerable <T> uploadOrMerge, IEnumerable <T> upload = null, IEnumerable <T> delete = null) where T : WorkDocument
{
var serviceClient = CreateSearchServiceClient();
var indexClient = serviceClient.Indexes.GetClient(_indexName);
var actions = new List <IndexAction <T> >();
if (uploadOrMerge != null)
{
foreach (var item in uploadOrMerge)
{
actions.Add(IndexAction.MergeOrUpload <T>(item));
}
}
if (upload != null)
{
foreach (var item in upload)
{
actions.Add(IndexAction.Upload <T>(item));
}
}
if (delete != null)
{
foreach (var item in delete)
{
actions.Add(IndexAction.Delete <T>(item));
}
}
var batch = IndexBatch.New(actions);
var retryStrategy =
new Incremental(3, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2));
var retryPolicy =
new RetryPolicy <SearchIndexErrorDetectionStrategy>(retryStrategy);
//there is a retry policy for the client search now, we might be able to use that instead
await retryPolicy.ExecuteAsync(async() => await indexClient.Documents.IndexAsync(batch));
}
protected virtual async Task ExecuteBatch <T>(IEnumerable <T> uploadOrMerge, IEnumerable <T> upload = null, IEnumerable <T> delete = null)
where T : class
{
var actions = new List <IndexAction <T> >();
if (uploadOrMerge != null)
{
foreach (var item in uploadOrMerge)
{
actions.Add(IndexAction.MergeOrUpload <T>(item));
}
}
if (upload != null)
{
foreach (var item in upload)
{
actions.Add(IndexAction.Upload <T>(item));
}
}
if (delete != null)
{
foreach (var item in delete)
{
actions.Add(IndexAction.Delete <T>(item));
}
}
var batch = IndexBatch.New(actions);
var retryStrategy =
new IncrementalRetryStrategy(3, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2));
var retryPolicy =
new RetryPolicy <SearchIndexErrorDetectionStrategy>(retryStrategy);
//there is a retry policy for the client search now, we might be able to use that instead
await retryPolicy.ExecuteAsync(async() => await IndexClient.Documents.IndexAsync(batch));
}
public async Task <bool> UpdateDataAsync(IEnumerable <T> items, IEnumerable <string> ids, CancellationToken token)
{
_logger.Info("processing");
if (items == null && ids == null)
{
throw new ArgumentNullException();
}
if (ids == null)
{
return(await UpdateDataAsync(items, token));
}
if (items == null)
{
return(await DeleteDataAsync(ids, token));
}
var actions = items.Select(IndexAction.MergeOrUpload).ToList();
var actionDelete = ids.Select(s => IndexAction.Delete(new T
{
Id = s
}));
actions.AddRange(actionDelete);
if (actions.Count <= 0)
{
return(false);
}
var batch = IndexBatch.New(actions);
var result = await IndexClient.Documents.IndexAsync(batch, cancellationToken : token);
foreach (var errorResult in result.Results.Where(w => !w.Succeeded))
{
_logger.Error($"Failed to process id {errorResult.Key} error {errorResult.ErrorMessage} on index {IndexClient.IndexName} ");
}
return(result.Results.Count > 0);
}
private IReadOnlyList <IndexAction <KeyedDocument> > AddOrUpdatePackage(
PackageRegistration registration,
bool isUpdate)
{
var encodedId = EncodePackageId(registration.PackageId.ToLowerInvariant());
var result = new List <IndexAction <KeyedDocument> >();
for (var i = 0; i < 4; i++)
{
var includePrerelease = (i & 1) != 0;
var includeSemVer2 = (i & 2) != 0;
var searchFilters = (SearchFilters)i;
var documentKey = $"{encodedId}-{searchFilters}";
var filtered = registration.Packages.Where(p => p.Listed);
if (!includePrerelease)
{
filtered = filtered.Where(p => !p.IsPrerelease);
}
if (!includeSemVer2)
{
filtered = filtered.Where(p => p.SemVerLevel != SemVerLevel.SemVer2);
}
var versions = filtered.OrderBy(p => p.Version).ToList();
if (versions.Count == 0)
{
if (isUpdate)
{
var action = IndexAction.Delete(
new KeyedDocument
{
Key = documentKey
});
result.Add(action);
}
continue;
}
var latest = versions.Last();
var dependencies = latest
.Dependencies
.Select(d => d.Id?.ToLowerInvariant())
.Where(d => d != null)
.Distinct()
.ToArray();
var document = new PackageDocument();
document.Key = $"{encodedId}-{searchFilters}";
document.Id = latest.Id;
document.Version = latest.Version.ToFullString();
document.Description = latest.Description;
document.Authors = latest.Authors;
document.IconUrl = latest.IconUrlString;
document.LicenseUrl = latest.LicenseUrlString;
document.ProjectUrl = latest.ProjectUrlString;
document.Published = latest.Published;
document.Summary = latest.Summary;
document.Tags = latest.Tags;
document.Title = latest.Title;
document.TotalDownloads = versions.Sum(p => p.Downloads);
document.DownloadsMagnitude = document.TotalDownloads.ToString().Length;
document.Versions = versions.Select(p => p.Version.ToFullString()).ToArray();
document.VersionDownloads = versions.Select(p => p.Downloads.ToString()).ToArray();
document.Dependencies = dependencies;
document.PackageTypes = latest.PackageTypes.Select(t => t.Name).ToArray();
document.Frameworks = latest.TargetFrameworks.Select(f => f.Moniker.ToLowerInvariant()).ToArray();
document.SearchFilters = searchFilters.ToString();
result.Add(
isUpdate
? IndexAction.MergeOrUpload <KeyedDocument>(document)
: IndexAction.Upload <KeyedDocument>(document));
}
return(result);
}
private static void UploadDocuments(ISearchIndexClient indexClient)
{
var actions =
new IndexAction <Hotel>[]
{
IndexAction.Upload(
new Hotel()
{
HotelId = "1",
BaseRate = 199.0,
Description = "Best hotel in town",
DescriptionFr = "Meilleur hôtel en ville",
HotelName = "Fancy Stay",
Category = "Luxury",
Tags = new[] { "pool", "view", "wifi", "concierge" },
ParkingIncluded = false,
SmokingAllowed = false,
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.Zero),
Rating = 5,
Location = GeographyPoint.Create(47.678581, -122.131577)
}),
IndexAction.Upload(
new Hotel()
{
HotelId = "2",
BaseRate = 79.99,
Description = "Cheapest hotel in town",
DescriptionFr = "Hôtel le moins cher en ville",
HotelName = "Roach Motel",
Category = "Budget",
Tags = new[] { "motel", "budget" },
ParkingIncluded = true,
SmokingAllowed = true,
LastRenovationDate = new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero),
Rating = 1,
Location = GeographyPoint.Create(49.678581, -122.131577)
}),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "3",
BaseRate = 129.99,
Description = "Close to town hall and the river"
}),
IndexAction.Delete(new Hotel()
{
HotelId = "6"
})
};
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
Console.WriteLine("Waiting for documents to be indexed...\n");
Thread.Sleep(2000);
}
private IndexAction <KeyedDocument> GetSearchIndexAction(
Context context,
SearchFilters searchFilters,
SearchIndexChangeType changeType,
string[] owners)
{
var latestFlags = _search.LatestFlagsOrNull(context.VersionLists, searchFilters);
Guard.Assert(
changeType == SearchIndexChangeType.Delete || latestFlags != null,
"Either the search document is being or there is a latest version.");
IndexAction <KeyedDocument> indexAction;
if (changeType == SearchIndexChangeType.Delete)
{
indexAction = IndexAction.Delete(_search.Keyed(
context.PackageId,
searchFilters));
}
else if (changeType == SearchIndexChangeType.UpdateVersionList)
{
if (owners != null)
{
// If we have owner information already fetched on behalf of another search document, send the
// latest owner information as well. This provides two benefits:
//
// 1. This keeps all search documents for a package ID in-sync with regards to their owners
// fields.
//
// 2. This means if an admin is reflowing for the purposes of fixing up owner information, all
// search documents get the benefit instead of having to reflow the latest version of each
// search filter.
//
indexAction = IndexAction.Merge <KeyedDocument>(_search.UpdateVersionListAndOwnersFromCatalog(
context.PackageId,
searchFilters,
lastCommitTimestamp: context.LastCommitTimestamp,
lastCommitId: context.LastCommitId,
versions: latestFlags.LatestVersionInfo.ListedFullVersions,
isLatestStable: latestFlags.IsLatestStable,
isLatest: latestFlags.IsLatest,
owners: owners));
}
else
{
indexAction = IndexAction.Merge <KeyedDocument>(_search.UpdateVersionListFromCatalog(
context.PackageId,
searchFilters,
lastCommitTimestamp: context.LastCommitTimestamp,
lastCommitId: context.LastCommitId,
versions: latestFlags.LatestVersionInfo.ListedFullVersions,
isLatestStable: latestFlags.IsLatestStable,
isLatest: latestFlags.IsLatest));
}
}
else if (IsUpdateLatest(changeType))
{
var leaf = context.GetLeaf(latestFlags.LatestVersionInfo.ParsedVersion);
var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf);
indexAction = IndexAction.MergeOrUpload <KeyedDocument>(_search.UpdateLatestFromCatalog(
searchFilters,
latestFlags.LatestVersionInfo.ListedFullVersions,
latestFlags.IsLatestStable,
latestFlags.IsLatest,
normalizedVersion,
latestFlags.LatestVersionInfo.FullVersion,
leaf,
owners));
}
else
{
throw new NotImplementedException($"The change type '{changeType}' is not supported.");
}
_logger.LogInformation(
"Search index action prepared for {PackageId} {SearchFilters}: {IndexAction} with a {DocumentType} document.",
context.PackageId,
searchFilters,
indexAction.ActionType,
indexAction.Document.GetType().FullName);
return(indexAction);
}
/// <inheritdoc/>
public async Task <bool> ChangeDocumentsInIndexAsync <T>(SortedDictionary <T, IndexActionType> changedDocuments, ILogger logger = null)
where T : class
{
Argument.EnsureNotEmpty(nameof(changedDocuments), changedDocuments);
if (logger == null)
{
logger = NullLogger.Instance;
}
await EnsureSearchIndexAsync <T>(logger);
var index = GetIndex <T>();
var indexActions = new List <IndexAction <T> >();
foreach (var keyValuePair in changedDocuments)
{
IndexAction <T> indexAction = null;
switch (keyValuePair.Value)
{
case IndexActionType.Upload:
indexAction = IndexAction.Upload(keyValuePair.Key);
break;
case IndexActionType.Delete:
indexAction = IndexAction.Delete(keyValuePair.Key);
break;
case IndexActionType.Merge:
indexAction = IndexAction.Merge(keyValuePair.Key);
break;
default:
indexAction = IndexAction.MergeOrUpload(keyValuePair.Key);
break;
}
indexActions.Add(indexAction);
}
var batch = IndexBatch.New(indexActions);
var indexClient = SearchClient.Value.Indexes.GetClient(index);
try
{
var documentIndexResult = await indexClient.Documents.IndexAsync(batch);
return(documentIndexResult.Results != null && documentIndexResult.Results.Count == changedDocuments.Count());
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
logger.Log(TraceEventType.Error, e, null, "Failed to index some of the documents: {0}",
string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
catch (Exception e)
{
logger.Log(TraceEventType.Error, e, null, "Search index failed");
}
return(false);
}
public void CanIndexDynamicDocuments()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var batch = IndexBatch.New(new[]
{
IndexAction.Upload(
new Document()
{
{ "hotelId", "1" },
{ "baseRate", 199.0 },
{ "description", "Best hotel in town" },
{ "descriptionFr", "Meilleur hôtel en ville" },
{ "hotelName", "Fancy Stay" },
{ "category", "Luxury" },
{ "tags", new[] { "pool", "view", "wifi", "concierge" } },
{ "parkingIncluded", false },
{ "smokingAllowed", false },
{ "lastRenovationDate", new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.FromHours(-8)) },
{ "rating", 5 },
{ "location", GeographyPoint.Create(47.678581, -122.131577) }
}),
IndexAction.Upload(
new Document()
{
{ "hotelId", "2" },
{ "baseRate", 79.99 },
{ "description", "Cheapest hotel in town" },
{ "descriptionFr", "Hôtel le moins cher en ville" },
{ "hotelName", "Roach Motel" },
{ "category", "Budget" },
{ "tags", new[] { "motel", "budget" } },
{ "parkingIncluded", true },
{ "smokingAllowed", true },
{ "lastRenovationDate", new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero) }, //aka.ms/sre-codescan/disable
{ "rating", 1 },
{ "location", GeographyPoint.Create(49.678581, -122.131577) }
}),
IndexAction.Merge(
new Document()
{
{ "hotelId", "3" },
{ "baseRate", 279.99 },
{ "description", "Surprisingly expensive" },
{ "lastRenovationDate", null }
}),
IndexAction.Delete(keyName: "hotelId", keyValue: "4"),
IndexAction.MergeOrUpload(
new Document()
{
{ "hotelId", "5" },
{ "baseRate", Double.NaN },
{ "hotelName", null },
{ "tags", new string[0] }
})
});
IndexBatchException e = Assert.Throws <IndexBatchException>(() => client.Documents.Index(batch));
AssertIsPartialFailure(e, "3");
Assert.Equal(5, e.IndexingResults.Count);
AssertIndexActionSucceeded("1", e.IndexingResults[0], 201);
AssertIndexActionSucceeded("2", e.IndexingResults[1], 201);
AssertIndexActionFailed("3", e.IndexingResults[2], "Document not found.", 404);
AssertIndexActionSucceeded("4", e.IndexingResults[3], 200);
AssertIndexActionSucceeded("5", e.IndexingResults[4], 201);
SearchTestUtilities.WaitForIndexing();
Assert.Equal(3L, client.Documents.Count());
});
}
public void CanIndexStaticallyTypedDocuments()
{
Run(() =>
{
SearchIndexClient client = Data.GetSearchIndexClient();
var batch = IndexBatch.New(new[]
{
IndexAction.Upload(
new Hotel()
{
HotelId = "1",
BaseRate = 199.0,
Description = "Best hotel in town",
DescriptionFr = "Meilleur hôtel en ville",
HotelName = "Fancy Stay",
Category = "Luxury",
Tags = new[] { "pool", "view", "wifi", "concierge" },
ParkingIncluded = false,
SmokingAllowed = false,
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.FromHours(-8)),
Rating = 5,
Location = GeographyPoint.Create(47.678581, -122.131577)
}),
IndexAction.Upload(
new Hotel()
{
HotelId = "2",
BaseRate = 79.99,
Description = "Cheapest hotel in town",
DescriptionFr = "Hôtel le moins cher en ville",
HotelName = "Roach Motel",
Category = "Budget",
Tags = new[] { "motel", "budget" },
ParkingIncluded = true,
SmokingAllowed = true,
LastRenovationDate = new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero), //aka.ms/sre-codescan/disable
Rating = 1,
Location = GeographyPoint.Create(49.678581, -122.131577)
}),
IndexAction.Merge(
new Hotel()
{
HotelId = "3",
BaseRate = 279.99,
Description = "Surprisingly expensive",
LastRenovationDate = null
}),
IndexAction.Delete(new Hotel()
{
HotelId = "4"
}),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "5",
BaseRate = Double.NaN,
HotelName = null,
Tags = new string[0]
})
});
IndexBatchException e = Assert.Throws <IndexBatchException>(() => client.Documents.Index(batch));
AssertIsPartialFailure(e, "3");
Assert.Equal(5, e.IndexingResults.Count);
AssertIndexActionSucceeded("1", e.IndexingResults[0], 201);
AssertIndexActionSucceeded("2", e.IndexingResults[1], 201);
AssertIndexActionFailed("3", e.IndexingResults[2], "Document not found.", 404);
AssertIndexActionSucceeded("4", e.IndexingResults[3], 200);
AssertIndexActionSucceeded("5", e.IndexingResults[4], 201);
SearchTestUtilities.WaitForIndexing();
Assert.Equal(3, client.Documents.Count());
});
}
public async Task Process(ContactMessage msg)
{
var client = new SearchServiceClient(_options.ServiceName, new SearchCredentials(_options.AdminApiKey));
// try to create index once
if (!_indexExists)
{
lock (_syncRoot)
{
if (!_indexExists)
{
if (!client.Indexes.ExistsAsync(_options.IndexName).Result)
{
var definition = new Index()
{
Name = _options.IndexName,
Fields = FieldBuilder.BuildForType <Contact>(new DefaultContractResolver()
{
NamingStrategy = new JsonLowercaseNamingStrategy()
})
};
client.Indexes.CreateAsync(definition).Wait();
}
_indexExists = true;
}
}
}
var action = default(IndexAction <Contact>);
if (msg.IsAddedMessage())
{
action = IndexAction.Upload(_mapper.Value.Map <Contact>(msg));
}
else if (msg.IsChangedMessage())
{
action = IndexAction.MergeOrUpload(_mapper.Value.Map <Contact>(msg));
}
else if (msg.IsDeletedMessage())
{
action = IndexAction.Delete(_mapper.Value.Map <Contact>(msg));
}
else
{
throw new InvalidOperationException($"Meesage type {msg.EventType} not supported.");
}
try
{
var indexClient = client.Indexes.GetClient(_options.IndexName);
indexClient.DeserializationSettings.ContractResolver = new DefaultContractResolver()
{
NamingStrategy = new JsonLowercaseNamingStrategy()
};
await indexClient.Documents.IndexAsync(IndexBatch.New(new[] { action }));
}
catch (IndexBatchException)
{
}
await Task.Delay(0);
}
/// <summary>
/// Removes old records from index
/// </summary>
/// <param name="oldRecords">The old records which should be deleted from the index.</param>
public async Task <int> RemoveFromIndexAsync(IEnumerable <EmployerSearchModel> oldRecords)
{
if (Disabled)
{
throw new Exception($"{nameof(AzureEmployerSearchRepository)} is disabled");
}
if (oldRecords == null || !oldRecords.Any())
{
throw new ArgumentNullException(nameof(oldRecords), "You must supply at least one record to index");
}
//Set the records to add or update
var actions = oldRecords.Select(r => IndexAction.Delete(_autoMapper.Map <AzureEmployerSearchModel>(r))).ToList();
var batches = new ConcurrentBag <IndexBatch <AzureEmployerSearchModel> >();
while (actions.Any())
{
var batchSize = actions.Count > 1000 ? 1000 : actions.Count;
var batch = IndexBatch.New(actions.Take(batchSize).ToList());
batches.Add(batch);
actions.RemoveRange(0, batchSize);
}
var deleteCount = 0;
var exceptions = new ConcurrentBag <Exception>();
var indexClient = await _indexClient.Value;
await batches.WaitForAllAsync(
async batch =>
{
var retries = 0;
retry:
try
{
await indexClient.Documents.IndexAsync(batch);
Interlocked.Add(ref deleteCount, batch.Actions.Count());
}
catch (IndexBatchException e)
{
if (retries < 30)
{
retries++;
Thread.Sleep(1000);
goto retry;
}
exceptions.Add(e);
}
catch (Exception ex)
{
exceptions.Add(ex);
}
});
if (exceptions.Count > 0)
{
throw new AggregateException(exceptions);
}
return(deleteCount);
}
public IActionResult Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = "update")]
HttpRequest req, ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
var actions =
new IndexAction <Hotel>[]
{
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "1",
BaseRate = 199.0,
Description = "Best hotel in town",
DescriptionFr = "Meilleur hôtel en ville",
HotelName = "Fancy Stay",
Category = "Luxury",
Tags = new[] { "pool", "view", "wifi", "concierge" },
ParkingIncluded = false,
SmokingAllowed = false,
LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.Zero),
Rating = 5,
Location = GeographyPoint.Create(47.678581, -122.131577)
}),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "2",
BaseRate = 79.99,
Description = "Cheapest hotel in town",
DescriptionFr = "Hôtel le moins cher en ville",
HotelName = "Roach Motel",
Category = "Budget",
Tags = new[] { "motel", "budget" },
ParkingIncluded = true,
SmokingAllowed = true,
LastRenovationDate = new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero),
Rating = 1,
Location = GeographyPoint.Create(49.678581, -122.131577)
}),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "3",
BaseRate = 129.99,
Description = "Close to town hall and the river"
}),
IndexAction.Delete(new Hotel()
{
HotelId = "6"
})
};
var batch = IndexBatch.New(actions);
var indexClient = _searchServiceClient.Indexes.GetClient("hotels");
indexClient.Documents.Index(batch);
return(new OkObjectResult("ok, updated"));
}
// Upload documents as a batch.
private static void UploadDocuments(ISearchIndexClient indexClient)
{
var actions = new IndexAction <Hotel>[]
{
IndexAction.Upload(
new Hotel()
{
HotelId = "1",
HotelName = "Secret Point Motel",
Description = "The hotel is ideally located on the main commercial artery of the city in the heart of New York. A few minutes away is Time's Square and the historic centre of the city, as well as other places of interest that make New York one of America's most attractive and cosmopolitan cities.",
DescriptionFr = "L'hôtel est idéalement situé sur la principale artère commerciale de la ville en plein cœur de New York. A quelques minutes se trouve la place du temps et le centre historique de la ville, ainsi que d'autres lieux d'intérêt qui font de New York l'une des villes les plus attractives et cosmopolites de l'Amérique.",
Category = "Boutique",
Tags = new[] { "pool", "air conditioning", "concierge" },
ParkingIncluded = false,
LastRenovationDate = new DateTimeOffset(1970, 1, 18, 0, 0, 0, TimeSpan.Zero),
Rating = 3.6,
Location = GeographyPoint.Create(40.760586, -73.975403),
Address = new Address()
{
StreetAddress = "677 5th Ave",
City = "New York",
StateProvince = "NY",
PostalCode = "10022",
Country = "USA"
},
Rooms = new Room[]
{
new Room()
{
Description = "Budget Room, 1 Queen Bed (Cityside)",
DescriptionFr = "Chambre Économique, 1 grand lit (côté ville)",
Type = "Budget Room",
BaseRate = 96.99,
BedOptions = "1 Queen Bed",
SleepsCount = 2,
SmokingAllowed = true,
Tags = new[] { "vcr/dvd" }
},
new Room()
{
Description = "Budget Room, 1 King Bed (Mountain View)",
DescriptionFr = "Chambre Économique, 1 très grand lit (Mountain View)",
Type = "Budget Room",
BaseRate = 80.99,
BedOptions = "1 King Bed",
SleepsCount = 2,
SmokingAllowed = true,
Tags = new[] { "vcr/dvd", "jacuzzi tub" }
},
new Room()
{
Description = "Deluxe Room, 2 Double Beds (City View)",
DescriptionFr = "Chambre Deluxe, 2 lits doubles (vue ville)",
Type = "Deluxe Room",
BaseRate = 150.99,
BedOptions = "2 Double Beds",
SleepsCount = 2,
SmokingAllowed = false,
Tags = new[] { "suite", "bathroom shower", "coffee maker" }
}
}
}
),
IndexAction.Upload(
new Hotel()
{
HotelId = "2",
HotelName = "Twin Dome Motel",
Description = "The hotel is situated in a nineteenth century plaza, which has been expanded and renovated to the highest architectural standards to create a modern, functional and first-class hotel in which art and unique historical elements coexist with the most modern comforts.",
DescriptionFr = "L'hôtel est situé dans une place du XIXe siècle, qui a été agrandie et rénovée aux plus hautes normes architecturales pour créer un hôtel moderne, fonctionnel et de première classe dans lequel l'art et les éléments historiques uniques coexistent avec le confort le plus moderne.",
Category = "Boutique",
Tags = new[] { "pool", "free wifi", "concierge" },
ParkingIncluded = false,
LastRenovationDate = new DateTimeOffset(1979, 2, 18, 0, 0, 0, TimeSpan.Zero),
Rating = 3.60,
Location = GeographyPoint.Create(27.384417, -82.452843),
Address = new Address()
{
StreetAddress = "140 University Town Center Dr",
City = "Sarasota",
StateProvince = "FL",
PostalCode = "34243",
Country = "USA"
},
Rooms = new Room[]
{
new Room()
{
Description = "Suite, 2 Double Beds (Mountain View)",
DescriptionFr = "Suite, 2 lits doubles (vue sur la montagne)",
Type = "Suite",
BaseRate = 250.99,
BedOptions = "2 Double Beds",
SleepsCount = 2,
SmokingAllowed = false,
Tags = new[] { "Room Tags" }
},
new Room()
{
Description = "Standard Room, 1 Queen Bed (City View)",
DescriptionFr = "Chambre Standard, 1 grand lit (vue ville)",
Type = "Standard Room",
BaseRate = 121.99,
BedOptions = "1 Queen Bed",
SleepsCount = 2,
SmokingAllowed = false,
Tags = new[] { "jacuzzi tub" }
},
new Room()
{
Description = "Budget Room, 1 King Bed (Waterfront View)",
DescriptionFr = "Chambre Économique, 1 très grand lit (vue sur le front de mer)",
Type = "Budget Room",
BaseRate = 88.99,
BedOptions = "1 King Bed",
SleepsCount = 2,
SmokingAllowed = false,
Tags = new[] { "suite", "tv", "jacuzzi tub" }
}
}
}
),
IndexAction.MergeOrUpload(
new Hotel()
{
HotelId = "3",
HotelName = "Triple Landscape Hotel",
Description = "The Hotel stands out for its gastronomic excellence under the management of William Dough, who advises on and oversees all of the Hotel’s restaurant services.",
DescriptionFr = "L'hôtel est situé dans une place du XIXe siècle, qui a été agrandie et rénovée aux plus hautes normes architecturales pour créer un hôtel moderne, fonctionnel et de première classe dans lequel l'art et les éléments historiques uniques coexistent avec le confort le plus moderne.",
Category = "Resort and Spa",
Tags = new[] { "air conditioning", "bar", "continental breakfast" },
ParkingIncluded = true,
LastRenovationDate = new DateTimeOffset(2015, 9, 20, 0, 0, 0, TimeSpan.Zero),
Rating = 4.80,
Location = GeographyPoint.Create(33.84643, -84.362465),
Address = new Address()
{
StreetAddress = "3393 Peachtree Rd",
City = "Atlanta",
StateProvince = "GA",
PostalCode = "30326",
Country = "USA"
},
Rooms = new Room[]
{
new Room()
{
Description = "Standard Room, 2 Queen Beds (Amenities)",
DescriptionFr = "Chambre Standard, 2 grands lits (Services)",
Type = "Standard Room",
BaseRate = 101.99,
BedOptions = "2 Queen Beds",
SleepsCount = 4,
SmokingAllowed = true,
Tags = new[] { "vcr/dvd", "vcr/dvd" }
},
new Room()
{
Description = "Standard Room, 2 Double Beds (Waterfront View)",
DescriptionFr = "Chambre Standard, 2 lits doubles (vue sur le front de mer)",
Type = "Standard Room",
BaseRate = 106.99,
BedOptions = "2 Double Beds",
SleepsCount = 2,
SmokingAllowed = true,
Tags = new[] { "coffee maker" }
},
new Room()
{
Description = "Deluxe Room, 2 Double Beds (Cityside)",
DescriptionFr = "Chambre Deluxe, 2 lits doubles (Cityside)",
Type = "Budget Room",
BaseRate = 180.99,
BedOptions = "2 Double Beds",
SleepsCount = 2,
SmokingAllowed = true,
Tags = new[] { "suite" }
}
}
}
),
IndexAction.Delete(new Hotel()
{
HotelId = "6"
})
};
var batch = IndexBatch.New(actions);
try
{
indexClient.Documents.Index(batch);
}
catch (IndexBatchException e)
{
// Sometimes when your Search service is under load, indexing will fail for some of the documents in
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
Console.WriteLine(
"Failed to index some of the documents: {0}",
String.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
}
Console.WriteLine("Waiting for documents to be indexed...\n");
Thread.Sleep(2000);
}