public async Task UpdateAsync <T>(T item, IGraphRequestContext graphRequestContext) where T : class { var client = Get <T>(graphRequestContext); var idx = IndexAction.Merge(item); await client.Documents.IndexAsync(new IndexBatch <T>( new List <IndexAction <T> > { idx })); }
[InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size. public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes) { var changeCount = 30; var expectedPushes = fullPushes + partialPushes; Config.AzureSearchBatchSize = 10; IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> >( Enumerable .Range(0, documentsPerId) .Select(x => IndexAction.Merge(new KeyedDocument()))), new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), new Mock <IAccessCondition>().Object)); AddChanges(changeCount); await Target.ExecuteAsync(); VerifyCompletedTelemetry(JobOutcome.Success); VerifyAllIdsAreProcessed(changeCount); IndexActionBuilder.Verify( x => x.UpdateAsync( It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Exactly(changeCount)); BatchPusher.Verify( x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()), Times.Exactly(changeCount)); BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes)); BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); SystemTime.Verify(x => x.Delay(It.IsAny <TimeSpan>()), Times.Exactly(expectedPushes - 1)); DownloadDataClient.Verify( x => x.ReplaceLatestIndexedAsync( NewDownloadData, It.Is <IAccessCondition>(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)), Times.Once); Assert.Equal( fullPushes, FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize)); Assert.Equal( partialPushes, FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize)); Assert.Empty(CurrentBatch); }
static void Main(string[] args) { bool clear = false; SearchServiceClient client = new SearchServiceClient(ConfigurationManager.AppSettings["AzureSearchName"], new SearchCredentials(ConfigurationManager.AppSettings["AzureSearchKey"])); SearchIndexClient indexClient = client.Indexes.GetClient("restaurant"); AzureMLRecommendations recos = new AzureMLRecommendations(); recos.Init(ConfigurationManager.AppSettings["RecoUser"], ConfigurationManager.AppSettings["RecoKey"]); string modelId = ConfigurationManager.AppSettings["RecoModelId"]; List <IndexAction> actions = new List <IndexAction>(); foreach (int id in GetCurrentRestaurantIds()) { List <string> recommendedIds = new List <string>(); if (!clear) { var r = recos.GetRecommendation(modelId, new List <string> { id.ToString() }, 3); recommendedIds.AddRange(r.Select(i => i.Id)); } actions.Add(IndexAction.Merge(new Document { { "RestaurantId", id.ToString() }, { "RecommendedIds", recommendedIds } })); } // Assume < 1000 actions, otherwise we'd need to split it in 1000-actions batches try { DocumentIndexResult indexResult = indexClient.Documents.Index(new IndexBatch(actions)); int succeeded = indexResult.Results.Where(r => r.Succeeded).Count(); Console.WriteLine($"Indexed completed. Items: {indexResult.Results.Count}, Succeeded: {succeeded}"); } catch (IndexBatchException ex) { // Sometimes when your Search service is under load, indexing will fail for some of the documents in // the batch. Depending on your application, you can take compensating actions like delaying and // retrying. For this simple demo, we just log the failed document keys and continue. Console.WriteLine( "Failed to index some of the documents: {0}", string.Join(", ", ex.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key))); } }
private IndexAction <KeyedDocument> GetHijackIndexAction( Context context, NuGetVersion version, HijackDocumentChanges changes) { IndexAction <KeyedDocument> indexAction; if (changes.Delete) { indexAction = IndexAction.Delete(_hijack.Keyed( context.PackageId, version.ToNormalizedString())); } else if (!changes.UpdateMetadata) { indexAction = IndexAction.Merge <KeyedDocument>(_hijack.LatestFromCatalog( context.PackageId, version.ToNormalizedString(), lastCommitTimestamp: context.LastCommitTimestamp, lastCommitId: context.LastCommitId, changes: changes)); } else { var leaf = context.GetLeaf(version); var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf); indexAction = IndexAction.MergeOrUpload <KeyedDocument>(_hijack.FullFromCatalog( normalizedVersion, changes, leaf)); } _logger.LogInformation( "Hijack index action prepared for {PackageId} {PackageVersion}: {IndexAction} with a {DocumentType} document.", context.PackageId, version.ToNormalizedString(), indexAction.ActionType, indexAction.Document.GetType().FullName); return(indexAction); }
public async Task Hijack404FailureIsNotApplicable() { IndexingResults.Add(new IndexingResult(key: "hijack-doc", statusCode: 404)); AllIndexActions.Add(new IdAndValue <IndexActions>( "NuGet.Versioning", new IndexActions( search: new List <IndexAction <KeyedDocument> >(), hijack: new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument { Key = "hijack-doc" }), }, versionListDataResult: new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), Mock.Of <IAccessCondition>())))); var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); Assert.False(result.Applicable); }
private IndexAction <KeyedDocument> GetSearchIndexAction( Context context, SearchFilters searchFilters, SearchIndexChangeType changeType, string[] owners) { var latestFlags = _search.LatestFlagsOrNull(context.VersionLists, searchFilters); Guard.Assert( changeType == SearchIndexChangeType.Delete || latestFlags != null, "Either the search document is being or there is a latest version."); IndexAction <KeyedDocument> indexAction; if (changeType == SearchIndexChangeType.Delete) { indexAction = IndexAction.Delete(_search.Keyed( context.PackageId, searchFilters)); } else if (changeType == SearchIndexChangeType.UpdateVersionList) { if (owners != null) { // If we have owner information already fetched on behalf of another search document, send the // latest owner information as well. This provides two benefits: // // 1. This keeps all search documents for a package ID in-sync with regards to their owners // fields. // // 2. This means if an admin is reflowing for the purposes of fixing up owner information, all // search documents get the benefit instead of having to reflow the latest version of each // search filter. // indexAction = IndexAction.Merge <KeyedDocument>(_search.UpdateVersionListAndOwnersFromCatalog( context.PackageId, searchFilters, lastCommitTimestamp: context.LastCommitTimestamp, lastCommitId: context.LastCommitId, versions: latestFlags.LatestVersionInfo.ListedFullVersions, isLatestStable: latestFlags.IsLatestStable, isLatest: latestFlags.IsLatest, owners: owners)); } else { indexAction = IndexAction.Merge <KeyedDocument>(_search.UpdateVersionListFromCatalog( context.PackageId, searchFilters, lastCommitTimestamp: context.LastCommitTimestamp, lastCommitId: context.LastCommitId, versions: latestFlags.LatestVersionInfo.ListedFullVersions, isLatestStable: latestFlags.IsLatestStable, isLatest: latestFlags.IsLatest)); } } else if (IsUpdateLatest(changeType)) { var leaf = context.GetLeaf(latestFlags.LatestVersionInfo.ParsedVersion); var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf); indexAction = IndexAction.MergeOrUpload <KeyedDocument>(_search.UpdateLatestFromCatalog( searchFilters, latestFlags.LatestVersionInfo.ListedFullVersions, latestFlags.IsLatestStable, latestFlags.IsLatest, normalizedVersion, latestFlags.LatestVersionInfo.FullVersion, leaf, owners)); } else { throw new NotImplementedException($"The change type '{changeType}' is not supported."); } _logger.LogInformation( "Search index action prepared for {PackageId} {SearchFilters}: {IndexAction} with a {DocumentType} document.", context.PackageId, searchFilters, indexAction.ActionType, indexAction.Document.GetType().FullName); return(indexAction); }
public async Task Search404MergeFailureIsApplicable() { ItemList.Add(new CatalogCommitItem( new Uri("https://example/catalog/0.json"), "commit-id-a", new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), new string[0], new[] { Schema.DataTypes.PackageDetails }, new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0")))); ItemList.Add(new CatalogCommitItem( new Uri("https://example/catalog/1.json"), "commit-id-a", new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), new string[0], new[] { Schema.DataTypes.PackageDetails }, new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("0.9.0-beta.1")))); IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404)); AllIndexActions.Add(new IdAndValue <IndexActions>( "NuGet.Versioning", new IndexActions( search: new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument { Key = "search-doc" }), }, hijack: new List <IndexAction <KeyedDocument> >(), versionListDataResult: new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), Mock.Of <IAccessCondition>())))); VersionListClient .Setup(x => x.ReadAsync(It.IsAny <string>())) .ReturnsAsync(() => new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { "1.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, }), Mock.Of <IAccessCondition>())); var leaf = new PackageDetailsCatalogLeaf { Url = "https://example/catalog/2.json", CommitId = "commit-id", CommitTimestamp = new DateTimeOffset(2020, 3, 17, 12, 5, 0, TimeSpan.Zero), Type = CatalogLeafType.PackageDetails, }; LeafFetcher .Setup(x => x.GetLatestLeavesAsync( It.IsAny <string>(), It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >())) .ReturnsAsync(() => new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse("1.0.0"), leaf }, })); var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); Assert.True(result.Applicable, "The fix up should be applicable."); Assert.Equal(3, result.ItemList.Count); Assert.Empty(ItemList.Except(result.ItemList)); var addedItem = Assert.Single(result.ItemList.Except(ItemList)); Assert.Equal(leaf.Url, addedItem.Uri.AbsoluteUri); Assert.Equal(leaf.CommitId, addedItem.CommitId); Assert.Equal(leaf.CommitTimestamp, addedItem.CommitTimeStamp); Assert.Empty(addedItem.Types); Assert.Equal(Schema.DataTypes.PackageDetails, Assert.Single(addedItem.TypeUris)); Assert.Equal(new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0")), addedItem.PackageIdentity); Assert.True(addedItem.IsPackageDetails, "The generated item should be a package details item."); Assert.False(addedItem.IsPackageDelete, "The generated item should not be a package delete item."); }
public void CanIndexDynamicDocuments() { Run(() => { SearchIndexClient client = Data.GetSearchIndexClient(); var batch = IndexBatch.New(new[] { IndexAction.Upload( new Document() { { "hotelId", "1" }, { "baseRate", 199.0 }, { "description", "Best hotel in town" }, { "descriptionFr", "Meilleur hôtel en ville" }, { "hotelName", "Fancy Stay" }, { "category", "Luxury" }, { "tags", new[] { "pool", "view", "wifi", "concierge" } }, { "parkingIncluded", false }, { "smokingAllowed", false }, { "lastRenovationDate", new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.FromHours(-8)) }, { "rating", 5 }, { "location", GeographyPoint.Create(47.678581, -122.131577) } }), IndexAction.Upload( new Document() { { "hotelId", "2" }, { "baseRate", 79.99 }, { "description", "Cheapest hotel in town" }, { "descriptionFr", "Hôtel le moins cher en ville" }, { "hotelName", "Roach Motel" }, { "category", "Budget" }, { "tags", new[] { "motel", "budget" } }, { "parkingIncluded", true }, { "smokingAllowed", true }, { "lastRenovationDate", new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero) }, //aka.ms/sre-codescan/disable { "rating", 1 }, { "location", GeographyPoint.Create(49.678581, -122.131577) } }), IndexAction.Merge( new Document() { { "hotelId", "3" }, { "baseRate", 279.99 }, { "description", "Surprisingly expensive" }, { "lastRenovationDate", null } }), IndexAction.Delete(keyName: "hotelId", keyValue: "4"), IndexAction.MergeOrUpload( new Document() { { "hotelId", "5" }, { "baseRate", Double.NaN }, { "hotelName", null }, { "tags", new string[0] } }) }); IndexBatchException e = Assert.Throws <IndexBatchException>(() => client.Documents.Index(batch)); AssertIsPartialFailure(e, "3"); Assert.Equal(5, e.IndexingResults.Count); AssertIndexActionSucceeded("1", e.IndexingResults[0], 201); AssertIndexActionSucceeded("2", e.IndexingResults[1], 201); AssertIndexActionFailed("3", e.IndexingResults[2], "Document not found.", 404); AssertIndexActionSucceeded("4", e.IndexingResults[3], 200); AssertIndexActionSucceeded("5", e.IndexingResults[4], 201); SearchTestUtilities.WaitForIndexing(); Assert.Equal(3L, client.Documents.Count()); }); }
public void CanIndexStaticallyTypedDocuments() { Run(() => { SearchIndexClient client = Data.GetSearchIndexClient(); var batch = IndexBatch.New(new[] { IndexAction.Upload( new Hotel() { HotelId = "1", BaseRate = 199.0, Description = "Best hotel in town", DescriptionFr = "Meilleur hôtel en ville", HotelName = "Fancy Stay", Category = "Luxury", Tags = new[] { "pool", "view", "wifi", "concierge" }, ParkingIncluded = false, SmokingAllowed = false, LastRenovationDate = new DateTimeOffset(2010, 6, 27, 0, 0, 0, TimeSpan.FromHours(-8)), Rating = 5, Location = GeographyPoint.Create(47.678581, -122.131577) }), IndexAction.Upload( new Hotel() { HotelId = "2", BaseRate = 79.99, Description = "Cheapest hotel in town", DescriptionFr = "Hôtel le moins cher en ville", HotelName = "Roach Motel", Category = "Budget", Tags = new[] { "motel", "budget" }, ParkingIncluded = true, SmokingAllowed = true, LastRenovationDate = new DateTimeOffset(1982, 4, 28, 0, 0, 0, TimeSpan.Zero), //aka.ms/sre-codescan/disable Rating = 1, Location = GeographyPoint.Create(49.678581, -122.131577) }), IndexAction.Merge( new Hotel() { HotelId = "3", BaseRate = 279.99, Description = "Surprisingly expensive", LastRenovationDate = null }), IndexAction.Delete(new Hotel() { HotelId = "4" }), IndexAction.MergeOrUpload( new Hotel() { HotelId = "5", BaseRate = Double.NaN, HotelName = null, Tags = new string[0] }) }); IndexBatchException e = Assert.Throws <IndexBatchException>(() => client.Documents.Index(batch)); AssertIsPartialFailure(e, "3"); Assert.Equal(5, e.IndexingResults.Count); AssertIndexActionSucceeded("1", e.IndexingResults[0], 201); AssertIndexActionSucceeded("2", e.IndexingResults[1], 201); AssertIndexActionFailed("3", e.IndexingResults[2], "Document not found.", 404); AssertIndexActionSucceeded("4", e.IndexingResults[3], 200); AssertIndexActionSucceeded("5", e.IndexingResults[4], 201); SearchTestUtilities.WaitForIndexing(); Assert.Equal(3, client.Documents.Count()); }); }
public static async Task CleanExpiredRecordsAsync( SearchIndexClient searchIndexClient, EkProductSourceEnum productSource, long lastUpdateUtcTimestamp, PriceListSynchronizationStatistics statistics, ILogger logger) { const int BatchSize = 1_000; string lastProcessedKey = null; while (true) { var filterBuilder = new StringBuilder(); filterBuilder.Append($"source eq {(int)productSource} and updatedOnUtcTimestamp ne {lastUpdateUtcTimestamp}"); if (lastProcessedKey != null) { filterBuilder.Append($" and key gt '{lastProcessedKey}'"); } var searchParameters = new SearchParameters() { Top = BatchSize, Filter = filterBuilder.ToString(), Select = new List <string>() { "key" }, OrderBy = new List <string>() { "key asc", }, }; var searchResult = await searchIndexClient.Documents.SearchAsync(null, searchParameters); var documents = searchResult.Results .Select(x => x.Document) .ToArray(); if (documents.Length > 0) { // remove records var indexBatch = IndexBatch.New(documents.Select(x => IndexAction.Merge(x))); try { await searchIndexClient.Documents.IndexAsync(indexBatch); } catch (IndexBatchException ex) { var reasons = string.Join(", ", ex.IndexingResults .Where(x => !x.Succeeded) .Select(x => $"{x.Key}-{x.ErrorMessage}")); var message = $"Failed to index products: {reasons}."; throw new InvalidOperationException(message); } // use last processed key as additional filter since indexing takes some time lastProcessedKey = (string)documents.Last()["key"]; logger.LogInformation(LoggingEvents.Synchronization, $"Deleted: {documents.Length}."); statistics.Removed += documents.Length; } else { logger.LogInformation(LoggingEvents.Synchronization, "No expired records."); break; } } }
public async Task SearchForAbsentThumbnailsAsync() { var totalProcessed = 0; var totalThumbnailFound = 0; try { const int BatchSize = 10; using (var indexClient = _ekSearchManagementClient.CreateSearchIndexClient()) { indexClient.IndexName = _ekSearchManagementClient.ProductsIndexName; string lastProcessedKey = null; while (true) { var filterBuilder = new StringBuilder(); filterBuilder.Append("isThumbnailSearchProvided eq null and thumbnailUrl eq null"); if (lastProcessedKey != null) { filterBuilder.Append($" and key gt '{lastProcessedKey}'"); } var searchParameters = new SearchParameters() { Top = BatchSize, Filter = filterBuilder.ToString(), OrderBy = new List <string>() { "key asc", }, }; var searchResult = await indexClient.Documents.SearchAsync <IndexProduct>(null, searchParameters); var records = searchResult.Results .Select(x => x.Document) .ToArray(); if (records.Length > 0) { // search for thumbnails var searchTasks = records.Select(record => Task.Run(async() => { var document = new Document() { ["key"] = record.Key, }; var fileInfo = await GetPartNumberThumbnailAsync( brandName: record.BrandName, partNumber: record.PartNumber); if (fileInfo?.Url != null) { document["thumbnailUrl"] = fileInfo.Url; Interlocked.Increment(ref totalThumbnailFound); } document["isThumbnailSearchProvided"] = true; Interlocked.Increment(ref totalProcessed); return(document); })); var documents = await Task.WhenAll(searchTasks); // index flags and thumbnail urls if found var indexBatch = IndexBatch.New(documents.Select(x => IndexAction.Merge(x))); try { await indexClient.Documents.IndexAsync(indexBatch); } catch (IndexBatchException ex) { var reasons = string.Join(", ", ex.IndexingResults .Where(x => !x.Succeeded) .Select(x => $"{x.Key}-{x.ErrorMessage}")); var message = $"Failed to index products: {reasons}."; throw new InvalidOperationException(message); } // use last processed key as additional filter since indexing takes some time lastProcessedKey = records.Last().Key; var foundPercentage = (decimal)totalThumbnailFound / totalProcessed; _logger.LogInformation($"Processed: {totalProcessed}, found: {totalThumbnailFound}, {foundPercentage:P}."); } else { _logger.LogInformation("No more records."); break; } } } } catch (Exception ex) { _logger.LogError(LoggingEvents.UnhandledException, ex, null); await _notificationManager.SendWorkerMessageAsync($"Search for absent thumbnails failed, processed: {totalProcessed}, found: {totalThumbnailFound}, {ex.Message}"); throw; } await _notificationManager.SendWorkerMessageAsync($"Search for absent thumbnails succeed, processed: {totalProcessed}, found: {totalThumbnailFound}."); }
public Facts(ITestOutputHelper output) { DatabaseOwnerFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); OwnerDataClient = new Mock <IOwnerDataClient>(); OwnerSetComparer = new Mock <IDataSetComparer>(); SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>(); SearchIndexActionBuilder = new Mock <ISearchIndexActionBuilder>(); Pusher = new Mock <IBatchPusher>(); Options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >(); TelemetryService = new Mock <IAzureSearchTelemetryService>(); Logger = output.GetLogger <UpdateOwnersCommand>(); Configuration = new AzureSearchJobConfiguration { MaxConcurrentBatches = 1, }; DatabaseResult = new SortedDictionary <string, SortedSet <string> >(); StorageResult = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >( new SortedDictionary <string, SortedSet <string> >(), new Mock <IAccessCondition>().Object); Changes = new SortedDictionary <string, string[]>(); IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), new Mock <IAccessCondition>().Object)); Pusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); Options .Setup(x => x.Value) .Returns(() => Configuration); DatabaseOwnerFetcher .Setup(x => x.GetPackageIdToOwnersAsync()) .ReturnsAsync(() => DatabaseResult); OwnerDataClient .Setup(x => x.ReadLatestIndexedAsync()) .ReturnsAsync(() => StorageResult); OwnerSetComparer .Setup(x => x.CompareOwners( It.IsAny <SortedDictionary <string, SortedSet <string> > >(), It.IsAny <SortedDictionary <string, SortedSet <string> > >())) .Returns(() => Changes); SearchIndexActionBuilder .Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >())) .ReturnsAsync(() => IndexActions); Target = new UpdateOwnersCommand( DatabaseOwnerFetcher.Object, OwnerDataClient.Object, OwnerSetComparer.Object, SearchDocumentBuilder.Object, SearchIndexActionBuilder.Object, () => Pusher.Object, Options.Object, TelemetryService.Object, Logger); }
public async Task <IndexActions> UpdateAsync(string packageId, Func <SearchFilters, KeyedDocument> buildDocument) { var versionListDataResult = await _versionListDataClient.ReadAsync(packageId); var versionLists = new VersionLists(versionListDataResult.Result); /// Update all of the search documents that exist for this package ID with the provided document builder. /// Here are some examples of different search filter combinations that could occur. /// /// Example #1: 1.0.0 (listed) /// /// A stable SemVer 1.0.0 package matches all search filters, so one index action will be produced for /// each search document. That is four in total. All of these search documents have the same latest /// version: 1.0.0. /// /// Example #2: 1.0.0 (unlisted), 2.0.0 (unlisted) /// /// There are no search documents at all in this case since there is no listed version. No index actions /// are produced in this case. /// /// Example #3: 1.0.0-beta (listed), 2.0.0-beta.1 (listed) /// /// All of the versions are prerelease so there are no search documents for "stable" search filters. There /// two search documents to be updated, one for <see cref="SearchFilters.IncludePrerelease"/> and one for /// <see cref="SearchFilters.IncludePrereleaseAndSemVer2"/>. The latest version for each of these two /// documents is different. var search = new List <IndexAction <KeyedDocument> >(); var searchFilters = new List <SearchFilters>(); foreach (var searchFilter in DocumentUtilities.AllSearchFilters) { // Determine if there is a document for this ID and search filter. if (versionLists.GetLatestVersionInfoOrNull(searchFilter) == null) { continue; } var document = buildDocument(searchFilter); var indexAction = IndexAction.Merge(document); search.Add(indexAction); searchFilters.Add(searchFilter); } _logger.LogInformation( "Package ID {PackageId} has {Count} search document changes for search filters: {SearchFilters}", packageId, searchFilters.Count, searchFilters); // No changes are made to the hijack index. var hijack = new List <IndexAction <KeyedDocument> >(); // We never make any change to the version list but still want to push it back to storage. This will give // us an etag mismatch if the version list has changed. This is good because if the version list has // changed it's possible there is another search document that we have to update. If we didn't do this, // then a race condition could occur where one of the search documents for an ID would not get an update. var newVersionListDataResult = versionListDataResult; return(new IndexActions( search, hijack, newVersionListDataResult)); }
/// <inheritdoc/> public async Task <bool> ChangeDocumentsInIndexAsync <T>(SortedDictionary <T, IndexActionType> changedDocuments, ILogger logger = null) where T : class { Argument.EnsureNotEmpty(nameof(changedDocuments), changedDocuments); if (logger == null) { logger = NullLogger.Instance; } await EnsureSearchIndexAsync <T>(logger); var index = GetIndex <T>(); var indexActions = new List <IndexAction <T> >(); foreach (var keyValuePair in changedDocuments) { IndexAction <T> indexAction = null; switch (keyValuePair.Value) { case IndexActionType.Upload: indexAction = IndexAction.Upload(keyValuePair.Key); break; case IndexActionType.Delete: indexAction = IndexAction.Delete(keyValuePair.Key); break; case IndexActionType.Merge: indexAction = IndexAction.Merge(keyValuePair.Key); break; default: indexAction = IndexAction.MergeOrUpload(keyValuePair.Key); break; } indexActions.Add(indexAction); } var batch = IndexBatch.New(indexActions); var indexClient = SearchClient.Value.Indexes.GetClient(index); try { var documentIndexResult = await indexClient.Documents.IndexAsync(batch); return(documentIndexResult.Results != null && documentIndexResult.Results.Count == changedDocuments.Count()); } catch (IndexBatchException e) { // Sometimes when your Search service is under load, indexing will fail for some of the documents in // the batch. Depending on your application, you can take compensating actions like delaying and // retrying. For this simple demo, we just log the failed document keys and continue. logger.Log(TraceEventType.Error, e, null, "Failed to index some of the documents: {0}", string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key))); } catch (Exception e) { logger.Log(TraceEventType.Error, e, null, "Search index failed"); } return(false); }
public async Task <ServiceResult <bool> > ChangeDocumentsInIndex <T>(IEnumerable <T> documents, IEnumerable <AzureSearchIndexType> crudTypes, string indexName = null) where T : class { var serviceResult = new ServiceResult <bool>(); indexName = GetIndexName <T>(indexName); var searchIndexCreateServiceResult = await CreateSearchIndex <T>(indexName); if (!searchIndexCreateServiceResult.IsStatusOk()) { return(serviceResult.CopyStatus(searchIndexCreateServiceResult)); } var indexActions = new List <IndexAction <T> >(); var documentCounter = 0; foreach (var document in documents) { var crudType = AzureSearchIndexType.Upload; IndexAction <T> indexAction = null; if (crudTypes.Count() > documentCounter) { crudType = crudTypes.ElementAt(documentCounter); } else { crudType = crudTypes.First(); } switch (crudType) { case AzureSearchIndexType.Upload: indexAction = IndexAction.Upload(document); break; case AzureSearchIndexType.Delete: indexAction = IndexAction.Delete(document); break; case AzureSearchIndexType.Merge: indexAction = IndexAction.Merge(document); break; default: indexAction = IndexAction.MergeOrUpload(document); break; } indexActions.Add(indexAction); documentCounter++; } var batch = IndexBatch.New(indexActions); var indexClient = _serviceClient.Indexes.GetClient(indexName); try { var documentIndexResult = await indexClient.Documents.IndexAsync(batch); serviceResult.Data = documentIndexResult.Results != null && documentIndexResult.Results.Count == documents.Count(); } catch (IndexBatchException e) { // Sometimes when your Search service is under load, indexing will fail for some of the documents in // the batch. Depending on your application, you can take compensating actions like delaying and // retrying. For this simple demo, we just log the failed document keys and continue. serviceResult.SetException(e); _logger.Log(TraceEventType.Error, e, null, "Failed to index some of the documents: {0}", string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key))); } catch (Exception e) { serviceResult.SetException(e); _logger.Log(TraceEventType.Error, e, null, "Search index failed"); } return(serviceResult); }
public Facts(ITestOutputHelper output) { AuxiliaryFileClient = new Mock <IAuxiliaryFileClient>(); DatabaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); DownloadDataClient = new Mock <IDownloadDataClient>(); DownloadSetComparer = new Mock <IDownloadSetComparer>(); DownloadTransferrer = new Mock <IDownloadTransferrer>(); PopularityTransferDataClient = new Mock <IPopularityTransferDataClient>(); SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>(); IndexActionBuilder = new Mock <ISearchIndexActionBuilder>(); BatchPusher = new Mock <IBatchPusher>(); SystemTime = new Mock <ISystemTime>(); FeatureFlags = new Mock <IFeatureFlagService>(); Options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >(); TelemetryService = new Mock <IAzureSearchTelemetryService>(); Logger = output.GetLogger <Auxiliary2AzureSearchCommand>(); Config = new Auxiliary2AzureSearchConfiguration { AzureSearchBatchSize = 10, MaxConcurrentBatches = 1, MaxConcurrentVersionListWriters = 1, EnablePopularityTransfers = true, MinPushPeriod = TimeSpan.FromSeconds(5), }; Options.Setup(x => x.Value).Returns(() => Config); OldDownloadData = new DownloadData(); OldDownloadResult = Data.GetAuxiliaryFileResult(OldDownloadData, "download-data-etag"); DownloadDataClient .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>())) .ReturnsAsync(() => OldDownloadResult); NewDownloadData = new DownloadData(); AuxiliaryFileClient.Setup(x => x.LoadDownloadDataAsync()).ReturnsAsync(() => NewDownloadData); Changes = new SortedDictionary <string, long>(); DownloadSetComparer .Setup(x => x.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>())) .Returns(() => Changes); OldTransfers = new PopularityTransferData(); OldTransferResult = new AuxiliaryFileResult <PopularityTransferData>( modified: true, data: OldTransfers, metadata: new AuxiliaryFileMetadata( DateTimeOffset.UtcNow, TimeSpan.Zero, fileSize: 0, etag: "etag")); PopularityTransferDataClient .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>())) .ReturnsAsync(OldTransferResult); NewTransfers = new PopularityTransferData(); DatabaseFetcher .Setup(x => x.GetPopularityTransfersAsync()) .ReturnsAsync(NewTransfers); TransferChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase); DownloadTransferrer .Setup(x => x.UpdateDownloadTransfers( It.IsAny <DownloadData>(), It.IsAny <SortedDictionary <string, long> >(), It.IsAny <PopularityTransferData>(), It.IsAny <PopularityTransferData>())) .Returns(TransferChanges); IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), Mock.Of <IAccessCondition>())); ProcessedIds = new ConcurrentBag <string>(); IndexActionBuilder .Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >())) .ReturnsAsync(() => IndexActions) .Callback <string, Func <SearchFilters, KeyedDocument> >((id, b) => { ProcessedIds.Add(id); b(SearchFilters.IncludePrereleaseAndSemVer2); }); // When pushing, delay for a little bit of time so the stopwatch has some measurable duration. PushedIds = new ConcurrentBag <string>(); CurrentBatch = new ConcurrentBag <IndexActions>(); FinishedBatches = new ConcurrentBag <List <IndexActions> >(); BatchPusher .Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>())) .Callback <string, IndexActions>((id, indexActions) => { CurrentBatch.Add(indexActions); PushedIds.Add(id); }); BatchPusher .Setup(x => x.TryFinishAsync()) .Returns(async() => { await Task.Delay(TimeSpan.FromMilliseconds(1)); return(new BatchPusherResult()); }) .Callback(() => { FinishedBatches.Add(CurrentBatch.ToList()); CurrentBatch = new ConcurrentBag <IndexActions>(); }); FeatureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); Target = new UpdateDownloadsCommand( AuxiliaryFileClient.Object, DatabaseFetcher.Object, DownloadDataClient.Object, DownloadSetComparer.Object, DownloadTransferrer.Object, PopularityTransferDataClient.Object, SearchDocumentBuilder.Object, IndexActionBuilder.Object, () => BatchPusher.Object, SystemTime.Object, FeatureFlags.Object, Options.Object, TelemetryService.Object, Logger); }