public override async Task BuildIndexAsync(BagPart bagPart, BuildPartIndexContext context) { var options = context.Settings.ToOptions(); if (options == DocumentIndexOptions.None) { return; } if (bagPart.ContentItems.Count != 0) { // Lazy resolution to prevent cyclic dependency var contentItemIndexHandlers = _serviceProvider.GetServices <IContentItemIndexHandler>(); foreach (var contentItemIndexHandler in contentItemIndexHandlers) { foreach (var contentItem in bagPart.ContentItems) { var keys = new List <string>(); keys.Add(contentItem.ContentType); foreach (var key in context.Keys) { keys.Add($"{key}.{contentItem.ContentType}"); } var buildIndexContext = new BuildIndexContext(context.DocumentIndex, contentItem, keys); await contentItemIndexHandler.BuildIndexAsync(buildIndexContext); } } } }
public async Task BuildIndexAsync(BuildIndexContext context) { var body = await _contentManager.PopulateAspectAsync(context.ContentItem, new BodyAspect()); if (body != null) { context.DocumentIndex.Set( IndexingConstants.BodyAspectBodyKey, body.Body, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize); } if (context.ContentItem.DisplayText != null) { context.DocumentIndex.Set( IndexingConstants.DisplayTextAnalyzedKey, context.ContentItem.DisplayText, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize); context.DocumentIndex.Set( IndexingConstants.DisplayTextKey, context.ContentItem.DisplayText, DocumentIndexOptions.Store); } }
public async Task BuildIndexAsync(BuildIndexContext context) { var body = await _contentManager.PopulateAspectAsync(context.ContentItem, new BodyAspect()); if (body != null) { context.DocumentIndex.Entries.Add( "Content.BodyAspect.Body", new DocumentIndex.DocumentIndexEntry( body.Body, DocumentIndex.Types.Text, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize)); } var contentItemMetadata = await _contentManager.PopulateAspectAsync <ContentItemMetadata>(context.ContentItem); if (contentItemMetadata?.DisplayText != null) { context.DocumentIndex.Entries.Add( "Content.ContentItemMetadata.DisplayText.Analyzed", new DocumentIndex.DocumentIndexEntry( contentItemMetadata.DisplayText, DocumentIndex.Types.Text, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize)); context.DocumentIndex.Entries.Add( "Content.ContentItemMetadata.DisplayText", new DocumentIndex.DocumentIndexEntry( contentItemMetadata.DisplayText, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); } }
private static async Task IndexingAsync(ShellScope scope, IEnumerable <ContentContextBase> contexts) { var services = scope.ServiceProvider; var contentManager = services.GetRequiredService <IContentManager>(); var contentItemIndexHandlers = services.GetServices <IContentItemIndexHandler>(); var luceneIndexManager = services.GetRequiredService <LuceneIndexManager>(); var luceneIndexSettingsService = services.GetRequiredService <LuceneIndexSettingsService>(); var logger = services.GetRequiredService <ILogger <LuceneIndexingContentHandler> >(); // Multiple items may have been updated in the same scope, e.g through a recipe. var contextsGroupById = contexts.GroupBy(c => c.ContentItem.ContentItemId, c => c); // Get all contexts for each content item id. foreach (var ContextsById in contextsGroupById) { // Only process the last context. var context = ContextsById.Last(); ContentItem published = null, latest = null; bool publishedLoaded = false, latestLoaded = false; foreach (var indexSettings in await luceneIndexSettingsService.GetSettingsAsync()) { var cultureAspect = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem); var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null; var ignoreIndexedCulture = indexSettings.Culture == "any" ? false : culture != indexSettings.Culture; if (indexSettings.IndexedContentTypes.Contains(context.ContentItem.ContentType) && !ignoreIndexedCulture) { if (!indexSettings.IndexLatest && !publishedLoaded) { publishedLoaded = true; published = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published); } if (indexSettings.IndexLatest && !latestLoaded) { latestLoaded = true; latest = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Latest); } var contentItem = !indexSettings.IndexLatest ? published : latest; if (contentItem == null) { await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId }); } else { var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId), contentItem, new string[] { contentItem.ContentType }); await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger); await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId }); await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex }); } } } } }
public Task BuildIndexAsync(BuildIndexContext context) { var body = _contentManager.PopulateAspect(context.ContentItem, new BodyAspect()); if (body != null) { context.DocumentIndex.Entries.Add( "Content.BodyAspect.Body", new DocumentIndex.DocumentIndexEntry( body.Body, DocumentIndex.Types.Text, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize)); } var contentItemMetadata = _contentManager.PopulateAspect(context.ContentItem, new ContentItemMetadata()); if (contentItemMetadata?.DisplayText != null) { context.DocumentIndex.Entries.Add( "Content.ContentItemMetadata.DisplayText", new DocumentIndex.DocumentIndexEntry( contentItemMetadata.DisplayText, DocumentIndex.Types.Text, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize | DocumentIndexOptions.Store)); } return(Task.CompletedTask); }
public Task BuildIndexAsync(BuildIndexContext context) { context.DocumentIndex.Set( IndexingConstants.ContentItemIdKey, context.ContentItem.ContentItemId, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.ContentItemVersionIdKey, context.ContentItem.ContentItemVersionId, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.ContentTypeKey, context.ContentItem.ContentType, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.CreatedUtcKey, context.ContentItem.CreatedUtc, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.LatestKey, context.ContentItem.Latest, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.OwnerKey, context.ContentItem.Owner, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.AuthorKey, context.ContentItem.Author, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.ModifiedUtcKey, context.ContentItem.ModifiedUtc, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.PublishedKey, context.ContentItem.Published, DocumentIndexOptions.Store); context.DocumentIndex.Set( IndexingConstants.PublishedUtcKey, context.ContentItem.PublishedUtc, DocumentIndexOptions.Store); return(Task.CompletedTask); }
public async Task BuildIndexAsync(BuildIndexContext context) { var result = await _contentManager.PopulateAspectAsync <FullTextAspect>(context.ContentItem); // Index each segment as a new value to prevent from allocation a new string foreach (var segment in result.Segments) { context.DocumentIndex.Set( IndexingConstants.FullTextKey, segment, DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize); } }
public override async Task PublishedAsync(PublishContentContext context) { // TODO: ignore if this index is not configured for the content type var buildIndexContext = new BuildIndexContext(new DocumentIndex(context.ContentItem.ContentItemId), context.ContentItem, new string[] { context.ContentItem.ContentType }); // Lazy resolution to prevent cyclic dependency var contentItemIndexHandlers = _serviceProvider.GetServices <IContentItemIndexHandler>(); await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), _logger); foreach (var index in _luceneIndexManager.List()) { _luceneIndexManager.DeleteDocuments(index, new string[] { context.ContentItem.ContentItemId }); _luceneIndexManager.StoreDocuments(index, new DocumentIndex[] { buildIndexContext.DocumentIndex }); } }
public Task BuildIndexAsync(BuildIndexContext context) { var parent = context.ContentItem.As <ContainedPart>(); if (parent == null) { return(Task.CompletedTask); } context.DocumentIndex.Set( ParentKey, parent.ListContentItemId, DocumentIndexOptions.Store); return(Task.CompletedTask); }
public async Task BuildIndexAsync(BuildIndexContext context) { var contentTypeDefinition = _contentDefinitionManager.GetTypeDefinition(context.ContentItem.ContentType); if (contentTypeDefinition == null) { return; } foreach (var contentTypePartDefinition in contentTypeDefinition.Parts) { var partName = contentTypePartDefinition.Name; var partTypeName = contentTypePartDefinition.PartDefinition.Name; var partActivator = _contentPartFactory.GetTypeActivator(partTypeName); var part = (ContentPart)context.ContentItem.Get(partActivator.Type, partName); var typePartIndexSettings = contentTypePartDefinition.GetSettings <ContentIndexSettings>(); // Skip this part if it's not included in the index and it's not the default type part if (partName != partTypeName && !typePartIndexSettings.Included) { continue; } await _partIndexHandlers.InvokeAsync((handler, part, contentTypePartDefinition, context, typePartIndexSettings) => handler.BuildIndexAsync(part, contentTypePartDefinition, context, typePartIndexSettings), part, contentTypePartDefinition, context, typePartIndexSettings, _logger); foreach (var contentPartFieldDefinition in contentTypePartDefinition.PartDefinition.Fields) { var partFieldIndexSettings = contentPartFieldDefinition.GetSettings <ContentIndexSettings>(); if (!partFieldIndexSettings.Included) { continue; } await _fieldIndexHandlers.InvokeAsync((handler, part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings) => handler.BuildIndexAsync(part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings), part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings, _logger); } } return; }
public Task BuildIndexAsync(BuildIndexContext context) { context.DocumentIndex.Entries.Add( "Content.ContentItem.ContentItemId", new DocumentIndex.DocumentIndexEntry( context.ContentItem.ContentItemId, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.ContentItemVersionId", new DocumentIndex.DocumentIndexEntry( context.ContentItem.ContentItemVersionId, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.ContentType", new DocumentIndex.DocumentIndexEntry( context.ContentItem.ContentType, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.CreatedUtc", new DocumentIndex.DocumentIndexEntry( context.ContentItem.CreatedUtc, DocumentIndex.Types.DateTime, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.Latest", new DocumentIndex.DocumentIndexEntry( context.ContentItem.Latest, DocumentIndex.Types.Boolean, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.Owner", new DocumentIndex.DocumentIndexEntry( context.ContentItem.Owner, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.Author", new DocumentIndex.DocumentIndexEntry( context.ContentItem.Author, DocumentIndex.Types.Text, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.ModifiedUtc", new DocumentIndex.DocumentIndexEntry( context.ContentItem.ModifiedUtc, DocumentIndex.Types.DateTime, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.Published", new DocumentIndex.DocumentIndexEntry( context.ContentItem.Published, DocumentIndex.Types.Boolean, DocumentIndexOptions.Store)); context.DocumentIndex.Entries.Add( "Content.ContentItem.PublishedUtc", new DocumentIndex.DocumentIndexEntry( context.ContentItem.PublishedUtc, DocumentIndex.Types.DateTime, DocumentIndexOptions.Store)); return(Task.CompletedTask); }
public async Task ProcessContentItemsAsync() { // TODO: Lock over the filesystem var allIndexes = new Dictionary <string, int>(); // Find the lowest task id to process int lastTaskId = int.MaxValue; foreach (var indexName in _indexManager.List()) { var taskId = _indexingState.GetLastTaskId(indexName); lastTaskId = Math.Min(lastTaskId, taskId); allIndexes.Add(indexName, taskId); } if (!allIndexes.Any()) { return; } IEnumerable <IndexingTask> batch; do { // Load the next batch of tasks batch = (await _indexTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray(); if (!batch.Any()) { break; } foreach (var task in batch) { foreach (var index in allIndexes) { // TODO: ignore if this index is not configured for the content type if (index.Value < task.Id) { _indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId }); } } if (task.Type == IndexingTaskTypes.Update) { var contentItem = await _contentManager.GetAsync(task.ContentItemId); var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, contentItem.ContentType); // Update the document from the index if its lastIndexId is smaller than the current task id. await _indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger); foreach (var index in allIndexes) { if (index.Value < task.Id) { _indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex }); } } } } // Update task ids lastTaskId = batch.Last().Id; foreach (var index in allIndexes) { if (index.Value < lastTaskId) { _indexingState.SetLastTaskId(index.Key, lastTaskId); } } _indexingState.Update(); } while (batch.Count() == BatchSize); }
public async Task ProcessContentItemsAsync() { // TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently. var allIndices = new Dictionary <string, int>(); // Find the lowest task id to process int lastTaskId = int.MaxValue; foreach (var indexName in _indexManager.List()) { var taskId = _indexingState.GetLastTaskId(indexName); lastTaskId = Math.Min(lastTaskId, taskId); allIndices.Add(indexName, taskId); } if (!allIndices.Any()) { return; } IndexingTask[] batch; do { // Create a scope for the content manager using (var scope = await _shellHost.GetScopeAsync(_shellSettings)) { // Load the next batch of tasks batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray(); if (!batch.Any()) { break; } foreach (var task in batch) { var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>(); var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>(); foreach (var index in allIndices) { // TODO: ignore if this index is not configured for the content type if (index.Value < task.Id) { _indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId }); } } if (task.Type == IndexingTaskTypes.Update) { var contentItem = await contentManager.GetAsync(task.ContentItemId); if (contentItem == null) { continue; } var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType }); // Update the document from the index if its lastIndexId is smaller than the current task id. await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger); foreach (var index in allIndices) { if (index.Value < task.Id) { _indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex }); } } } } // Update task ids lastTaskId = batch.Last().Id; foreach (var index in allIndices) { if (index.Value < lastTaskId) { _indexingState.SetLastTaskId(index.Key, lastTaskId); } } _indexingState.Update(); } } while (batch.Length == BatchSize); }
public async Task ProcessContentItemsAsync(string indexName = default) { // TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently. var allIndices = new Dictionary <string, int>(); var lastTaskId = Int32.MaxValue; IEnumerable <LuceneIndexSettings> indexSettingsList = null; if (String.IsNullOrEmpty(indexName)) { indexSettingsList = await _luceneIndexSettingsService.GetSettingsAsync(); if (!indexSettingsList.Any()) { return; } // Find the lowest task id to process foreach (var indexSetting in indexSettingsList) { var taskId = _indexingState.GetLastTaskId(indexSetting.IndexName); lastTaskId = Math.Min(lastTaskId, taskId); allIndices.Add(indexSetting.IndexName, taskId); } } else { var settings = await _luceneIndexSettingsService.GetSettingsAsync(indexName); if (settings == null) { return; } indexSettingsList = new LuceneIndexSettings[1] { settings }.AsEnumerable(); var taskId = _indexingState.GetLastTaskId(indexName); lastTaskId = Math.Min(lastTaskId, taskId); allIndices.Add(indexName, taskId); } if (allIndices.Count == 0) { return; } var batch = Array.Empty <IndexingTask>(); do { // Create a scope for the content manager var shellScope = await _shellHost.GetScopeAsync(_shellSettings); await shellScope.UsingAsync(async scope => { // Load the next batch of tasks batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray(); if (!batch.Any()) { return; } var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>(); var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>(); // Pre-load all content items to prevent SELECT N+1 var updatedContentItemIds = batch .Where(x => x.Type == IndexingTaskTypes.Update) .Select(x => x.ContentItemId) .ToArray(); var allPublished = await contentManager.GetAsync(updatedContentItemIds); var allLatest = await contentManager.GetAsync(updatedContentItemIds, latest: true); // Group all DocumentIndex by index to batch update them var updatedDocumentsByIndex = new Dictionary <string, List <DocumentIndex> >(); foreach (var index in allIndices) { updatedDocumentsByIndex[index.Key] = new List <DocumentIndex>(); } if (indexName != null) { indexSettingsList = indexSettingsList.Where(x => x.IndexName == indexName); } var needLatest = indexSettingsList.FirstOrDefault(x => x.IndexLatest) != null; var needPublished = indexSettingsList.FirstOrDefault(x => !x.IndexLatest) != null; var settingsByIndex = indexSettingsList.ToDictionary(x => x.IndexName, x => x); foreach (var task in batch) { if (task.Type == IndexingTaskTypes.Update) { BuildIndexContext publishedIndexContext = null, latestIndexContext = null; if (needPublished) { var contentItem = await contentManager.GetAsync(task.ContentItemId); if (contentItem != null) { publishedIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType }); await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(publishedIndexContext), _logger); } } if (needLatest) { var contentItem = await contentManager.GetAsync(task.ContentItemId, VersionOptions.Latest); if (contentItem != null) { latestIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType }); await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(latestIndexContext), _logger); } } // Update the document from the index if its lastIndexId is smaller than the current task id. foreach (var index in allIndices) { if (index.Value >= task.Id || !settingsByIndex.TryGetValue(index.Key, out var settings)) { continue; } var context = !settings.IndexLatest ? publishedIndexContext : latestIndexContext; //We index only if we actually found a content item in the database if (context == null) { //TODO purge these content items from IndexingTask table continue; } var cultureAspect = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem); var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null; var ignoreIndexedCulture = settings.Culture == "any" ? false : culture != settings.Culture; // Ignore if the content item content type or culture is not indexed in this index if (!settings.IndexedContentTypes.Contains(context.ContentItem.ContentType) || ignoreIndexedCulture) { continue; } updatedDocumentsByIndex[index.Key].Add(context.DocumentIndex); } } } // Delete all the existing documents foreach (var index in updatedDocumentsByIndex) { var deletedDocuments = updatedDocumentsByIndex[index.Key].Select(x => x.ContentItemId); await _indexManager.DeleteDocumentsAsync(index.Key, deletedDocuments); } // Submits all the new documents to the index foreach (var index in updatedDocumentsByIndex) { await _indexManager.StoreDocumentsAsync(index.Key, updatedDocumentsByIndex[index.Key]); } // Update task ids lastTaskId = batch.Last().Id; foreach (var indexStatus in allIndices) { if (indexStatus.Value < lastTaskId) { _indexingState.SetLastTaskId(indexStatus.Key, lastTaskId); } } _indexingState.Update(); }, activateShell : false); } while (batch.Length == BatchSize); }