public override async Task BuildIndexAsync(BagPart bagPart, BuildPartIndexContext context)
{
var options = context.Settings.ToOptions();
if (options == DocumentIndexOptions.None)
{
return;
}
if (bagPart.ContentItems.Count != 0)
{
// Lazy resolution to prevent cyclic dependency
var contentItemIndexHandlers = _serviceProvider.GetServices <IContentItemIndexHandler>();
foreach (var contentItemIndexHandler in contentItemIndexHandlers)
{
foreach (var contentItem in bagPart.ContentItems)
{
var keys = new List <string>();
keys.Add(contentItem.ContentType);
foreach (var key in context.Keys)
{
keys.Add($"{key}.{contentItem.ContentType}");
}
var buildIndexContext = new BuildIndexContext(context.DocumentIndex, contentItem, keys);
await contentItemIndexHandler.BuildIndexAsync(buildIndexContext);
}
}
}
}
public async Task BuildIndexAsync(BuildIndexContext context)
{
var body = await _contentManager.PopulateAspectAsync(context.ContentItem, new BodyAspect());
if (body != null)
{
context.DocumentIndex.Set(
IndexingConstants.BodyAspectBodyKey,
body.Body,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize);
}
if (context.ContentItem.DisplayText != null)
{
context.DocumentIndex.Set(
IndexingConstants.DisplayTextAnalyzedKey,
context.ContentItem.DisplayText,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize);
context.DocumentIndex.Set(
IndexingConstants.DisplayTextKey,
context.ContentItem.DisplayText,
DocumentIndexOptions.Store);
}
}
public async Task BuildIndexAsync(BuildIndexContext context)
{
var body = await _contentManager.PopulateAspectAsync(context.ContentItem, new BodyAspect());
if (body != null)
{
context.DocumentIndex.Entries.Add(
"Content.BodyAspect.Body",
new DocumentIndex.DocumentIndexEntry(
body.Body,
DocumentIndex.Types.Text,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize));
}
var contentItemMetadata = await _contentManager.PopulateAspectAsync <ContentItemMetadata>(context.ContentItem);
if (contentItemMetadata?.DisplayText != null)
{
context.DocumentIndex.Entries.Add(
"Content.ContentItemMetadata.DisplayText.Analyzed",
new DocumentIndex.DocumentIndexEntry(
contentItemMetadata.DisplayText,
DocumentIndex.Types.Text,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize));
context.DocumentIndex.Entries.Add(
"Content.ContentItemMetadata.DisplayText",
new DocumentIndex.DocumentIndexEntry(
contentItemMetadata.DisplayText,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
}
}
private static async Task IndexingAsync(ShellScope scope, IEnumerable <ContentContextBase> contexts)
{
var services = scope.ServiceProvider;
var contentManager = services.GetRequiredService <IContentManager>();
var contentItemIndexHandlers = services.GetServices <IContentItemIndexHandler>();
var luceneIndexManager = services.GetRequiredService <LuceneIndexManager>();
var luceneIndexSettingsService = services.GetRequiredService <LuceneIndexSettingsService>();
var logger = services.GetRequiredService <ILogger <LuceneIndexingContentHandler> >();
// Multiple items may have been updated in the same scope, e.g through a recipe.
var contextsGroupById = contexts.GroupBy(c => c.ContentItem.ContentItemId, c => c);
// Get all contexts for each content item id.
foreach (var ContextsById in contextsGroupById)
{
// Only process the last context.
var context = ContextsById.Last();
ContentItem published = null, latest = null;
bool publishedLoaded = false, latestLoaded = false;
foreach (var indexSettings in await luceneIndexSettingsService.GetSettingsAsync())
{
var cultureAspect = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem);
var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null;
var ignoreIndexedCulture = indexSettings.Culture == "any" ? false : culture != indexSettings.Culture;
if (indexSettings.IndexedContentTypes.Contains(context.ContentItem.ContentType) && !ignoreIndexedCulture)
{
if (!indexSettings.IndexLatest && !publishedLoaded)
{
publishedLoaded = true;
published = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published);
}
if (indexSettings.IndexLatest && !latestLoaded)
{
latestLoaded = true;
latest = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Latest);
}
var contentItem = !indexSettings.IndexLatest ? published : latest;
if (contentItem == null)
{
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
else
{
var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId), contentItem, new string[] { contentItem.ContentType });
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex });
}
}
}
}
}
public Task BuildIndexAsync(BuildIndexContext context)
{
var body = _contentManager.PopulateAspect(context.ContentItem, new BodyAspect());
if (body != null)
{
context.DocumentIndex.Entries.Add(
"Content.BodyAspect.Body",
new DocumentIndex.DocumentIndexEntry(
body.Body,
DocumentIndex.Types.Text,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize));
}
var contentItemMetadata = _contentManager.PopulateAspect(context.ContentItem, new ContentItemMetadata());
if (contentItemMetadata?.DisplayText != null)
{
context.DocumentIndex.Entries.Add(
"Content.ContentItemMetadata.DisplayText",
new DocumentIndex.DocumentIndexEntry(
contentItemMetadata.DisplayText,
DocumentIndex.Types.Text,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize | DocumentIndexOptions.Store));
}
return(Task.CompletedTask);
}
public Task BuildIndexAsync(BuildIndexContext context)
{
context.DocumentIndex.Set(
IndexingConstants.ContentItemIdKey,
context.ContentItem.ContentItemId,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.ContentItemVersionIdKey,
context.ContentItem.ContentItemVersionId,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.ContentTypeKey,
context.ContentItem.ContentType,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.CreatedUtcKey,
context.ContentItem.CreatedUtc,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.LatestKey,
context.ContentItem.Latest,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.OwnerKey,
context.ContentItem.Owner,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.AuthorKey,
context.ContentItem.Author,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.ModifiedUtcKey,
context.ContentItem.ModifiedUtc,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.PublishedKey,
context.ContentItem.Published,
DocumentIndexOptions.Store);
context.DocumentIndex.Set(
IndexingConstants.PublishedUtcKey,
context.ContentItem.PublishedUtc,
DocumentIndexOptions.Store);
return(Task.CompletedTask);
}
public async Task BuildIndexAsync(BuildIndexContext context)
{
var result = await _contentManager.PopulateAspectAsync <FullTextAspect>(context.ContentItem);
// Index each segment as a new value to prevent from allocation a new string
foreach (var segment in result.Segments)
{
context.DocumentIndex.Set(
IndexingConstants.FullTextKey,
segment,
DocumentIndexOptions.Analyze | DocumentIndexOptions.Sanitize);
}
}
public override async Task PublishedAsync(PublishContentContext context)
{
// TODO: ignore if this index is not configured for the content type
var buildIndexContext = new BuildIndexContext(new DocumentIndex(context.ContentItem.ContentItemId), context.ContentItem, new string[] { context.ContentItem.ContentType });
// Lazy resolution to prevent cyclic dependency
var contentItemIndexHandlers = _serviceProvider.GetServices <IContentItemIndexHandler>();
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), _logger);
foreach (var index in _luceneIndexManager.List())
{
_luceneIndexManager.DeleteDocuments(index, new string[] { context.ContentItem.ContentItemId });
_luceneIndexManager.StoreDocuments(index, new DocumentIndex[] { buildIndexContext.DocumentIndex });
}
}
public Task BuildIndexAsync(BuildIndexContext context)
{
var parent = context.ContentItem.As <ContainedPart>();
if (parent == null)
{
return(Task.CompletedTask);
}
context.DocumentIndex.Set(
ParentKey,
parent.ListContentItemId,
DocumentIndexOptions.Store);
return(Task.CompletedTask);
}
public async Task BuildIndexAsync(BuildIndexContext context)
{
var contentTypeDefinition = _contentDefinitionManager.GetTypeDefinition(context.ContentItem.ContentType);
if (contentTypeDefinition == null)
{
return;
}
foreach (var contentTypePartDefinition in contentTypeDefinition.Parts)
{
var partName = contentTypePartDefinition.Name;
var partTypeName = contentTypePartDefinition.PartDefinition.Name;
var partActivator = _contentPartFactory.GetTypeActivator(partTypeName);
var part = (ContentPart)context.ContentItem.Get(partActivator.Type, partName);
var typePartIndexSettings = contentTypePartDefinition.GetSettings <ContentIndexSettings>();
// Skip this part if it's not included in the index and it's not the default type part
if (partName != partTypeName && !typePartIndexSettings.Included)
{
continue;
}
await _partIndexHandlers.InvokeAsync((handler, part, contentTypePartDefinition, context, typePartIndexSettings) =>
handler.BuildIndexAsync(part, contentTypePartDefinition, context, typePartIndexSettings),
part, contentTypePartDefinition, context, typePartIndexSettings, _logger);
foreach (var contentPartFieldDefinition in contentTypePartDefinition.PartDefinition.Fields)
{
var partFieldIndexSettings = contentPartFieldDefinition.GetSettings <ContentIndexSettings>();
if (!partFieldIndexSettings.Included)
{
continue;
}
await _fieldIndexHandlers.InvokeAsync((handler, part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings) =>
handler.BuildIndexAsync(part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings),
part, contentTypePartDefinition, contentPartFieldDefinition, context, partFieldIndexSettings, _logger);
}
}
return;
}
public Task BuildIndexAsync(BuildIndexContext context)
{
context.DocumentIndex.Entries.Add(
"Content.ContentItem.ContentItemId",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.ContentItemId,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.ContentItemVersionId",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.ContentItemVersionId,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.ContentType",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.ContentType,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.CreatedUtc",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.CreatedUtc,
DocumentIndex.Types.DateTime,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.Latest",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.Latest,
DocumentIndex.Types.Boolean,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.Owner",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.Owner,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.Author",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.Author,
DocumentIndex.Types.Text,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.ModifiedUtc",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.ModifiedUtc,
DocumentIndex.Types.DateTime,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.Published",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.Published,
DocumentIndex.Types.Boolean,
DocumentIndexOptions.Store));
context.DocumentIndex.Entries.Add(
"Content.ContentItem.PublishedUtc",
new DocumentIndex.DocumentIndexEntry(
context.ContentItem.PublishedUtc,
DocumentIndex.Types.DateTime,
DocumentIndexOptions.Store));
return(Task.CompletedTask);
}
public async Task ProcessContentItemsAsync()
{
// TODO: Lock over the filesystem
var allIndexes = new Dictionary <string, int>();
// Find the lowest task id to process
int lastTaskId = int.MaxValue;
foreach (var indexName in _indexManager.List())
{
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndexes.Add(indexName, taskId);
}
if (!allIndexes.Any())
{
return;
}
IEnumerable <IndexingTask> batch;
do
{
// Load the next batch of tasks
batch = (await _indexTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
break;
}
foreach (var task in batch)
{
foreach (var index in allIndexes)
{
// TODO: ignore if this index is not configured for the content type
if (index.Value < task.Id)
{
_indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId });
}
}
if (task.Type == IndexingTaskTypes.Update)
{
var contentItem = await _contentManager.GetAsync(task.ContentItemId);
var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, contentItem.ContentType);
// Update the document from the index if its lastIndexId is smaller than the current task id.
await _indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger);
foreach (var index in allIndexes)
{
if (index.Value < task.Id)
{
_indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex });
}
}
}
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var index in allIndexes)
{
if (index.Value < lastTaskId)
{
_indexingState.SetLastTaskId(index.Key, lastTaskId);
}
}
_indexingState.Update();
} while (batch.Count() == BatchSize);
}
public async Task ProcessContentItemsAsync()
{
// TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.
var allIndices = new Dictionary <string, int>();
// Find the lowest task id to process
int lastTaskId = int.MaxValue;
foreach (var indexName in _indexManager.List())
{
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexName, taskId);
}
if (!allIndices.Any())
{
return;
}
IndexingTask[] batch;
do
{
// Create a scope for the content manager
using (var scope = await _shellHost.GetScopeAsync(_shellSettings))
{
// Load the next batch of tasks
batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
break;
}
foreach (var task in batch)
{
var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();
foreach (var index in allIndices)
{
// TODO: ignore if this index is not configured for the content type
if (index.Value < task.Id)
{
_indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId });
}
}
if (task.Type == IndexingTaskTypes.Update)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId);
if (contentItem == null)
{
continue;
}
var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
// Update the document from the index if its lastIndexId is smaller than the current task id.
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger);
foreach (var index in allIndices)
{
if (index.Value < task.Id)
{
_indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex });
}
}
}
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var index in allIndices)
{
if (index.Value < lastTaskId)
{
_indexingState.SetLastTaskId(index.Key, lastTaskId);
}
}
_indexingState.Update();
}
} while (batch.Length == BatchSize);
}
public async Task ProcessContentItemsAsync(string indexName = default)
{
// TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.
var allIndices = new Dictionary <string, int>();
var lastTaskId = Int32.MaxValue;
IEnumerable <LuceneIndexSettings> indexSettingsList = null;
if (String.IsNullOrEmpty(indexName))
{
indexSettingsList = await _luceneIndexSettingsService.GetSettingsAsync();
if (!indexSettingsList.Any())
{
return;
}
// Find the lowest task id to process
foreach (var indexSetting in indexSettingsList)
{
var taskId = _indexingState.GetLastTaskId(indexSetting.IndexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexSetting.IndexName, taskId);
}
}
else
{
var settings = await _luceneIndexSettingsService.GetSettingsAsync(indexName);
if (settings == null)
{
return;
}
indexSettingsList = new LuceneIndexSettings[1] {
settings
}.AsEnumerable();
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexName, taskId);
}
if (allIndices.Count == 0)
{
return;
}
var batch = Array.Empty <IndexingTask>();
do
{
// Create a scope for the content manager
var shellScope = await _shellHost.GetScopeAsync(_shellSettings);
await shellScope.UsingAsync(async scope =>
{
// Load the next batch of tasks
batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
return;
}
var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();
// Pre-load all content items to prevent SELECT N+1
var updatedContentItemIds = batch
.Where(x => x.Type == IndexingTaskTypes.Update)
.Select(x => x.ContentItemId)
.ToArray();
var allPublished = await contentManager.GetAsync(updatedContentItemIds);
var allLatest = await contentManager.GetAsync(updatedContentItemIds, latest: true);
// Group all DocumentIndex by index to batch update them
var updatedDocumentsByIndex = new Dictionary <string, List <DocumentIndex> >();
foreach (var index in allIndices)
{
updatedDocumentsByIndex[index.Key] = new List <DocumentIndex>();
}
if (indexName != null)
{
indexSettingsList = indexSettingsList.Where(x => x.IndexName == indexName);
}
var needLatest = indexSettingsList.FirstOrDefault(x => x.IndexLatest) != null;
var needPublished = indexSettingsList.FirstOrDefault(x => !x.IndexLatest) != null;
var settingsByIndex = indexSettingsList.ToDictionary(x => x.IndexName, x => x);
foreach (var task in batch)
{
if (task.Type == IndexingTaskTypes.Update)
{
BuildIndexContext publishedIndexContext = null, latestIndexContext = null;
if (needPublished)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId);
if (contentItem != null)
{
publishedIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(publishedIndexContext), _logger);
}
}
if (needLatest)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId, VersionOptions.Latest);
if (contentItem != null)
{
latestIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(latestIndexContext), _logger);
}
}
// Update the document from the index if its lastIndexId is smaller than the current task id.
foreach (var index in allIndices)
{
if (index.Value >= task.Id || !settingsByIndex.TryGetValue(index.Key, out var settings))
{
continue;
}
var context = !settings.IndexLatest ? publishedIndexContext : latestIndexContext;
//We index only if we actually found a content item in the database
if (context == null)
{
//TODO purge these content items from IndexingTask table
continue;
}
var cultureAspect = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem);
var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null;
var ignoreIndexedCulture = settings.Culture == "any" ? false : culture != settings.Culture;
// Ignore if the content item content type or culture is not indexed in this index
if (!settings.IndexedContentTypes.Contains(context.ContentItem.ContentType) || ignoreIndexedCulture)
{
continue;
}
updatedDocumentsByIndex[index.Key].Add(context.DocumentIndex);
}
}
}
// Delete all the existing documents
foreach (var index in updatedDocumentsByIndex)
{
var deletedDocuments = updatedDocumentsByIndex[index.Key].Select(x => x.ContentItemId);
await _indexManager.DeleteDocumentsAsync(index.Key, deletedDocuments);
}
// Submits all the new documents to the index
foreach (var index in updatedDocumentsByIndex)
{
await _indexManager.StoreDocumentsAsync(index.Key, updatedDocumentsByIndex[index.Key]);
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var indexStatus in allIndices)
{
if (indexStatus.Value < lastTaskId)
{
_indexingState.SetLastTaskId(indexStatus.Key, lastTaskId);
}
}
_indexingState.Update();
}, activateShell : false);
} while (batch.Length == BatchSize);
}
