public async Task ProcessContentItemsAsync()
{
// TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.
var allIndices = new Dictionary <string, int>();
// Find the lowest task id to process
int lastTaskId = int.MaxValue;
foreach (var indexName in _indexManager.List())
{
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexName, taskId);
}
if (!allIndices.Any())
{
return;
}
IndexingTask[] batch;
do
{
// Create a scope for the content manager
using (var scope = await _shellHost.GetScopeAsync(_shellSettings))
{
// Load the next batch of tasks
batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
break;
}
foreach (var task in batch)
{
var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();
foreach (var index in allIndices)
{
// TODO: ignore if this index is not configured for the content type
if (index.Value < task.Id)
{
_indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId });
}
}
if (task.Type == IndexingTaskTypes.Update)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId);
if (contentItem == null)
{
continue;
}
var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
// Update the document from the index if its lastIndexId is smaller than the current task id.
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger);
foreach (var index in allIndices)
{
if (index.Value < task.Id)
{
_indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex });
}
}
}
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var index in allIndices)
{
if (index.Value < lastTaskId)
{
_indexingState.SetLastTaskId(index.Key, lastTaskId);
}
}
_indexingState.Update();
}
} while (batch.Length == BatchSize);
}
public async Task ProcessContentItemsAsync()
{
// TODO: Lock over the filesystem
var allIndexes = new Dictionary <string, int>();
// Find the lowest task id to process
int lastTaskId = int.MaxValue;
foreach (var indexName in _indexManager.List())
{
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndexes.Add(indexName, taskId);
}
if (!allIndexes.Any())
{
return;
}
IEnumerable <IndexingTask> batch;
do
{
// Load the next batch of tasks
batch = (await _indexTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
break;
}
foreach (var task in batch)
{
foreach (var index in allIndexes)
{
// TODO: ignore if this index is not configured for the content type
if (index.Value < task.Id)
{
_indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId });
}
}
if (task.Type == IndexingTaskTypes.Update)
{
var contentItem = await _contentManager.GetAsync(task.ContentItemId);
var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, contentItem.ContentType);
// Update the document from the index if its lastIndexId is smaller than the current task id.
await _indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger);
foreach (var index in allIndexes)
{
if (index.Value < task.Id)
{
_indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex });
}
}
}
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var index in allIndexes)
{
if (index.Value < lastTaskId)
{
_indexingState.SetLastTaskId(index.Key, lastTaskId);
}
}
_indexingState.Update();
} while (batch.Count() == BatchSize);
}
public async Task ProcessContentItemsAsync(string indexName = default)
{
// TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.
var allIndices = new Dictionary <string, int>();
var lastTaskId = Int32.MaxValue;
IEnumerable <LuceneIndexSettings> indexSettingsList = null;
if (String.IsNullOrEmpty(indexName))
{
indexSettingsList = await _luceneIndexSettingsService.GetSettingsAsync();
if (!indexSettingsList.Any())
{
return;
}
// Find the lowest task id to process
foreach (var indexSetting in indexSettingsList)
{
var taskId = _indexingState.GetLastTaskId(indexSetting.IndexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexSetting.IndexName, taskId);
}
}
else
{
var settings = await _luceneIndexSettingsService.GetSettingsAsync(indexName);
if (settings == null)
{
return;
}
indexSettingsList = new LuceneIndexSettings[1] {
settings
}.AsEnumerable();
var taskId = _indexingState.GetLastTaskId(indexName);
lastTaskId = Math.Min(lastTaskId, taskId);
allIndices.Add(indexName, taskId);
}
if (allIndices.Count == 0)
{
return;
}
var batch = Array.Empty <IndexingTask>();
do
{
// Create a scope for the content manager
var shellScope = await _shellHost.GetScopeAsync(_shellSettings);
await shellScope.UsingAsync(async scope =>
{
// Load the next batch of tasks
batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();
if (!batch.Any())
{
return;
}
var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
var indexHandlers = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();
// Pre-load all content items to prevent SELECT N+1
var updatedContentItemIds = batch
.Where(x => x.Type == IndexingTaskTypes.Update)
.Select(x => x.ContentItemId)
.ToArray();
var allPublished = await contentManager.GetAsync(updatedContentItemIds);
var allLatest = await contentManager.GetAsync(updatedContentItemIds, latest: true);
// Group all DocumentIndex by index to batch update them
var updatedDocumentsByIndex = new Dictionary <string, List <DocumentIndex> >();
foreach (var index in allIndices)
{
updatedDocumentsByIndex[index.Key] = new List <DocumentIndex>();
}
if (indexName != null)
{
indexSettingsList = indexSettingsList.Where(x => x.IndexName == indexName);
}
var needLatest = indexSettingsList.FirstOrDefault(x => x.IndexLatest) != null;
var needPublished = indexSettingsList.FirstOrDefault(x => !x.IndexLatest) != null;
var settingsByIndex = indexSettingsList.ToDictionary(x => x.IndexName, x => x);
foreach (var task in batch)
{
if (task.Type == IndexingTaskTypes.Update)
{
BuildIndexContext publishedIndexContext = null, latestIndexContext = null;
if (needPublished)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId);
if (contentItem != null)
{
publishedIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(publishedIndexContext), _logger);
}
}
if (needLatest)
{
var contentItem = await contentManager.GetAsync(task.ContentItemId, VersionOptions.Latest);
if (contentItem != null)
{
latestIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(latestIndexContext), _logger);
}
}
// Update the document from the index if its lastIndexId is smaller than the current task id.
foreach (var index in allIndices)
{
if (index.Value >= task.Id || !settingsByIndex.TryGetValue(index.Key, out var settings))
{
continue;
}
var context = !settings.IndexLatest ? publishedIndexContext : latestIndexContext;
//We index only if we actually found a content item in the database
if (context == null)
{
//TODO purge these content items from IndexingTask table
continue;
}
var cultureAspect = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem);
var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null;
var ignoreIndexedCulture = settings.Culture == "any" ? false : culture != settings.Culture;
// Ignore if the content item content type or culture is not indexed in this index
if (!settings.IndexedContentTypes.Contains(context.ContentItem.ContentType) || ignoreIndexedCulture)
{
continue;
}
updatedDocumentsByIndex[index.Key].Add(context.DocumentIndex);
}
}
}
// Delete all the existing documents
foreach (var index in updatedDocumentsByIndex)
{
var deletedDocuments = updatedDocumentsByIndex[index.Key].Select(x => x.ContentItemId);
await _indexManager.DeleteDocumentsAsync(index.Key, deletedDocuments);
}
// Submits all the new documents to the index
foreach (var index in updatedDocumentsByIndex)
{
await _indexManager.StoreDocumentsAsync(index.Key, updatedDocumentsByIndex[index.Key]);
}
// Update task ids
lastTaskId = batch.Last().Id;
foreach (var indexStatus in allIndices)
{
if (indexStatus.Value < lastTaskId)
{
_indexingState.SetLastTaskId(indexStatus.Key, lastTaskId);
}
}
_indexingState.Update();
}, activateShell : false);
} while (batch.Length == BatchSize);
}