public async Task ProcessContentItemsAsync()
        {
            // TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.

            var allIndices = new Dictionary <string, int>();

            // Find the lowest task id to process
            int lastTaskId = int.MaxValue;

            foreach (var indexName in _indexManager.List())
            {
                var taskId = _indexingState.GetLastTaskId(indexName);
                lastTaskId = Math.Min(lastTaskId, taskId);
                allIndices.Add(indexName, taskId);
            }

            if (!allIndices.Any())
            {
                return;
            }

            IndexingTask[] batch;

            do
            {
                // Create a scope for the content manager
                using (var scope = await _shellHost.GetScopeAsync(_shellSettings))
                {
                    // Load the next batch of tasks
                    batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();

                    if (!batch.Any())
                    {
                        break;
                    }

                    foreach (var task in batch)
                    {
                        var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
                        var indexHandlers  = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();

                        foreach (var index in allIndices)
                        {
                            // TODO: ignore if this index is not configured for the content type

                            if (index.Value < task.Id)
                            {
                                _indexManager.DeleteDocuments(index.Key, new string[] { task.ContentItemId });
                            }
                        }

                        if (task.Type == IndexingTaskTypes.Update)
                        {
                            var contentItem = await contentManager.GetAsync(task.ContentItemId);

                            if (contentItem == null)
                            {
                                continue;
                            }

                            var context = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });

                            // Update the document from the index if its lastIndexId is smaller than the current task id.
                            await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(context), Logger);

                            foreach (var index in allIndices)
                            {
                                if (index.Value < task.Id)
                                {
                                    _indexManager.StoreDocuments(index.Key, new DocumentIndex[] { context.DocumentIndex });
                                }
                            }
                        }
                    }


                    // Update task ids
                    lastTaskId = batch.Last().Id;

                    foreach (var index in allIndices)
                    {
                        if (index.Value < lastTaskId)
                        {
                            _indexingState.SetLastTaskId(index.Key, lastTaskId);
                        }
                    }

                    _indexingState.Update();
                }
            } while (batch.Length == BatchSize);
        }
        public async Task ProcessContentItemsAsync(string indexName = default)
        {
            // TODO: Lock over the filesystem in case two instances get a command to rebuild the index concurrently.
            var allIndices = new Dictionary <string, int>();
            var lastTaskId = Int32.MaxValue;
            IEnumerable <LuceneIndexSettings> indexSettingsList = null;

            if (String.IsNullOrEmpty(indexName))
            {
                indexSettingsList = await _luceneIndexSettingsService.GetSettingsAsync();

                if (!indexSettingsList.Any())
                {
                    return;
                }

                // Find the lowest task id to process
                foreach (var indexSetting in indexSettingsList)
                {
                    var taskId = _indexingState.GetLastTaskId(indexSetting.IndexName);
                    lastTaskId = Math.Min(lastTaskId, taskId);
                    allIndices.Add(indexSetting.IndexName, taskId);
                }
            }
            else
            {
                var settings = await _luceneIndexSettingsService.GetSettingsAsync(indexName);

                if (settings == null)
                {
                    return;
                }

                indexSettingsList = new LuceneIndexSettings[1] {
                    settings
                }.AsEnumerable();

                var taskId = _indexingState.GetLastTaskId(indexName);
                lastTaskId = Math.Min(lastTaskId, taskId);
                allIndices.Add(indexName, taskId);
            }

            if (allIndices.Count == 0)
            {
                return;
            }

            var batch = Array.Empty <IndexingTask>();

            do
            {
                // Create a scope for the content manager
                var shellScope = await _shellHost.GetScopeAsync(_shellSettings);

                await shellScope.UsingAsync(async scope =>
                {
                    // Load the next batch of tasks
                    batch = (await _indexingTaskManager.GetIndexingTasksAsync(lastTaskId, BatchSize)).ToArray();

                    if (!batch.Any())
                    {
                        return;
                    }

                    var contentManager = scope.ServiceProvider.GetRequiredService <IContentManager>();
                    var indexHandlers  = scope.ServiceProvider.GetServices <IContentItemIndexHandler>();

                    // Pre-load all content items to prevent SELECT N+1
                    var updatedContentItemIds = batch
                                                .Where(x => x.Type == IndexingTaskTypes.Update)
                                                .Select(x => x.ContentItemId)
                                                .ToArray();

                    var allPublished = await contentManager.GetAsync(updatedContentItemIds);
                    var allLatest    = await contentManager.GetAsync(updatedContentItemIds, latest: true);

                    // Group all DocumentIndex by index to batch update them
                    var updatedDocumentsByIndex = new Dictionary <string, List <DocumentIndex> >();

                    foreach (var index in allIndices)
                    {
                        updatedDocumentsByIndex[index.Key] = new List <DocumentIndex>();
                    }

                    if (indexName != null)
                    {
                        indexSettingsList = indexSettingsList.Where(x => x.IndexName == indexName);
                    }

                    var needLatest    = indexSettingsList.FirstOrDefault(x => x.IndexLatest) != null;
                    var needPublished = indexSettingsList.FirstOrDefault(x => !x.IndexLatest) != null;

                    var settingsByIndex = indexSettingsList.ToDictionary(x => x.IndexName, x => x);

                    foreach (var task in batch)
                    {
                        if (task.Type == IndexingTaskTypes.Update)
                        {
                            BuildIndexContext publishedIndexContext = null, latestIndexContext = null;

                            if (needPublished)
                            {
                                var contentItem = await contentManager.GetAsync(task.ContentItemId);
                                if (contentItem != null)
                                {
                                    publishedIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
                                    await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(publishedIndexContext), _logger);
                                }
                            }

                            if (needLatest)
                            {
                                var contentItem = await contentManager.GetAsync(task.ContentItemId, VersionOptions.Latest);
                                if (contentItem != null)
                                {
                                    latestIndexContext = new BuildIndexContext(new DocumentIndex(task.ContentItemId), contentItem, new string[] { contentItem.ContentType });
                                    await indexHandlers.InvokeAsync(x => x.BuildIndexAsync(latestIndexContext), _logger);
                                }
                            }

                            // Update the document from the index if its lastIndexId is smaller than the current task id.
                            foreach (var index in allIndices)
                            {
                                if (index.Value >= task.Id || !settingsByIndex.TryGetValue(index.Key, out var settings))
                                {
                                    continue;
                                }

                                var context = !settings.IndexLatest ? publishedIndexContext : latestIndexContext;

                                //We index only if we actually found a content item in the database
                                if (context == null)
                                {
                                    //TODO purge these content items from IndexingTask table
                                    continue;
                                }

                                var cultureAspect        = await contentManager.PopulateAspectAsync <CultureAspect>(context.ContentItem);
                                var culture              = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null;
                                var ignoreIndexedCulture = settings.Culture == "any" ? false : culture != settings.Culture;

                                // Ignore if the content item content type or culture is not indexed in this index
                                if (!settings.IndexedContentTypes.Contains(context.ContentItem.ContentType) || ignoreIndexedCulture)
                                {
                                    continue;
                                }

                                updatedDocumentsByIndex[index.Key].Add(context.DocumentIndex);
                            }
                        }
                    }

                    // Delete all the existing documents
                    foreach (var index in updatedDocumentsByIndex)
                    {
                        var deletedDocuments = updatedDocumentsByIndex[index.Key].Select(x => x.ContentItemId);

                        await _indexManager.DeleteDocumentsAsync(index.Key, deletedDocuments);
                    }

                    // Submits all the new documents to the index
                    foreach (var index in updatedDocumentsByIndex)
                    {
                        await _indexManager.StoreDocumentsAsync(index.Key, updatedDocumentsByIndex[index.Key]);
                    }

                    // Update task ids
                    lastTaskId = batch.Last().Id;

                    foreach (var indexStatus in allIndices)
                    {
                        if (indexStatus.Value < lastTaskId)
                        {
                            _indexingState.SetLastTaskId(indexStatus.Key, lastTaskId);
                        }
                    }

                    _indexingState.Update();
                }, activateShell : false);
            } while (batch.Length == BatchSize);
        }