protected virtual async Task <IndexingResult> ProcessPartialDocumentsAsync(
IEnumerable <IndexDocumentChange> changes,
BatchIndexingOptions batchOptions,
ICancellationToken cancellationToken)
{
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray();
var changeIds = indexDocumentChanges.Select(x => x.DocumentId).Distinct();
foreach (var id in changeIds)
{
var builders = indexDocumentChanges
.Where(x => x.DocumentId == id)
.SelectMany(x => _configs.GetBuildersForProvider(x.Provider.GetType()));
var documents = await GetDocumentsAsync(new[] { id }, null, builders, cancellationToken);
IndexingResult indexingResult;
indexingResult = await((ISupportPartialUpdate)_searchProvider).IndexPartialAsync(batchOptions.DocumentType, documents);
result.Items.AddRange(indexingResult.Items);
}
return(result);
}
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes,
BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray();
// Full changes don't have changes provider specified because we don't set it for manual indexation.
var fullChanges = _searchProvider is ISupportPartialUpdate?indexDocumentChanges
.Where(x =>
x.ChangeType is IndexDocumentChangeType.Deleted or IndexDocumentChangeType.Created ||
!_configs.GetBuildersForProvider(x.Provider?.GetType()).Any()
)
.ToArray() : indexDocumentChanges;
var partialChanges = indexDocumentChanges.Except(fullChanges);
var partialResult = await ProcessPartialDocumentsAsync(partialChanges, batchOptions, cancellationToken);
var groups = GetLatestChangesForEachDocumentGroupedByChangeType(fullChanges);
foreach (var group in groups)
{
var changeType = group.Key;
var changesGroup = group.Value;
var groupResult =
await ProcessDocumentsAsync(changeType, changesGroup, batchOptions, cancellationToken);
if (groupResult?.Items != null)
{
result.Items.AddRange(groupResult.Items);
}
}
result.Items.AddRange(partialResult.Items);
return(result);
}
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType, IList <string> documentIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
IndexingResult result = null;
if (changeType == IndexDocumentChangeType.Deleted)
{
result = await DeleteDocumentsAsync(batchOptions.DocumentType, documentIds.ToArray());
}
else if (changeType == IndexDocumentChangeType.Modified)
{
result = await IndexDocumentsAsync(batchOptions.DocumentType, documentIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken);
}
return(result);
}
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var result = new IndexingResult();
var groups = GetLatestChangesForEachDocumentGroupedByChangeType(changes);
foreach (var group in groups)
{
var changeType = group.Key;
var documentIds = group.Value;
var groupResult = await ProcessDocumentsAsync(changeType, documentIds, batchOptions, cancellationToken);
if (groupResult?.Items != null)
{
if (result.Items == null)
{
result.Items = new List <IndexingResultItem>();
}
result.Items.AddRange(groupResult.Items);
}
}
return(result);
}
protected virtual async Task ProcessConfigurationAsync(IndexDocumentConfiguration configuration, IndexingOptions options, Action <IndexingProgress> progressCallback, ICancellationToken cancellationToken)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (string.IsNullOrEmpty(configuration.DocumentType))
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentType)}");
}
if (configuration.DocumentSource == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}");
}
if (configuration.DocumentSource.ChangesProvider == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.ChangesProvider)}");
}
if (configuration.DocumentSource.DocumentBuilder == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.DocumentBuilder)}");
}
cancellationToken.ThrowIfCancellationRequested();
var documentType = options.DocumentType;
progressCallback?.Invoke(new IndexingProgress($"{documentType}: calculating total count", documentType));
var batchOptions = new BatchIndexingOptions
{
DocumentType = options.DocumentType,
PrimaryDocumentBuilder = configuration.DocumentSource.DocumentBuilder,
SecondaryDocumentBuilders = configuration.RelatedSources
?.Where(s => s.DocumentBuilder != null)
.Select(s => s.DocumentBuilder)
.ToList(),
};
var feeds = await GetChangeFeeds(configuration, options);
// Try to get total count to indicate progress. Some feeds don't have a total count.
var totalCount = feeds.Any(x => x.TotalCount == null)
? (long?)null
: feeds.Sum(x => x.TotalCount ?? 0);
long processedCount = 0;
var changes = await GetNextChangesAsync(feeds);
while (changes.Any())
{
IList <string> errors = null;
if (_backgroundWorker == null)
{
var indexingResult = await ProcessChangesAsync(changes, batchOptions, cancellationToken);
errors = GetIndexingErrors(indexingResult);
}
else
{
// We're executing a job to index all documents or the changes since a specific time.
// Priority for this indexation work should be quite low.
var documentIds = changes
.Select(x => x.DocumentId)
.Distinct()
.ToArray();
_backgroundWorker.IndexDocuments(configuration.DocumentType, documentIds, IndexingPriority.Background);
}
processedCount += changes.Count;
var description = totalCount != null
? $"{documentType}: {processedCount} of {totalCount} have been indexed"
: $"{documentType}: {processedCount} have been indexed";
progressCallback?.Invoke(new IndexingProgress(description, documentType, totalCount, processedCount, errors));
changes = await GetNextChangesAsync(feeds);
}
progressCallback?.Invoke(new IndexingProgress($"{documentType}: indexation finished", documentType, totalCount ?? processedCount, processedCount));
}
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType,
string[] changedIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
IndexingResult result = null;
if (changeType == IndexDocumentChangeType.Deleted)
{
result = await DeleteDocumentsAsync(batchOptions.DocumentType, changedIds);
}
else if (changeType is IndexDocumentChangeType.Modified or IndexDocumentChangeType.Created)
{
var documents = await GetDocumentsAsync(changedIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken);
if (batchOptions.Reindex && _searchProvider is ISupportIndexSwap supportIndexSwapProvider)
{
result = await supportIndexSwapProvider.IndexWithBackupAsync(batchOptions.DocumentType, documents);
}
else
{
result = await _searchProvider.IndexAsync(batchOptions.DocumentType, documents);
}
}
return(result);
}