private async Task IndexSpecifications <TSourceItem>(IEnumerable <TSourceItem> specifications)
{
IEnumerable <SpecificationIndex> searchIndices = GetSearchIndices(specifications);
PagedContext <SpecificationIndex> context = new PagedContext <SpecificationIndex>(searchIndices);
IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceSpecificationIndices,
IndexSpecifications,
20,
1,
_logger);
await producerConsumer.Run(context);
}
private async Task <(bool isComplete, IEnumerable <SpecificationIndex> items)> ProduceSpecificationIndices(CancellationToken token,
dynamic context)
{
PagedContext <SpecificationIndex> indexingContext = (PagedContext <SpecificationIndex>)context;
while (indexingContext.HasPages)
{
SpecificationIndex[] searchIndices = indexingContext.NextPage();
foreach (SpecificationIndex specificationIndex in searchIndices)
{
string specificationId = specificationIndex.Id;
LogInformation($"Querying dataset relationships for specification {specificationId}");
ApiResponse <IEnumerable <DatasetSpecificationRelationshipViewModel> > response =
await _datasetsPolicy.ExecuteAsync(() => _datasets.GetRelationshipsBySpecificationId(specificationId));
if (response?.StatusCode.IsSuccess() == false || response?.Content == null)
{
LogError($"Unable to retrieve dataset relationships for specification {specificationId}");
}
DatasetSpecificationRelationshipViewModel[] mappedDatasets = response?.Content?.Where(_ => _.DatasetId.IsNotNullOrWhitespace()).ToArray() ??
EmptyRelationshipArray;
DateTimeOffset?latestMappedUpdatedDate = mappedDatasets.Max(_ => _.LastUpdatedDate);
int mappedCount = mappedDatasets.Length;
LogInformation(
$"Setting total mapped datasets count {mappedCount} and map dataset last updated {latestMappedUpdatedDate} on search index for specification id {specificationId}");
specificationIndex.TotalMappedDataSets = mappedCount;
specificationIndex.MapDatasetLastUpdated = latestMappedUpdatedDate;
}
return(false, searchIndices);
}
return(true, ArraySegment <SpecificationIndex> .Empty);
}