public async Task <SemanticSearchBatchTaskResult> Process(string category, SemanticCard[] semanticCards)
{
if (string.IsNullOrWhiteSpace(category))
{
throw new ArgumentException(nameof(category));
}
if (semanticCards == null)
{
throw new ArgumentException(nameof(semanticCards));
}
var response = new SemanticSearchBatchTaskResult();
foreach (var semanticCard in semanticCards)
{
try
{
var result = await _semanticCardProcessor.Process(category, semanticCard);
if (result.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
catch (Exception ex)
{
_logger.Error("{1} | ' {0} '", semanticCard.Name, category);
_logger.Error(ex);
response.Failed.Add(new SemanticSearchException {
Card = semanticCard, Exception = ex
});
}
}
return(response);
}
public Task <SemanticSearchBatchTaskResult> ProcessUrl(string category, string url)
{
var response = new SemanticSearchBatchTaskResult {
Url = url
};
var processorCount = Environment.ProcessorCount;
// Pipeline members
var cardBatchBufferBlock = new BufferBlock <SemanticCard[]>();
var cardTransformBlock = new TransformBlock <SemanticCard[], SemanticSearchBatchTaskResult>(semanticCards => _semanticSearchBatchProcessor.Process(category, semanticCards));
var cardActionBlock = new ActionBlock <SemanticSearchBatchTaskResult>(delegate(SemanticSearchBatchTaskResult result)
{
response.Processed += result.Processed;
response.Failed.AddRange(result.Failed);
},
// Specify a maximum degree of parallelism.
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = processorCount
});
// Form the pipeline
cardBatchBufferBlock.LinkTo(cardTransformBlock);
cardTransformBlock.LinkTo(cardActionBlock);
// Create the completion tasks:
cardBatchBufferBlock.Completion
.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)cardTransformBlock).Fault(t.Exception);
}
else
{
cardTransformBlock.Complete();
}
});
cardTransformBlock.Completion
.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)cardActionBlock).Fault(t.Exception);
}
else
{
cardActionBlock.Complete();
}
});
// Process "Category" and generate article batch data
_semanticSearchDataSource.Producer(url, cardBatchBufferBlock);
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
cardActionBlock.Completion.Wait();
return(Task.FromResult(response));
}
