Example #1
0
        public async Task <SemanticSearchBatchTaskResult> Process(string category, SemanticCard[] semanticCards)
        {
            if (string.IsNullOrWhiteSpace(category))
            {
                throw new ArgumentException(nameof(category));
            }

            if (semanticCards == null)
            {
                throw new ArgumentException(nameof(semanticCards));
            }

            var response = new SemanticSearchBatchTaskResult();

            foreach (var semanticCard in semanticCards)
            {
                try
                {
                    var result = await _semanticCardProcessor.Process(category, semanticCard);

                    if (result.IsSuccessfullyProcessed)
                    {
                        response.Processed += 1;
                    }
                }
                catch (Exception ex)
                {
                    _logger.Error("{1} | ' {0} '", semanticCard.Name, category);
                    _logger.Error(ex);
                    response.Failed.Add(new SemanticSearchException {
                        Card = semanticCard, Exception = ex
                    });
                }
            }

            return(response);
        }
Example #2
0
        public Task <SemanticSearchBatchTaskResult> ProcessUrl(string category, string url)
        {
            var response = new SemanticSearchBatchTaskResult {
                Url = url
            };

            var processorCount = Environment.ProcessorCount;

            // Pipeline members
            var cardBatchBufferBlock = new BufferBlock <SemanticCard[]>();
            var cardTransformBlock   = new TransformBlock <SemanticCard[], SemanticSearchBatchTaskResult>(semanticCards => _semanticSearchBatchProcessor.Process(category, semanticCards));
            var cardActionBlock      = new ActionBlock <SemanticSearchBatchTaskResult>(delegate(SemanticSearchBatchTaskResult result)
            {
                response.Processed += result.Processed;
                response.Failed.AddRange(result.Failed);
            },
                                                                                       // Specify a maximum degree of parallelism.
                                                                                       new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = processorCount
            });

            // Form the pipeline
            cardBatchBufferBlock.LinkTo(cardTransformBlock);
            cardTransformBlock.LinkTo(cardActionBlock);

            //  Create the completion tasks:
            cardBatchBufferBlock.Completion
            .ContinueWith(t =>
            {
                if (t.IsFaulted)
                {
                    ((IDataflowBlock)cardTransformBlock).Fault(t.Exception);
                }
                else
                {
                    cardTransformBlock.Complete();
                }
            });

            cardTransformBlock.Completion
            .ContinueWith(t =>
            {
                if (t.IsFaulted)
                {
                    ((IDataflowBlock)cardActionBlock).Fault(t.Exception);
                }
                else
                {
                    cardActionBlock.Complete();
                }
            });

            // Process "Category" and generate article batch data
            _semanticSearchDataSource.Producer(url, cardBatchBufferBlock);

            // Mark the head of the pipeline as complete. The continuation tasks
            // propagate completion through the pipeline as each part of the
            // pipeline finishes.
            cardActionBlock.Completion.Wait();

            return(Task.FromResult(response));
        }