Exemplo n.º 1
0
        private async Task <ServiceStatus> RequestData(BufferBlock <string> entitiesQueue,
                                                       IDictionary <Type, BufferBlock <BasicInfo> > rawDataQueues)
        {
            try
            {
                bool status = await CrawlersManagerAsync.CollectAllResponses(
                    entitiesQueue, rawDataQueues, _dataFlowOptions
                    );

                if (status)
                {
                    GlobalMessageHandler.OutputMessage(
                        "Crawlers have received responses from services."
                        );
                    return(ServiceStatus.Ok);
                }

                GlobalMessageHandler.OutputMessage(
                    "Crawlers have not received responses from services. Result is empty."
                    );
                return(ServiceStatus.Nothing);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Exception occured during collecting data.");
                return(ServiceStatus.RequestError);
            }
        }
Exemplo n.º 2
0
        public void Dispose()
        {
            if (_disposed)
            {
                return;
            }
            _disposed = true;

            CrawlersManagerAsync.Dispose();
        }
Exemplo n.º 3
0
        private DataflowPipeline ConstructPipeline(string storageName)
        {
            // Input component work.
            InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName);

            // Crawlers component work.
            CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow();

            // Appraisers component work.
            AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow();

            // Output component work.
            OutputtersFlow outputtersFlow =
                OutputManagerAsync.CreateFlow(storageName: string.Empty);

            // Constructing pipeline.
            inputtersFlow.LinkTo(crawlersFlow);
            crawlersFlow.LinkTo(appraisersFlow);
            appraisersFlow.LinkTo(outputtersFlow);

            return(new DataflowPipeline(inputtersFlow, outputtersFlow));
        }