private async Task <ServiceStatus> RequestData(BufferBlock <string> entitiesQueue, IDictionary <Type, BufferBlock <BasicInfo> > rawDataQueues) { try { bool status = await CrawlersManagerAsync.CollectAllResponses( entitiesQueue, rawDataQueues, _dataFlowOptions ); if (status) { GlobalMessageHandler.OutputMessage( "Crawlers have received responses from services." ); return(ServiceStatus.Ok); } GlobalMessageHandler.OutputMessage( "Crawlers have not received responses from services. Result is empty." ); return(ServiceStatus.Nothing); } catch (Exception ex) { _logger.Error(ex, "Exception occured during collecting data."); return(ServiceStatus.RequestError); } }
public void Dispose() { if (_disposed) { return; } _disposed = true; CrawlersManagerAsync.Dispose(); }
private DataflowPipeline ConstructPipeline(string storageName) { // Input component work. InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName); // Crawlers component work. CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow(); // Appraisers component work. AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow(); // Output component work. OutputtersFlow outputtersFlow = OutputManagerAsync.CreateFlow(storageName: string.Empty); // Constructing pipeline. inputtersFlow.LinkTo(crawlersFlow); crawlersFlow.LinkTo(appraisersFlow); appraisersFlow.LinkTo(outputtersFlow); return(new DataflowPipeline(inputtersFlow, outputtersFlow)); }