private async Task <ServiceStatus> RequestData(BufferBlock <string> entitiesQueue,
IDictionary <Type, BufferBlock <BasicInfo> > rawDataQueues)
{
try
{
bool status = await CrawlersManagerAsync.CollectAllResponses(
entitiesQueue, rawDataQueues, _dataFlowOptions
);
if (status)
{
GlobalMessageHandler.OutputMessage(
"Crawlers have received responses from services."
);
return(ServiceStatus.Ok);
}
GlobalMessageHandler.OutputMessage(
"Crawlers have not received responses from services. Result is empty."
);
return(ServiceStatus.Nothing);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception occured during collecting data.");
return(ServiceStatus.RequestError);
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
CrawlersManagerAsync.Dispose();
}
private DataflowPipeline ConstructPipeline(string storageName)
{
// Input component work.
InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName);
// Crawlers component work.
CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow();
// Appraisers component work.
AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow();
// Output component work.
OutputtersFlow outputtersFlow =
OutputManagerAsync.CreateFlow(storageName: string.Empty);
// Constructing pipeline.
inputtersFlow.LinkTo(crawlersFlow);
crawlersFlow.LinkTo(appraisersFlow);
appraisersFlow.LinkTo(outputtersFlow);
return(new DataflowPipeline(inputtersFlow, outputtersFlow));
}
