public CrawlersFlow CreateFlow()
{
var crawlersFunc = _crawlersAsync.Select(crawlerAsync =>
new Func <string, IAsyncEnumerable <BasicInfo> >(
entityName => TryGetResponse(crawlerAsync, entityName)
)
);
var crawlersFlow = new CrawlersFlow(crawlersFunc);
_logger.Info("Constructed crawlers pipeline.");
return(crawlersFlow);
}
private DataflowPipeline ConstructPipeline(string storageName)
{
// Input component work.
InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName);
// Crawlers component work.
CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow();
// Appraisers component work.
AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow();
// Output component work.
OutputtersFlow outputtersFlow =
OutputManagerAsync.CreateFlow(storageName: string.Empty);
// Constructing pipeline.
inputtersFlow.LinkTo(crawlersFlow);
crawlersFlow.LinkTo(appraisersFlow);
appraisersFlow.LinkTo(outputtersFlow);
return(new DataflowPipeline(inputtersFlow, outputtersFlow));
}