public CrawlersFlow CreateFlow() { var crawlersFunc = _crawlersAsync.Select(crawlerAsync => new Func <string, IAsyncEnumerable <BasicInfo> >( entityName => TryGetResponse(crawlerAsync, entityName) ) ); var crawlersFlow = new CrawlersFlow(crawlersFunc); _logger.Info("Constructed crawlers pipeline."); return(crawlersFlow); }
private DataflowPipeline ConstructPipeline(string storageName) { // Input component work. InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName); // Crawlers component work. CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow(); // Appraisers component work. AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow(); // Output component work. OutputtersFlow outputtersFlow = OutputManagerAsync.CreateFlow(storageName: string.Empty); // Constructing pipeline. inputtersFlow.LinkTo(crawlersFlow); crawlersFlow.LinkTo(appraisersFlow); appraisersFlow.LinkTo(outputtersFlow); return(new DataflowPipeline(inputtersFlow, outputtersFlow)); }