public CrawlersFlow(IEnumerable <Func <IReadOnlyList <string>, IReadOnlyList <IData> > > crawlers) : base(DataflowOptions.Default) { if (crawlers is null) { throw new ArgumentNullException(nameof(crawlers)); } _inputBroadcaster = new DataBroadcaster <IReadOnlyList <string> >(filteredData => { Console.WriteLine($"Broadcasts all filtered inputters data. {filteredData.Count.ToString()}"); return(filteredData); }, DataflowOptions.Default); _resultConsumer = new TransformBlock <IReadOnlyList <IData>, IReadOnlyList <IData> >( crawlersData => crawlersData ).ToDataflow(DataflowOptions.Default); var crawlerFlows = crawlers .Select(crawler => DataflowUtils.FromDelegate(crawler, DataflowOptions.Default)); foreach (var crawlerFlow in crawlerFlows) { _inputBroadcaster.LinkTo(crawlerFlow); crawlerFlow.LinkTo(_resultConsumer); _resultConsumer.RegisterDependency(crawlerFlow); RegisterChild(crawlerFlow); } RegisterChild(_inputBroadcaster); RegisterChild(_resultConsumer); }
public async Task TestAutoComplete1() { var block = new TransformManyBlock <Int, Int>(i => { int j = i.Val + 1; Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j); Thread.Sleep(500); if (j < 10) { return new[] { new Int { Val = j } } } ; else { return(Enumerable.Empty <Int>()); } }); var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => i); var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1)); dataflow1.GoTo(dataflow2).GoTo(dataflow1); dataflow1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(10))); }
public OutputtersFlow( IEnumerable <Action <IReadOnlyList <string> > > outputters) : base(DataflowOptions.Default) { if (outputters is null) { throw new ArgumentNullException(nameof(outputters)); } _resultList = new ConcurrentBag <string>(); _inputConsumer = new DataBroadcaster <IReadOnlyList <string> >(appraisersData => { Console.WriteLine($"Consuming all appraised data. {appraisersData.Count.ToString()}\n"); foreach (string datum in appraisersData) { _resultList.Add(datum); } return(appraisersData); }, DataflowOptions.Default); var outputterFlows = outputters .Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default)); foreach (var outputterFlow in outputterFlows) { _inputConsumer.LinkTo(outputterFlow); RegisterChild(outputterFlow); } RegisterChild(_inputConsumer); }
public AppraisersFlow(IEnumerable <Appraiser> appraisers) : base(DataflowOptions.Default) { if (appraisers is null) { throw new ArgumentNullException(nameof(appraisers)); } _inputConsumer = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData => { Console.WriteLine($"Broadcasts all crawlers data. {crawlersData.Count.ToString()}"); return(crawlersData); }, DataflowOptions.Default); _resultConsumer = new TransformBlock <IReadOnlyList <string>, IReadOnlyList <string> >( appraisedData => appraisedData ).ToDataflow(DataflowOptions.Default); var usedTypes = new Dictionary <Type, DataBroadcaster <IReadOnlyList <IData> > >(); foreach (var appraiser in appraisers) { if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster)) { broadcaster = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData => { Console.WriteLine($"Broadcasts specified data of type {appraiser.DataType.Name}. {crawlersData.Count.ToString()}"); return(crawlersData); }, DataflowOptions.Default); usedTypes.Add(appraiser.DataType, broadcaster); _inputConsumer.TransformAndLink( broadcaster, l => l, l => l.All(d => d.GetType().IsAssignableFrom(appraiser.DataType)) ); RegisterChild(broadcaster); } var appraiserFlow = DataflowUtils.FromDelegate(appraiser.Func, DataflowOptions.Default); broadcaster.LinkTo(appraiserFlow); appraiserFlow.LinkTo(_resultConsumer); _resultConsumer.RegisterDependency(appraiserFlow); RegisterChild(appraiserFlow); } RegisterChild(_inputConsumer); RegisterChild(_resultConsumer); }
private void InitFlow(IEnumerable <Action <RatingDataContainer> > outputters) { var outputterFlows = outputters.Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default) ); foreach (Dataflow <RatingDataContainer> outputterFlow in outputterFlows) { _inputConsumer.LinkTo(outputterFlow); RegisterChild(outputterFlow); } RegisterChild(_inputConsumer); }
public ComplexIntFlow() : base(DataflowOptions.Default) { Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i); Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1); Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>( i => { if (i % 2 == 0) { node2.Post(i); } else { node3.Post(i); } return(999); }); Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine); node1.Name = "node1"; node2.Name = "node2"; node3.Name = "node3"; printer.Name = "printer"; this.RegisterChild(node1); this.RegisterChild(node2); this.RegisterChild(node3); this.RegisterChild(printer, t => { if (t.Status == TaskStatus.RanToCompletion) { Console.WriteLine("Printer done!"); } }); node1.LinkTo(printer); node2.LinkTo(printer); node3.LinkTo(printer); //Completion propagation: node1 ---> node2 node2.RegisterDependency(node1); //Completion propagation: node1 + node2 ---> node3 node3.RegisterDependency(node1); node3.RegisterDependency(node2); this._headBlock = node1.InputBlock; }
private void InitFlow(IEnumerable <Func <string, IEnumerable <string> > > inputters) { var inputFlows = inputters.Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default) ); foreach (Dataflow <string, string> inputFlow in inputFlows) { _inputBroadcaster.LinkTo(inputFlow); inputFlow.LinkTo(_resultTransformer, FilterInputData); _resultTransformer.RegisterDependency(inputFlow); RegisterChild(inputFlow); } RegisterChild(_inputBroadcaster); RegisterChild(_resultTransformer); }
public async Task TestAutoComplete2() { var block2 = new TransformManyBlock <Int, Int>(i => { return(Enumerable.Empty <Int>()); }); var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => new[] { i }); //preserve the guid var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1)); dataflow1.GoTo(dataflow2).GoTo(dataflow1); dataflow1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(2))); Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete")); }
private void InitFlow(IEnumerable <Funcotype> appraisers) { var usedTypes = new Dictionary <Type, DataBroadcaster <BasicInfo> >(); foreach (Funcotype appraiser in appraisers) { if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster)) { broadcaster = new DataBroadcaster <BasicInfo>( crawlersData => crawlersData, DataflowOptions.Default ); usedTypes.Add(appraiser.DataType, broadcaster); _inputConsumer.TransformAndLink( broadcaster, info => info, info => info.GetType().IsAssignableFrom(appraiser.DataType) ); RegisterChild(broadcaster); } Dataflow <BasicInfo, RatingDataContainer> appraiserFlow = DataflowUtils.FromDelegate( appraiser.Func, DataflowOptions.Default ); broadcaster.LinkTo(appraiserFlow); appraiserFlow.LinkTo(_resultConsumer); _resultConsumer.RegisterDependency(appraiserFlow); RegisterChild(appraiserFlow); } RegisterChild(_inputConsumer); RegisterChild(_resultConsumer); }
public InputtersFlow(IEnumerable <Func <string, IReadOnlyList <string> > > inputters) : base(DataflowOptions.Default) { if (inputters is null) { throw new ArgumentNullException(nameof(inputters)); } _filteringSet = new ConcurrentDictionary <string, byte>(); _inputBroadcaster = new DataBroadcaster <string>(input => { Console.WriteLine("Broadcasts input to further blocks."); return(input); }, DataflowOptions.Default); _resultTransformer = DataflowUtils.FromDelegate <IReadOnlyList <string>, IReadOnlyList <string> >( FilterInputData, DataflowOptions.Default ); var inputFlows = inputters .Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default)); foreach (var inputFlow in inputFlows) { _inputBroadcaster.LinkTo(inputFlow); inputFlow.LinkTo(_resultTransformer); _resultTransformer.RegisterDependency(inputFlow); RegisterChild(inputFlow); } RegisterChild(_inputBroadcaster); RegisterChild(_resultTransformer); }