public CrawlersFlow(IEnumerable <Func <IReadOnlyList <string>, IReadOnlyList <IData> > > crawlers)
: base(DataflowOptions.Default)
{
if (crawlers is null)
{
throw new ArgumentNullException(nameof(crawlers));
}
_inputBroadcaster = new DataBroadcaster <IReadOnlyList <string> >(filteredData =>
{
Console.WriteLine($"Broadcasts all filtered inputters data. {filteredData.Count.ToString()}");
return(filteredData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <IData>, IReadOnlyList <IData> >(
crawlersData => crawlersData
).ToDataflow(DataflowOptions.Default);
var crawlerFlows = crawlers
.Select(crawler => DataflowUtils.FromDelegate(crawler, DataflowOptions.Default));
foreach (var crawlerFlow in crawlerFlows)
{
_inputBroadcaster.LinkTo(crawlerFlow);
crawlerFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(crawlerFlow);
RegisterChild(crawlerFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultConsumer);
}
public async Task TestAutoComplete1()
{
var block = new TransformManyBlock <Int, Int>(i =>
{
int j = i.Val + 1;
Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j);
Thread.Sleep(500);
if (j < 10)
{
return new[] { new Int {
Val = j
} }
}
;
else
{
return(Enumerable.Empty <Int>());
}
});
var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => i);
var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(10)));
}
public OutputtersFlow(
IEnumerable <Action <IReadOnlyList <string> > > outputters)
: base(DataflowOptions.Default)
{
if (outputters is null)
{
throw new ArgumentNullException(nameof(outputters));
}
_resultList = new ConcurrentBag <string>();
_inputConsumer = new DataBroadcaster <IReadOnlyList <string> >(appraisersData =>
{
Console.WriteLine($"Consuming all appraised data. {appraisersData.Count.ToString()}\n");
foreach (string datum in appraisersData)
{
_resultList.Add(datum);
}
return(appraisersData);
}, DataflowOptions.Default);
var outputterFlows = outputters
.Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default));
foreach (var outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public AppraisersFlow(IEnumerable <Appraiser> appraisers)
: base(DataflowOptions.Default)
{
if (appraisers is null)
{
throw new ArgumentNullException(nameof(appraisers));
}
_inputConsumer = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts all crawlers data. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <string>, IReadOnlyList <string> >(
appraisedData => appraisedData
).ToDataflow(DataflowOptions.Default);
var usedTypes = new Dictionary <Type, DataBroadcaster <IReadOnlyList <IData> > >();
foreach (var appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts specified data of type {appraiser.DataType.Name}. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
l => l,
l => l.All(d => d.GetType().IsAssignableFrom(appraiser.DataType))
);
RegisterChild(broadcaster);
}
var appraiserFlow = DataflowUtils.FromDelegate(appraiser.Func, DataflowOptions.Default);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
private void InitFlow(IEnumerable <Action <RatingDataContainer> > outputters)
{
var outputterFlows = outputters.Select(outputters =>
DataflowUtils.FromDelegate(outputters, DataflowOptions.Default)
);
foreach (Dataflow <RatingDataContainer> outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public ComplexIntFlow() : base(DataflowOptions.Default)
{
Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i);
Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1);
Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>(
i => {
if (i % 2 == 0)
{
node2.Post(i);
}
else
{
node3.Post(i);
}
return(999);
});
Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine);
node1.Name = "node1";
node2.Name = "node2";
node3.Name = "node3";
printer.Name = "printer";
this.RegisterChild(node1);
this.RegisterChild(node2);
this.RegisterChild(node3);
this.RegisterChild(printer, t =>
{
if (t.Status == TaskStatus.RanToCompletion)
{
Console.WriteLine("Printer done!");
}
});
node1.LinkTo(printer);
node2.LinkTo(printer);
node3.LinkTo(printer);
//Completion propagation: node1 ---> node2
node2.RegisterDependency(node1);
//Completion propagation: node1 + node2 ---> node3
node3.RegisterDependency(node1);
node3.RegisterDependency(node2);
this._headBlock = node1.InputBlock;
}
private void InitFlow(IEnumerable <Func <string, IEnumerable <string> > > inputters)
{
var inputFlows = inputters.Select(inputter =>
DataflowUtils.FromDelegate(inputter, DataflowOptions.Default)
);
foreach (Dataflow <string, string> inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer, FilterInputData);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
public async Task TestAutoComplete2()
{
var block2 = new TransformManyBlock <Int, Int>(i =>
{
return(Enumerable.Empty <Int>());
});
var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => new[] { i }); //preserve the guid
var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(2)));
Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete"));
}
private void InitFlow(IEnumerable <Funcotype> appraisers)
{
var usedTypes = new Dictionary <Type, DataBroadcaster <BasicInfo> >();
foreach (Funcotype appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <BasicInfo>(
crawlersData => crawlersData, DataflowOptions.Default
);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
info => info,
info => info.GetType().IsAssignableFrom(appraiser.DataType)
);
RegisterChild(broadcaster);
}
Dataflow <BasicInfo, RatingDataContainer> appraiserFlow = DataflowUtils.FromDelegate(
appraiser.Func, DataflowOptions.Default
);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
public InputtersFlow(IEnumerable <Func <string, IReadOnlyList <string> > > inputters)
: base(DataflowOptions.Default)
{
if (inputters is null)
{
throw new ArgumentNullException(nameof(inputters));
}
_filteringSet = new ConcurrentDictionary <string, byte>();
_inputBroadcaster = new DataBroadcaster <string>(input =>
{
Console.WriteLine("Broadcasts input to further blocks.");
return(input);
}, DataflowOptions.Default);
_resultTransformer =
DataflowUtils.FromDelegate <IReadOnlyList <string>, IReadOnlyList <string> >(
FilterInputData, DataflowOptions.Default
);
var inputFlows = inputters
.Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default));
foreach (var inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
