public async Task TestDataBroadcaster1()
{
var random = new Random();
var dataCopier = new DataBroadcaster <int>();
int sum1 = 0;
int sum2 = 0;
var action1 = new ActionBlock <int>(i => sum1 = sum1 + i);
var action2 = new ActionBlock <int>(i => sum2 = sum2 + i);
dataCopier.LinkTo(DataflowUtils.FromBlock(action1));
dataCopier.LinkTo(DataflowUtils.FromBlock(action2));
for (int j = 0; j < 1000; j++)
{
dataCopier.InputBlock.Post((int)(random.NextDouble() * 10000));
}
dataCopier.InputBlock.Complete();
await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion);
Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2);
Assert.AreEqual(sum1, sum2);
}
public OutputtersFlow(
IEnumerable <Action <IReadOnlyList <string> > > outputters)
: base(DataflowOptions.Default)
{
if (outputters is null)
{
throw new ArgumentNullException(nameof(outputters));
}
_resultList = new ConcurrentBag <string>();
_inputConsumer = new DataBroadcaster <IReadOnlyList <string> >(appraisersData =>
{
Console.WriteLine($"Consuming all appraised data. {appraisersData.Count.ToString()}\n");
foreach (string datum in appraisersData)
{
_resultList.Add(datum);
}
return(appraisersData);
}, DataflowOptions.Default);
var outputterFlows = outputters
.Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default));
foreach (var outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public async Task TestAutoComplete1()
{
var block = new TransformManyBlock <Int, Int>(i =>
{
int j = i.Val + 1;
Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j);
Thread.Sleep(500);
if (j < 10)
{
return new[] { new Int {
Val = j
} }
}
;
else
{
return(Enumerable.Empty <Int>());
}
});
var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => i);
var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(10)));
}
public async Task TestLinkLeftToError()
{
var flow = DataflowUtils.FromBlock(new BufferBlock <object>());
flow.LinkSubTypeTo(DataflowUtils.FromBlock(DataflowBlock.NullTarget <string>()));
flow.LinkLeftToError();
flow.InputBlock.SafePost("abc");
flow.InputBlock.SafePost(new object());
await flow.SignalAndWaitForCompletionAsync();
}
public async Task TestLinkLeftToNull4()
{
var flow = DataflowUtils.FromBlock(new BufferBlock <object>());
flow.LinkSubTypeTo(DataflowUtils.FromBlock(DataflowBlock.NullTarget <string>()));
flow.LinkLeftToNull();
flow.InputBlock.SafePost("abc");
flow.InputBlock.SafePost(new object());
flow.InputBlock.Complete();
await flow.CompletionTask;
}
public AppraisersFlow(IEnumerable <Appraiser> appraisers)
: base(DataflowOptions.Default)
{
if (appraisers is null)
{
throw new ArgumentNullException(nameof(appraisers));
}
_inputConsumer = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts all crawlers data. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <string>, IReadOnlyList <string> >(
appraisedData => appraisedData
).ToDataflow(DataflowOptions.Default);
var usedTypes = new Dictionary <Type, DataBroadcaster <IReadOnlyList <IData> > >();
foreach (var appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts specified data of type {appraiser.DataType.Name}. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
l => l,
l => l.All(d => d.GetType().IsAssignableFrom(appraiser.DataType))
);
RegisterChild(broadcaster);
}
var appraiserFlow = DataflowUtils.FromDelegate(appraiser.Func, DataflowOptions.Default);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
private void InitFlow(IEnumerable <Action <RatingDataContainer> > outputters)
{
var outputterFlows = outputters.Select(outputters =>
DataflowUtils.FromDelegate(outputters, DataflowOptions.Default)
);
foreach (Dataflow <RatingDataContainer> outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public ComplexIntFlow() : base(DataflowOptions.Default)
{
Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i);
Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1);
Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>(
i => {
if (i % 2 == 0)
{
node2.Post(i);
}
else
{
node3.Post(i);
}
return(999);
});
Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine);
node1.Name = "node1";
node2.Name = "node2";
node3.Name = "node3";
printer.Name = "printer";
this.RegisterChild(node1);
this.RegisterChild(node2);
this.RegisterChild(node3);
this.RegisterChild(printer, t =>
{
if (t.Status == TaskStatus.RanToCompletion)
{
Console.WriteLine("Printer done!");
}
});
node1.LinkTo(printer);
node2.LinkTo(printer);
node3.LinkTo(printer);
//Completion propagation: node1 ---> node2
node2.RegisterDependency(node1);
//Completion propagation: node1 + node2 ---> node3
node3.RegisterDependency(node1);
node3.RegisterDependency(node2);
this._headBlock = node1.InputBlock;
}
private void InitFlow(IEnumerable <Func <string, IEnumerable <string> > > inputters)
{
var inputFlows = inputters.Select(inputter =>
DataflowUtils.FromDelegate(inputter, DataflowOptions.Default)
);
foreach (Dataflow <string, string> inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer, FilterInputData);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
public AutoCompleteWrapper(Dataflow <TIn, TOut> dataflow, TimeSpan processTimeout, DataflowOptions options) : base(options)
{
m_Dataflow = dataflow;
m_processTimeout = processTimeout;
m_timer = new Timer();
m_timer.Interval = m_processTimeout.TotalMilliseconds;
m_timer.Elapsed += OnTimerElapsed;
var before = new TransformBlock <TIn, TIn>(@in =>
{
if (m_last == null || @in.UniqueId == m_last.Value)
{
//The last one is back, so there is nothing else in the pipeline.
//Set a timer: if nothing new produced when timer expires, the whole loop ends.
m_timer.Start();
}
return(@in);
});
m_before = DataflowUtils.FromBlock(before);
var after = new TransformBlock <TOut, TOut>(@out =>
{
if (@out.UniqueId != Guid.Empty)
{
m_last = @out.UniqueId;
m_timer.Stop();
}
else
{
LogHelper.Logger.WarnFormat("Empty guid found in output. You may have forgotten to set it.");
}
return(@out);
});
m_after = DataflowUtils.FromBlock(after);
m_before.GoTo(dataflow).GoTo(m_after);
RegisterChild(m_before);
RegisterChild(dataflow);
RegisterChild(m_after);
}
public async Task TestAutoComplete2()
{
var block2 = new TransformManyBlock <Int, Int>(i =>
{
return(Enumerable.Empty <Int>());
});
var dataflow1 = DataflowUtils.FromDelegate <Int, Int>(i => new[] { i }); //preserve the guid
var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await dataflow2.Completion.FinishesIn(TimeSpan.FromSeconds(2)));
Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete"));
}
public async Task TestCompletionPropagation2()
{
var block1 = new TransformManyBlock <int, int>(i =>
{
int j = i + 1;
Console.WriteLine("block1: i = {0}, j = {1}", i, j);
if (j < 100)
{
return new[] { j }
}
;
else
{
return(Enumerable.Empty <int>());
}
});
var block2 = new TransformManyBlock <int, int>(i =>
{
int j = i + 1;
Console.WriteLine("block2: i = {0}, j = {1}", i, j);
if (j < 100)
{
return new[] { j }
}
;
else
{
return(Enumerable.Empty <int>());
}
});
var container1 = DataflowUtils.FromBlock(block1);
var container2 = DataflowUtils.FromBlock(block2);
container1.GoTo(container2).GoTo(container1); //circular
container1.InputBlock.Post(1);
await Task.Delay(1000); //IMPORTANT: wait for block work done (nothing left in their input/output queue)
container1.InputBlock.Complete();
Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(1)));
}
public void TestContainerName()
{
var block1 = new TransformBlock <string, string>(i => i);
var block2 = new TransformBlock <string, string>(i => i);
var container1 = DataflowUtils.FromBlock(block1);
var container2 = DataflowUtils.FromBlock(block2);
var container3 = new FaultyBlocks();
var container4 = new FaultyBlocks();
Assert.AreEqual("[PropagatorDataflow<String, String>1]", container1.FullName);
Assert.AreEqual("[PropagatorDataflow<String, String>2]", container2.FullName);
Assert.IsTrue(container3.Name.StartsWith("FaultyBlocks"));
Assert.IsTrue(container4.Name.StartsWith("FaultyBlocks"));
container3.RegisterChild(container1);
container4.RegisterChild(container2);
Assert.IsTrue(container1.FullName.StartsWith("[FaultyBlocks"));
Assert.IsTrue(container2.FullName.StartsWith("[FaultyBlocks"));
}
public SimpleLogReader(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
this.m_parsingBlock = new TransformManyBlock <string, string>(
s =>
{
if (string.IsNullOrEmpty(s))
{
return(Enumerable.Empty <string>());
}
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
return(s.Split(Splitor));
});
this.m_recorder = new StatisticsRecorder(this);
this.m_recordBlock = new ActionBlock <string>(
s =>
{
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
this.m_recorder.RecordEvent(s);
});
var df1 = DataflowUtils.FromBlock(m_parsingBlock);
var df2 = DataflowUtils.FromBlock(m_recordBlock);
df1.LinkTo(df2);
RegisterChild(df1);
RegisterChild(df2);
}
public InputtersFlow(IEnumerable <Func <string, IReadOnlyList <string> > > inputters)
: base(DataflowOptions.Default)
{
if (inputters is null)
{
throw new ArgumentNullException(nameof(inputters));
}
_filteringSet = new ConcurrentDictionary <string, byte>();
_inputBroadcaster = new DataBroadcaster <string>(input =>
{
Console.WriteLine("Broadcasts input to further blocks.");
return(input);
}, DataflowOptions.Default);
_resultTransformer =
DataflowUtils.FromDelegate <IReadOnlyList <string>, IReadOnlyList <string> >(
FilterInputData, DataflowOptions.Default
);
var inputFlows = inputters
.Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default));
foreach (var inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
public async Task TestCompletionPropagation()
{
var block1 = new TransformBlock <int, int>(i => 2 * i);
var block2 = new TransformBlock <int, int>(i => 2 * i);
var container1 = DataflowUtils.FromBlock(block1);
var container2 = DataflowUtils.FromBlock(block2);
bool postTaskDone = false;
container2.RegisterPostDataflowTask(() => Task.Run(
() =>
{
postTaskDone = true;
}));
container1.LinkTo(container2);
container2.LinkLeftToNull();
container1.InputBlock.SafePost(1);
container1.InputBlock.Complete();
Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(1)));
Assert.IsTrue(postTaskDone);
}
private void InitFlow(IEnumerable <Funcotype> appraisers)
{
var usedTypes = new Dictionary <Type, DataBroadcaster <BasicInfo> >();
foreach (Funcotype appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <BasicInfo>(
crawlersData => crawlersData, DataflowOptions.Default
);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
info => info,
info => info.GetType().IsAssignableFrom(appraiser.DataType)
);
RegisterChild(broadcaster);
}
Dataflow <BasicInfo, RatingDataContainer> appraiserFlow = DataflowUtils.FromDelegate(
appraiser.Func, DataflowOptions.Default
);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
