public async Task BatchedJoinToAction()
{
var b = new BatchedJoinBlock <int, int>(1);
int completedCount = 0;
var c = new ActionBlock <Tuple <IList <int>, IList <int> > >(i => completedCount++);
b.LinkTo(c, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < Iterations; i++)
{
if (i % 2 == 0)
{
b.Target1.Post(i);
}
else
{
b.Target2.Post(i);
}
}
b.Target1.Complete();
b.Target2.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public void TestPostThenReceive3()
{
const int Iters = 10;
var block = new BatchedJoinBlock <int, string, int>(3);
for (int i = 0; i < Iters; i++)
{
Tuple <IList <int>, IList <string>, IList <int> > item;
Assert.Equal(expected: 0, actual: block.OutputCount);
block.Target1.Post(i);
Assert.Equal(expected: 0, actual: block.OutputCount);
Assert.False(block.TryReceive(out item));
block.Target2.Post(i.ToString());
Assert.Equal(expected: 0, actual: block.OutputCount);
Assert.False(block.TryReceive(out item));
block.Target3.Post(i);
Assert.Equal(expected: 1, actual: block.OutputCount);
Tuple <IList <int>, IList <string>, IList <int> > msg;
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
Assert.Equal(expected: 1, actual: msg.Item3.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j].ToString(), msg.Item2[j]);
}
}
}
public void TestPostThenReceive2()
{
const int Iters = 10;
var block = new BatchedJoinBlock <int, string>(2);
for (int i = 0; i < Iters; i++)
{
int prevCount = block.OutputCount;
block.Target1.Post(i);
Assert.Equal(expected: prevCount, actual: block.OutputCount);
block.Target2.Post(i.ToString());
if (i % block.BatchSize == 0)
{
Assert.Equal(expected: prevCount + 1, actual: block.OutputCount);
Tuple <IList <int>, IList <string> > msg;
Assert.False(block.TryReceive(f => false, out msg));
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j].ToString(), msg.Item2[j]);
}
}
}
}
public void TestUnbalanced2()
{
const int Iters = 10, NumBatches = 2;
int batchSize = Iters / NumBatches;
var block = new BatchedJoinBlock <string, int>(batchSize);
for (int i = 0; i < Iters; i++)
{
block.Target2.Post(i);
Assert.Equal(expected: (i + 1) / batchSize, actual: block.OutputCount);
}
IList <Tuple <IList <string>, IList <int> > > items;
Assert.True(block.TryReceiveAll(out items));
Assert.Equal(expected: NumBatches, actual: items.Count);
for (int i = 0; i < items.Count; i++)
{
var item = items[i];
Assert.NotNull(item.Item1);
Assert.NotNull(item.Item2);
Assert.Equal(expected: batchSize, actual: item.Item2.Count);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: (i * batchSize) + j, actual: item.Item2[j]);
}
}
Assert.False(block.TryReceiveAll(out items));
}
public void TestPostAllThenReceive()
{
const int Iters = 10;
var block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.Equal(expected: Iters, actual: block.OutputCount);
for (int i = 0; i < block.OutputCount; i++)
{
Tuple <IList <int>, IList <int> > msg;
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j], msg.Item2[j]);
}
}
}
public void MaxNumberOfGroupsTest()
{
var scheduler = new TestScheduler();
var block = new BatchedJoinBlock <int, int> (1,
new GroupingDataflowBlockOptions
{
MaxNumberOfGroups = 2, TaskScheduler = scheduler
});
Assert.IsTrue(block.Target1.Post(1));
Assert.IsTrue(block.Target2.Post(2));
Assert.IsFalse(block.Target2.Post(3));
Assert.IsFalse(block.Target1.Post(4));
Tuple <IList <int>, IList <int> > batch;
Assert.IsTrue(block.TryReceive(out batch));
CollectionAssert.AreEqual(new[] { 1 }, batch.Item1);
CollectionAssert.IsEmpty(batch.Item2);
Assert.IsTrue(block.TryReceive(out batch));
CollectionAssert.IsEmpty(batch.Item1);
CollectionAssert.AreEqual(new[] { 2 }, batch.Item2);
scheduler.ExecuteAll();
Assert.IsTrue(block.Completion.Wait(100));
}
public void BasicUsageTest()
{
Tuple <IList <int>, IList <int> > result = null;
var evt = new ManualResetEventSlim(false);
var actionBlock = new ActionBlock <Tuple <IList <int>, IList <int> > > (r =>
{
result = r;
evt.Set();
});
var block = new BatchedJoinBlock <int, int> (2);
block.LinkTo(actionBlock);
// both targets once
Assert.IsTrue(block.Target1.Post(1));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target2.Post(2));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 1 }, result.Item1);
CollectionAssert.AreEqual(new[] { 2 }, result.Item2);
result = null;
evt.Reset();
// target 1 twice
Assert.IsTrue(block.Target1.Post(3));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target1.Post(4));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 3, 4 }, result.Item1);
CollectionAssert.IsEmpty(result.Item2);
result = null;
evt.Reset();
// target 2 twice
Assert.IsTrue(block.Target2.Post(5));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target2.Post(6));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.IsEmpty(result.Item1);
CollectionAssert.AreEqual(new[] { 5, 6 }, result.Item2);
}
public void TestUnbalanced3()
{
const int Iters = 10, NumBatches = 2;
int batchSize = Iters / NumBatches;
Tuple <IList <int>, IList <string>, IList <double> > item;
var block = new BatchedJoinBlock <int, string, double>(batchSize);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
Assert.Equal(expected: (i + 1) / batchSize, actual: block.OutputCount);
}
for (int i = 0; i < NumBatches; i++)
{
Assert.True(block.TryReceive(out item));
Assert.NotNull(item.Item1);
Assert.NotNull(item.Item2);
Assert.NotNull(item.Item3);
Assert.Equal(expected: batchSize, actual: item.Item1.Count);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: (i * batchSize) + j, actual: item.Item1[j]);
}
}
Assert.False(block.TryReceive(out item));
}
public void BasicUsageTest()
{
Tuple <IList <int>, IList <int>, IList <string> > result = null;
var evt = new ManualResetEventSlim(false);
var actionBlock =
new ActionBlock <Tuple <IList <int>, IList <int>, IList <string> > > (r =>
{
result = r;
evt.Set();
});
var block = new BatchedJoinBlock <int, int, string> (3);
block.LinkTo(actionBlock);
// all targets once
Assert.IsTrue(block.Target1.Post(1));
Assert.IsTrue(block.Target2.Post(2));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target3.Post("foo"));
Assert.IsTrue(evt.Wait(1000));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 1 }, result.Item1);
CollectionAssert.AreEqual(new[] { 2 }, result.Item2);
CollectionAssert.AreEqual(new[] { "foo" }, result.Item3);
}
public void TestCompletion()
{
const int Iters = 10;
var block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.Equal(expected: Iters, actual: block.OutputCount);
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
Assert.Equal(expected: Iters + 1, actual: block.OutputCount);
Tuple <IList <int>, IList <int> > item;
for (int i = 0; i < Iters; i++)
{
Assert.True(block.TryReceive(out item));
Assert.Equal(expected: 1, actual: item.Item1.Count);
Assert.Equal(expected: 1, actual: item.Item2.Count);
}
Assert.True(block.TryReceive(out item));
Assert.Equal(expected: 1, actual: item.Item1.Count);
Assert.Equal(expected: 0, actual: item.Item2.Count);
}
public void BasicUsageTest()
{
Tuple<IList<int>, IList<int>> result = null;
var evt = new ManualResetEventSlim (false);
var actionBlock = new ActionBlock<Tuple<IList<int>, IList<int>>> (r =>
{
result = r;
evt.Set ();
});
var block = new BatchedJoinBlock<int, int> (2);
block.LinkTo (actionBlock);
// both targets once
Assert.IsTrue (block.Target1.Post (1));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target2.Post (2));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 1 }, result.Item1);
CollectionAssert.AreEqual (new[] { 2 }, result.Item2);
result = null;
evt.Reset ();
// target 1 twice
Assert.IsTrue (block.Target1.Post (3));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target1.Post (4));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 3, 4 }, result.Item1);
CollectionAssert.IsEmpty (result.Item2);
result = null;
evt.Reset ();
// target 2 twice
Assert.IsTrue (block.Target2.Post (5));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target2.Post (6));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.IsEmpty (result.Item1);
CollectionAssert.AreEqual (new[] { 5, 6 }, result.Item2);
}
/// <summary>
/// Creates a dataflow builder from a batched join of multiple source blocks.
/// </summary>
/// <typeparam name="TOutput1"></typeparam>
/// <typeparam name="TOutput2"></typeparam>
/// <typeparam name="TOutput3"></typeparam>
/// <param name="sourceBlock1"></param>
/// <param name="sourceBlock2"></param>
/// <param name="sourceBlock3"></param>
/// <param name="batchSize"></param>
/// <param name="joinOptions"></param>
/// <returns></returns>
public ISourceDataflowBuilder <Tuple <IList <TOutput1>, IList <TOutput2>, IList <TOutput3> > > BatchedJoin <TOutput1, TOutput2, TOutput3>(ISourceBlock <TOutput1> sourceBlock1, ISourceBlock <TOutput2> sourceBlock2, ISourceBlock <TOutput3> sourceBlock3, int batchSize, DataflowJoinOptions joinOptions = default(DataflowJoinOptions))
{
if (sourceBlock1 == null)
{
throw new ArgumentNullException("sourceBlock1");
}
if (sourceBlock2 == null)
{
throw new ArgumentNullException("sourceBlock2");
}
if (sourceBlock3 == null)
{
throw new ArgumentNullException("sourceBlock3");
}
var batchedJoinBlock = new BatchedJoinBlock <TOutput1, TOutput2, TOutput3>(batchSize, joinOptions.JoinBlockOptions);
LinkHelper.Link(sourceBlock1, batchedJoinBlock.Target1, joinOptions.Target1LinkOptions);
LinkHelper.Link(sourceBlock2, batchedJoinBlock.Target2, joinOptions.Target2LinkOptions);
LinkHelper.Link(sourceBlock3, batchedJoinBlock.Target3, joinOptions.Target3LinkOptions);
var multipleSourcesWrapper = new MultipleSourceDataflowWrapper(new IDataflowBlock[] { sourceBlock1, sourceBlock2, sourceBlock3 });
var sourceWrapper = new ReceivableSourceDataflowWrapper <Tuple <IList <TOutput1>, IList <TOutput2>, IList <TOutput3> > >(multipleSourcesWrapper, batchedJoinBlock, batchedJoinBlock);
return(FromSource(sourceWrapper));
}
public async Task TestPrecanceled3()
{
var b = new BatchedJoinBlock <int, int, int>(42,
new GroupingDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1
});
Tuple <IList <int>, IList <int>, IList <int> > ignoredValue;
IList <Tuple <IList <int>, IList <int>, IList <int> > > ignoredValues;
Assert.NotNull(b.LinkTo(new ActionBlock <Tuple <IList <int>, IList <int>, IList <int> > >(delegate { })));
Assert.False(b.Target1.Post(42));
Assert.False(b.Target2.Post(42));
foreach (var target in new[] { b.Target1, b.Target2 })
{
var t = target.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
}
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.False(b.TryReceive(out ignoredValue));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Target1.Complete();
b.Target2.Complete();
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion);
}
internal static bool BatchedJoinToAction()
{
const int ITERS = 2;
var b = new BatchedJoinBlock <int, int>(1);
int completedCount = 0;
var c = new ActionBlock <Tuple <IList <int>, IList <int> > >(i => completedCount++);
b.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++)
{
if (i % 2 == 0)
{
b.Target1.Post(i);
}
else
{
b.Target2.Post(i);
}
}
b.Target1.Complete();
b.Target2.Complete();
c.Completion.Wait();
return(completedCount == ITERS / b.BatchSize);
}
public async Task TestConsumeToAccept()
{
var wob = new WriteOnceBlock <int>(i => i * 2);
wob.Post(1);
await wob.Completion;
var b2 = new BatchedJoinBlock <int, int>(1);
wob.LinkTo(b2.Target2, new DataflowLinkOptions {
PropagateCompletion = true
});
Tuple <IList <int>, IList <int> > item2 = await b2.ReceiveAsync();
Assert.Equal(expected: 0, actual: item2.Item1.Count);
Assert.Equal(expected: 1, actual: item2.Item2.Count);
b2.Target1.Complete();
var b3 = new BatchedJoinBlock <int, int, int>(1);
wob.LinkTo(b3.Target3, new DataflowLinkOptions {
PropagateCompletion = true
});
Tuple <IList <int>, IList <int>, IList <int> > item3 = await b3.ReceiveAsync();
Assert.Equal(expected: 0, actual: item3.Item1.Count);
Assert.Equal(expected: 0, actual: item3.Item2.Count);
Assert.Equal(expected: 1, actual: item3.Item3.Count);
b3.Target1.Complete();
b3.Target2.Complete();
await Task.WhenAll(b2.Completion, b3.Completion);
}
public void BasicUsageTest ()
{
Tuple<IList<int>, IList<int>, IList<string>> result = null;
var evt = new ManualResetEventSlim (false);
var actionBlock =
new ActionBlock<Tuple<IList<int>, IList<int>, IList<string>>> (r =>
{
result = r;
evt.Set ();
});
var block = new BatchedJoinBlock<int, int, string> (3);
block.LinkTo (actionBlock);
// all targets once
Assert.IsTrue (block.Target1.Post (1));
Assert.IsTrue (block.Target2.Post (2));
Assert.IsFalse (evt.Wait (100));
Assert.IsNull (result);
Assert.IsTrue (block.Target3.Post ("foo"));
Assert.IsTrue (evt.Wait (1000));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 1 }, result.Item1);
CollectionAssert.AreEqual (new[] { 2 }, result.Item2);
CollectionAssert.AreEqual (new[] { "foo" }, result.Item3);
}
public async Task TestCompletesThroughTargets()
{
var b2 = new BatchedJoinBlock <int, int>(99);
b2.Target1.Post(1);
b2.Target1.Complete();
b2.Target2.Complete();
Tuple <IList <int>, IList <int> > item2 = await b2.ReceiveAsync();
Assert.Equal(expected: 1, actual: item2.Item1.Count);
Assert.Equal(expected: 0, actual: item2.Item2.Count);
await b2.Completion;
var b3 = new BatchedJoinBlock <int, int, int>(99);
b3.Target2.Post(1);
b3.Target3.Complete();
b3.Target2.Complete();
b3.Target1.Complete();
Tuple <IList <int>, IList <int>, IList <int> > item3 = await b3.ReceiveAsync();
Assert.Equal(expected: 0, actual: item3.Item1.Count);
Assert.Equal(expected: 1, actual: item3.Item2.Count);
Assert.Equal(expected: 0, actual: item3.Item3.Count);
await b3.Completion;
}
public async Task TestReserveReleaseConsume()
{
var b2 = new BatchedJoinBlock <int, int>(2);
b2.Target1.Post(1);
b2.Target2.Post(2);
await DataflowTestHelpers.TestReserveAndRelease(b2);
b2 = new BatchedJoinBlock <int, int>(2);
b2.Target1.Post(1);
b2.Target2.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(b2);
var b3 = new BatchedJoinBlock <int, int, int>(1);
b3.Target2.Post(3);
await DataflowTestHelpers.TestReserveAndRelease(b3);
b3 = new BatchedJoinBlock <int, int, int>(4);
b3.Target3.Post(1);
b3.Target3.Post(2);
b3.Target3.Post(3);
b3.Target2.Post(3);
await DataflowTestHelpers.TestReserveAndConsume(b3);
}
public async Task TestMaxNumberOfGroups()
{
const int MaxGroups = 2;
var b2 = new BatchedJoinBlock <int, int>(1, new GroupingDataflowBlockOptions {
MaxNumberOfGroups = MaxGroups
});
b2.Target1.PostRange(0, MaxGroups);
Assert.False(b2.Target1.Post(42));
Assert.False(b2.Target2.Post(42));
IList <Tuple <IList <int>, IList <int> > > items2;
Assert.True(b2.TryReceiveAll(out items2));
Assert.Equal(expected: MaxGroups, actual: items2.Count);
await b2.Completion;
var b3 = new BatchedJoinBlock <int, int, int>(1, new GroupingDataflowBlockOptions {
MaxNumberOfGroups = MaxGroups
});
b3.Target1.PostRange(0, MaxGroups);
Assert.False(b3.Target1.Post(42));
Assert.False(b3.Target2.Post(42));
Assert.False(b3.Target3.Post(42));
IList <Tuple <IList <int>, IList <int>, IList <int> > > items3;
Assert.True(b3.TryReceiveAll(out items3));
Assert.Equal(expected: MaxGroups, actual: items3.Count);
await b3.Completion;
}
public override void Run()
{
// 注意：每 2 个数据为一组，每 3 组数据组成一批，所以 batchSize = 2 * 3
BatchedJoinBlock <RunModel, string> batchedJoinBlock = new BatchedJoinBlock <RunModel, string>(2 * 3);
var models = this.CreateCollection();
Parallel.For(0, 10, (index) =>
{
// 注意：并发操作下，必须 lock，否则组内的数据可能因为顺序而匹配为错误的组
lock (batchedJoinBlock)
{
batchedJoinBlock.Target1.Post(models[index]);
batchedJoinBlock.Target2.Post($"posted as {index}");
}
});
batchedJoinBlock.Complete();
// BatchedJoinBlock 每个 Target 的数据必须匹配为完整组且组的数量足够组成一次批处理时才可以触发处理逻辑
for (int index = 1; index <= 4; index++)
{
var result = batchedJoinBlock.Receive();
Helper.PrintLine($"第 {index} 批：元素数量={result.Item1.Count}, {result.Item2.Count}");
for (int i = 0; i < result.Item1.Count; i++)
{
Helper.PrintLine($"{result.Item1[i].Name} {result.Item2[i]}");
}
}
}
public void Run()
{
var broadcastBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Mesaj {a} a1 tarafından işlenilmekte.");
Task.Delay(300).Wait();
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(-a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Mesaj {a} a2 tarafından işlenilmekte.");
Task.Delay(150).Wait();
return(a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1);
broadcastBlock.LinkTo(a2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(joinBlock.Target1);
a2.LinkTo(joinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a => Console.WriteLine($"Mesaj: [{string.Join(',', a.Item1)}] , [{string.Join(',', a.Item2)}]"));
joinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
broadcastBlock.SendAsync(i).ContinueWith(a =>
{
if (a.Result)
{
Console.WriteLine($"{i} mesajı kabul edildi.");
}
else
{
Console.WriteLine($"{i} mesajı reddedildi.");
}
});
}
}
// BatchJoinBlocks allow you to optimize on message rate vs. message order.
// MessageCount is what is batched from all the inputs.
//
// The output is difficult to discern but you could have:
// An empty Batch1, an empty Batch2, or both empty!
// A full Batch1, a full Batch2, or both full!
// A full Batch1, a partial/empty Batch2!
// A partial/empty Batch1, a full Batch2!
// Both partially filled!
//
// This is a fast way to consume data, but you will have to add some boiler plate condtions
// to your code to handle all the possible scenarios since it is unpredictable in
// nature.
private static async Task SimpleDemoAsync()
{
Console.WriteLine("BatchJoinBlockDemo has started!");
var bufferExecutionOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 10
};
var transformOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 5
}; // add parallelism back
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
var bufferBlock = new BufferBlock <string>(bufferExecutionOptions);
var broadCastBlock = new BroadcastBlock <string>(
input => input);
var transform1 = new TransformBlock <string, string>(
(input) => $"TB1: {input}",
transformOptions);
var transform2 = new TransformBlock <string, string>(
(input) => $"TB2: {input}",
transformOptions);
bufferBlock.LinkTo(broadCastBlock, options);
broadCastBlock.LinkTo(transform1, options);
broadCastBlock.LinkTo(transform2, options);
var batchJoinBlock = new BatchedJoinBlock <string, string>(2);
transform1.LinkTo(batchJoinBlock.Target1, options); // You have to stitch up where the executions are going in your join block.
transform2.LinkTo(batchJoinBlock.Target2, options);
var actionBlock = new ActionBlock <Tuple <IList <string>, IList <string> > >(
(inputs) => Console.WriteLine($"Batch1: {string.Join(", ", inputs.Item1)}\r\nBatch2: {string.Join(", ", inputs.Item2)}\r\n-------"));
batchJoinBlock.LinkTo(actionBlock, options);
for (int i = 0; i < 20; i++)
{
await bufferBlock
.SendAsync(ProduceTimeData(i))
.ConfigureAwait(false);
}
bufferBlock.Complete();
//await bufferBlock.Completion.ConfigureAwait(false);
//await broadCastBlock.Completion.ConfigureAwait(false);
//await transform1.Completion.ConfigureAwait(false);
//await transform2.Completion.ConfigureAwait(false);
// Because we are using PropagateCompletion = true and the last block is a single ActionBlock,
// we will wait for all our marbles to reach the finish line with a single line.
await actionBlock.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
private static void TestBatchedJoinBlock()
{
//Func<int, int> DoWork = n =>
//{
// if (n < 0)
// {
// throw new ArgumentOutOfRangeException();
// }
// return n;
//};
//var batchedJoinBlock = new BatchedJoinBlock<int, Exception>(7);
//foreach (int i in new int[] { 5, 6, -7, -22, 13, 55, 0 })
//{
// try
// {
// batchedJoinBlock.Target1.Post(DoWork(i));
// }
// catch (ArgumentOutOfRangeException ex)
// {
// batchedJoinBlock.Target2.Post(ex);
// }
//}
//var res = batchedJoinBlock.Receive();
//foreach (int n in res.Item1)
//{
// Console.WriteLine(n);
//}
//foreach (Exception e in res.Item2)
//{
// Console.WriteLine(e.Message);
//}
var bjb = new BatchedJoinBlock <int, string>(3);
var ab = new ActionBlock <Tuple <IList <int>, IList <string> > >(i =>
{
Console.WriteLine("------------------------------------");
foreach (int m in i.Item1)
{
Console.WriteLine(m);
}
foreach (string s in i.Item2)
{
Console.WriteLine(s);
}
});
bjb.LinkTo(ab);
for (int i = 0; i < 5; i++)
{
bjb.Target1.Post(i);
}
for (int i = 5; i > 0; i--)
{
bjb.Target2.Post(i.ToString());
}
}
public void TestBatchedJoinBlockConstructor()
{
// *** 2-way BatchedJoinBlock ***
// batch size without decline without option
var block = new BatchedJoinBlock <int, int>(42);
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 42, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline without option
block = new BatchedJoinBlock <int, int>(43, new GroupingDataflowBlockOptions {
MaxNumberOfGroups = 1
});
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 43, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline with not cancelled token and default scheduler
block = new BatchedJoinBlock <int, int>(44, new GroupingDataflowBlockOptions {
MaxMessagesPerTask = 1
});
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 44, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// decline with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new BatchedJoinBlock <int, int>(45, new GroupingDataflowBlockOptions {
MaxMessagesPerTask = 1, CancellationToken = token, MaxNumberOfGroups = 1
});
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 45, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// *** 3-way BatchedJoinBlock ***
// batch size without decline without option
var block3 = new BatchedJoinBlock <int, int, int>(42);
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 42, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline without option
block3 = new BatchedJoinBlock <int, int, int>(43, new GroupingDataflowBlockOptions {
MaxNumberOfGroups = 1
});
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 43, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline with not cancelled token and default scheduler
block3 = new BatchedJoinBlock <int, int, int>(44, new GroupingDataflowBlockOptions {
MaxNumberOfGroups = 1
});
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 44, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// decline with a cancelled token and default scheduler
token = new CancellationToken(true);
block3 = new BatchedJoinBlock <int, int, int>(45, new GroupingDataflowBlockOptions {
MaxMessagesPerTask = 1, CancellationToken = token, MaxNumberOfGroups = 1
});
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 45, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
}
public async Task SimpleBatchedJoin()
{
var block = new BroadcastBlock <int>(a => a);
Random random = new Random();
var action1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Processed by consumer 1 : {a}");
Task.Delay(random.Next(0, 1000)).Wait();
return(a * -1);
}, new ExecutionDataflowBlockOptions());
var action2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Processed by consumer 2 : {a}");
Task.Delay(random.Next(0, 1000)).Wait();
return(a);
}, new ExecutionDataflowBlockOptions());
block.LinkTo(action1);
block.LinkTo(action2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
action1.LinkTo(joinBlock.Target1);
action2.LinkTo(joinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a =>
{
Console.WriteLine($"Processed: [{string.Join(",",a.Item1)}],[{string.Join(",", a.Item2)}]");
});
joinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
await block.SendAsync(i);
Console.WriteLine($"Message {i} was sended");
}
block.Complete();
await block.Completion;
action2.Complete();
action1.Complete();
await action2.Completion;
await action1.Completion;
Console.WriteLine("Done!");
}
public async Task TestOfferMessage3()
{
Func <ITargetBlock <int> > generator = () => {
var b = new BatchedJoinBlock <int, int, int>(1);
return(b.Target1);
};
DataflowTestHelpers.TestOfferMessage_ArgumentValidation(generator());
DataflowTestHelpers.TestOfferMessage_AcceptsDataDirectly(generator());
await DataflowTestHelpers.TestOfferMessage_AcceptsViaLinking(generator());
}
private static BatchedJoinBlock <int, int> ConstructBatchedJoin2NewWithNMessages(int messagesCount)
{
var block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < messagesCount; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
SpinWait.SpinUntil(() => block.OutputCount == messagesCount); // spin until messages available
return(block);
}
static void Main(string[] args)
{
var broadcastBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(n => {
Console.WriteLine($"Message {n} received by Consumer 1");
Task.Delay(n % 2 == 0 ? 300 : 100).Wait(); // Join block will still pair them correctly
return(n * -1);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(n => {
Console.WriteLine($"Message {n} received by Consumer 2");
Task.Delay(n % 2 == 0 ? 100 : 300).Wait(); // Join block will still pair them correctly
return(n);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1); // Messages will be proposed in this order
broadcastBlock.LinkTo(a2);
var batchedJoinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(batchedJoinBlock.Target1);
a2.LinkTo(batchedJoinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(
a => Console.WriteLine($"Message {string.Join(",", a.Item1)} : {string.Join(",", a.Item2)}")
);
batchedJoinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
broadcastBlock.SendAsync(i) // List Post, if receivers are not ready then they will be ignored, there is no retry
.ContinueWith(a =>
{
if (a.Result)
{
Console.WriteLine($"Accepted {i}");
}
else
{
Console.WriteLine($"Rejected {i}");
}
});
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
private static void DataStructure_BatchedJoinBlock()
{
// For demonstration, create a Func<int, int> that
// returns its argument, or throws ArgumentOutOfRangeException
// if the argument is less than zero.
Func <int, int> DoWork = n =>
{
if (n < 0)
{
throw new ArgumentOutOfRangeException();
}
return(n);
};
// Create a BatchedJoinBlock<int, Exception> object that holds
// seven elements per batch.
var batchedJoinBlock = new BatchedJoinBlock <int, Exception>(7);
// Post several items to the block.
foreach (int i in new int[] { 5, 6, -7, -22, 13, 55, 0 })
{
try
{
// Post the result of the worker to the
// first target of the block.
batchedJoinBlock.Target1.Post(DoWork(i));
}
catch (ArgumentOutOfRangeException e)
{
// If an error occurred, post the Exception to the
// second target of the block.
batchedJoinBlock.Target2.Post(e);
}
}
// Read the results from the block.
var results = batchedJoinBlock.Receive();
// Print the results to the console.
// Print the results.
foreach (int n in results.Item1)
{
Console.WriteLine(n);
}
// Print failures.
foreach (Exception e in results.Item2)
{
Console.WriteLine(e.Message);
}
}
public async Task TestFaultsThroughTargets()
{
var b2 = new BatchedJoinBlock <int, int>(99);
b2.Target1.Post(1);
((IDataflowBlock)b2.Target1).Fault(new FormatException());
await Assert.ThrowsAsync <FormatException>(() => b2.Completion);
var b3 = new BatchedJoinBlock <int, int, int>(99);
b3.Target3.Post(1);
((IDataflowBlock)b3.Target2).Fault(new FormatException());
await Assert.ThrowsAsync <FormatException>(() => b3.Completion);
}
public SynchronizedJoinBlock(Func <T1, T2, int> compareFunction)
{
_compareFunction = compareFunction
?? throw new ArgumentNullException(nameof(compareFunction));
_batchedJoinBlock = new BatchedJoinBlock <Tuple <T1, T2>, Tuple <T1, T2> >(1);
_target1Messages = new Queue <T1>();
_target2Messages = new Queue <T2>();
var syncObject = new object();
Func <ICollection <Tuple <T1, T2> > > getMessagesFunction = () =>
{
lock (syncObject)
{
if (_target1Messages.Count > 0 && _target2Messages.Count > 0)
{
return(GetMessagesRecursive(_target1Messages.Peek(), _target2Messages.Peek()).ToArray());
}
else
{
return(new Tuple <T1, T2> [0]);
}
}
};
_target1 = new TransformManyBlock <T1, Tuple <T1, T2> >((element) =>
{
_target1Messages.Enqueue(element);
return(getMessagesFunction());
});
_target1.LinkTo(_batchedJoinBlock.Target1, new DataflowLinkOptions()
{
PropagateCompletion = true
});
_target2 = new TransformManyBlock <T2, Tuple <T1, T2> >((element) =>
{
_target2Messages.Enqueue(element);
return(getMessagesFunction());
});
_target2.LinkTo(_batchedJoinBlock.Target2, new DataflowLinkOptions()
{
PropagateCompletion = true
});
_transformManyBlock = new TransformManyBlock <Tuple <IList <Tuple <T1, T2> >, IList <Tuple <T1, T2> > >, Tuple <T1, T2> >(
element => element.Item1.Concat(element.Item2)
);
_batchedJoinBlock.LinkTo(_transformManyBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
private static BatchedJoinBlock<int, int> ConstructBatchedJoin2NewWithNMessages(int messagesCount)
{
var block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < messagesCount; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return block;
}
private static BatchedJoinBlock <int, int> ConstructBatchedJoin2NewWithNMessages(int messagesCount)
{
var block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < messagesCount; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return(block);
}
public void CompletionTest ()
{
var block = new BatchedJoinBlock<int, int, int> (2);
Assert.IsTrue (block.Target1.Post (1));
block.Complete ();
Tuple<IList<int>, IList<int>, IList<int>> batch;
Assert.IsTrue (block.TryReceive (out batch), batch.ToString ());
CollectionAssert.AreEqual (new[] { 1 }, batch.Item1);
CollectionAssert.IsEmpty (batch.Item2);
CollectionAssert.IsEmpty (batch.Item3);
Assert.IsTrue (block.Completion.Wait (1000));
}
public void TestBatchedJoinBlockConstructor()
{
// *** 2-way BatchedJoinBlock ***
// batch size without decline without option
var block = new BatchedJoinBlock<int, int>(42);
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 42, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline without option
block = new BatchedJoinBlock<int, int>(43, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1 });
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 43, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline with not cancelled token and default scheduler
block = new BatchedJoinBlock<int, int>(44, new GroupingDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 44, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// decline with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new BatchedJoinBlock<int, int>(45, new GroupingDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token, MaxNumberOfGroups = 1 });
Assert.False(block.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block.BatchSize != 45, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// *** 3-way BatchedJoinBlock ***
// batch size without decline without option
var block3 = new BatchedJoinBlock<int, int, int>(42);
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 42, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline without option
block3 = new BatchedJoinBlock<int, int, int>(43, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1 });
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 43, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// batch size with decline with not cancelled token and default scheduler
block3 = new BatchedJoinBlock<int, int, int>(44, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1 });
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 44, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
// decline with a cancelled token and default scheduler
token = new CancellationToken(true);
block3 = new BatchedJoinBlock<int, int, int>(45, new GroupingDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token, MaxNumberOfGroups = 1 });
Assert.False(block3.OutputCount != 0, "Constructor failed! OutputCount returned a non zero value for a brand new BatchedJoinBlock.");
Assert.False(block3.BatchSize != 45, "Constructor failed! BatchSize does not match for a brand new BatchedJoinBlock.");
}
public async Task TestConsumeToAccept()
{
var wob = new WriteOnceBlock<int>(i => i * 2);
wob.Post(1);
await wob.Completion;
var b2 = new BatchedJoinBlock<int, int>(1);
wob.LinkTo(b2.Target2, new DataflowLinkOptions { PropagateCompletion = true });
Tuple<IList<int>, IList<int>> item2 = await b2.ReceiveAsync();
Assert.Equal(expected: 0, actual: item2.Item1.Count);
Assert.Equal(expected: 1, actual: item2.Item2.Count);
b2.Target1.Complete();
var b3 = new BatchedJoinBlock<int, int, int>(1);
wob.LinkTo(b3.Target3, new DataflowLinkOptions { PropagateCompletion = true });
Tuple<IList<int>, IList<int>, IList<int>> item3 = await b3.ReceiveAsync();
Assert.Equal(expected: 0, actual: item3.Item1.Count);
Assert.Equal(expected: 0, actual: item3.Item2.Count);
Assert.Equal(expected: 1, actual: item3.Item3.Count);
b3.Target1.Complete();
b3.Target2.Complete();
await Task.WhenAll(b2.Completion, b3.Completion);
}
public void RunBatchedJoinBlockConformanceTests()
{
// Test Post/Receive single block
var block = new BatchedJoinBlock<int, int>(2);
int iter = 10;
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
if (i % block.BatchSize == 0)
{
var msg = block.Receive();
Assert.False(msg.Item1.Count != msg.Item2.Count, "BatchedJoinBlock Post/Receive failed, returned arrays of differnet length");
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock Post/Receive failed, returned arrys items are different");
}
}
}
}
// Test PostAll then Receive single block
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.False(block.OutputCount != iter, string.Format("BatchedJoinBlock Post failed, expected, incorrect OutputCount. Expected {0} actual {1}", iter, block.OutputCount));
for (int i = 0; i < block.OutputCount; i++)
{
var msg = block.Receive();
if (msg.Item1.Count != msg.Item2.Count)
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrays of differnet length");
}
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrys items are different");
}
}
}
//Test one target Post < patchSize msg with TryReceive
block = new BatchedJoinBlock<int, int>(2);
block.Target1.Post(0);
Tuple<IList<int>, IList<int>> result;
if (block.TryReceive(out result))
{
Assert.False(true, "BatchedJoinBlock.TryReceive failed, returned true and the number of messages is less than the batch size");
}
if (block.OutputCount > 0)
{
Assert.False(true, "BatchedJoinBlock.OutputCount failed, returned count > 0 and only one target posted a message");
}
// Test handling of stragglers at end of block's life
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < 10; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
if (block.OutputCount != 11)
{
Assert.False(true, "BatchedJoinBlock last batch not generated correctly");
}
for (int i = 0; i < 10; i++) block.Receive();
var lastResult = block.Receive();
if (lastResult.Item1.Count != 1 || lastResult.Item2.Count != 0)
{
Assert.False(true, "BatchedJoinBlock last batch contains incorrect data");
}
// Test BatchedJoinBlock`2 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int>(42, dbo);
Tuple<IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`2");
}
// Test BatchedJoinBlock`3 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int, int>(42, dbo);
Tuple<IList<int>, IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target3.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.Target3.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
bjb.Target3.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`3");
}
// Test BatchedJoinBlock`2 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
batchedJoin.Target3.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`2 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
}
internal static bool BatchedJoinToAction()
{
const int ITERS = 2;
var b = new BatchedJoinBlock<int, int>(1);
int completedCount = 0;
var c = new ActionBlock<Tuple<IList<int>, IList<int>>>(i => completedCount++);
b.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++)
{
if (i % 2 == 0) b.Target1.Post(i);
else b.Target2.Post(i);
}
b.Target1.Complete();
b.Target2.Complete();
c.Completion.Wait();
return (completedCount == ITERS / b.BatchSize);
}
public void TestPostThenReceive2()
{
const int Iters = 10;
var block = new BatchedJoinBlock<int, string>(2);
for (int i = 0; i < Iters; i++)
{
int prevCount = block.OutputCount;
block.Target1.Post(i);
Assert.Equal(expected: prevCount, actual: block.OutputCount);
block.Target2.Post(i.ToString());
if (i % block.BatchSize == 0)
{
Assert.Equal(expected: prevCount + 1, actual: block.OutputCount);
Tuple<IList<int>, IList<string>> msg;
Assert.False(block.TryReceive(f => false, out msg));
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j].ToString(), msg.Item2[j]);
}
}
}
}
public async Task TestMaxNumberOfGroups()
{
const int MaxGroups = 2;
var b2 = new BatchedJoinBlock<int, int>(1, new GroupingDataflowBlockOptions { MaxNumberOfGroups = MaxGroups });
b2.Target1.PostRange(0, MaxGroups);
Assert.False(b2.Target1.Post(42));
Assert.False(b2.Target2.Post(42));
IList<Tuple<IList<int>, IList<int>>> items2;
Assert.True(b2.TryReceiveAll(out items2));
Assert.Equal(expected: MaxGroups, actual: items2.Count);
await b2.Completion;
var b3 = new BatchedJoinBlock<int, int, int>(1, new GroupingDataflowBlockOptions { MaxNumberOfGroups = MaxGroups });
b3.Target1.PostRange(0, MaxGroups);
Assert.False(b3.Target1.Post(42));
Assert.False(b3.Target2.Post(42));
Assert.False(b3.Target3.Post(42));
IList<Tuple<IList<int>, IList<int>, IList<int>>> items3;
Assert.True(b3.TryReceiveAll(out items3));
Assert.Equal(expected: MaxGroups, actual: items3.Count);
await b3.Completion;
}
public async Task TestOfferMessage3()
{
Func<ITargetBlock<int>> generator = () => {
var b = new BatchedJoinBlock<int, int, int>(1);
return b.Target1;
};
DataflowTestHelpers.TestOfferMessage_ArgumentValidation(generator());
DataflowTestHelpers.TestOfferMessage_AcceptsDataDirectly(generator());
await DataflowTestHelpers.TestOfferMessage_AcceptsViaLinking(generator());
}
public void TestPostAllThenReceive()
{
const int Iters = 10;
var block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.Equal(expected: Iters, actual: block.OutputCount);
for (int i = 0; i < block.OutputCount; i++)
{
Tuple<IList<int>, IList<int>> msg;
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j], msg.Item2[j]);
}
}
}
public void TestUnbalanced3()
{
const int Iters = 10, NumBatches = 2;
int batchSize = Iters / NumBatches;
Tuple<IList<int>, IList<string>, IList<double>> item;
var block = new BatchedJoinBlock<int, string, double>(batchSize);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
Assert.Equal(expected: (i + 1) / batchSize, actual: block.OutputCount);
}
for (int i = 0; i < NumBatches; i++)
{
Assert.True(block.TryReceive(out item));
Assert.NotNull(item.Item1);
Assert.NotNull(item.Item2);
Assert.NotNull(item.Item3);
Assert.Equal(expected: batchSize, actual: item.Item1.Count);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: (i * batchSize) + j, actual: item.Item1[j]);
}
}
Assert.False(block.TryReceive(out item));
}
public void TestCompletion()
{
const int Iters = 10;
var block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < Iters; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.Equal(expected: Iters, actual: block.OutputCount);
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
Assert.Equal(expected: Iters + 1, actual: block.OutputCount);
Tuple<IList<int>, IList<int>> item;
for (int i = 0; i < Iters; i++)
{
Assert.True(block.TryReceive(out item));
Assert.Equal(expected: 1, actual: item.Item1.Count);
Assert.Equal(expected: 1, actual: item.Item2.Count);
}
Assert.True(block.TryReceive(out item));
Assert.Equal(expected: 1, actual: item.Item1.Count);
Assert.Equal(expected: 0, actual: item.Item2.Count);
}
public async Task TestReserveReleaseConsume()
{
var b2 = new BatchedJoinBlock<int, int>(2);
b2.Target1.Post(1);
b2.Target2.Post(2);
await DataflowTestHelpers.TestReserveAndRelease(b2);
b2 = new BatchedJoinBlock<int, int>(2);
b2.Target1.Post(1);
b2.Target2.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(b2);
var b3 = new BatchedJoinBlock<int, int, int>(1);
b3.Target2.Post(3);
await DataflowTestHelpers.TestReserveAndRelease(b3);
b3 = new BatchedJoinBlock<int, int, int>(4);
b3.Target3.Post(1);
b3.Target3.Post(2);
b3.Target3.Post(3);
b3.Target2.Post(3);
await DataflowTestHelpers.TestReserveAndConsume(b3);
}
public void TestUnbalanced2()
{
const int Iters = 10, NumBatches = 2;
int batchSize = Iters / NumBatches;
var block = new BatchedJoinBlock<string, int>(batchSize);
for (int i = 0; i < Iters; i++)
{
block.Target2.Post(i);
Assert.Equal(expected: (i + 1) / batchSize, actual: block.OutputCount);
}
IList<Tuple<IList<string>, IList<int>>> items;
Assert.True(block.TryReceiveAll(out items));
Assert.Equal(expected: NumBatches, actual: items.Count);
for (int i = 0; i < items.Count; i++)
{
var item = items[i];
Assert.NotNull(item.Item1);
Assert.NotNull(item.Item2);
Assert.Equal(expected: batchSize, actual: item.Item2.Count);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: (i * batchSize) + j, actual: item.Item2[j]);
}
}
Assert.False(block.TryReceiveAll(out items));
}
public async Task TestCompletesThroughBlock()
{
var b2 = new BatchedJoinBlock<int, int>(99);
b2.Target1.Post(1);
b2.Complete();
Tuple<IList<int>, IList<int>> item2 = await b2.ReceiveAsync();
Assert.Equal(expected: 1, actual: item2.Item1.Count);
Assert.Equal(expected: 0, actual: item2.Item2.Count);
await b2.Completion;
var b3 = new BatchedJoinBlock<int, int, int>(99);
b3.Target3.Post(1);
b3.Complete();
Tuple<IList<int>, IList<int>, IList<int>> item3 = await b3.ReceiveAsync();
Assert.Equal(expected: 0, actual: item3.Item1.Count);
Assert.Equal(expected: 0, actual: item3.Item2.Count);
Assert.Equal(expected: 1, actual: item3.Item3.Count);
await b3.Completion;
}
public void TestPostThenReceive3()
{
const int Iters = 10;
var block = new BatchedJoinBlock<int, string, int>(3);
for (int i = 0; i < Iters; i++)
{
Tuple<IList<int>, IList<string>, IList<int>> item;
Assert.Equal(expected: 0, actual: block.OutputCount);
block.Target1.Post(i);
Assert.Equal(expected: 0, actual: block.OutputCount);
Assert.False(block.TryReceive(out item));
block.Target2.Post(i.ToString());
Assert.Equal(expected: 0, actual: block.OutputCount);
Assert.False(block.TryReceive(out item));
block.Target3.Post(i);
Assert.Equal(expected: 1, actual: block.OutputCount);
Tuple<IList<int>, IList<string>, IList<int>> msg;
Assert.True(block.TryReceive(out msg));
Assert.Equal(expected: 1, actual: msg.Item1.Count);
Assert.Equal(expected: 1, actual: msg.Item2.Count);
Assert.Equal(expected: 1, actual: msg.Item3.Count);
for (int j = 0; j < msg.Item1.Count; j++)
{
Assert.Equal(msg.Item1[j].ToString(), msg.Item2[j]);
}
}
}
public async Task TestFaultsThroughTargets()
{
var b2 = new BatchedJoinBlock<int, int>(99);
b2.Target1.Post(1);
((IDataflowBlock)b2.Target1).Fault(new FormatException());
await Assert.ThrowsAsync<FormatException>(() => b2.Completion);
var b3 = new BatchedJoinBlock<int, int, int>(99);
b3.Target3.Post(1);
((IDataflowBlock)b3.Target2).Fault(new FormatException());
await Assert.ThrowsAsync<FormatException>(() => b3.Completion);
}
public async Task TestPrecanceled3()
{
var b = new BatchedJoinBlock<int, int, int>(42,
new GroupingDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1 });
Tuple<IList<int>, IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues;
Assert.NotNull(b.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })));
Assert.False(b.Target1.Post(42));
Assert.False(b.Target2.Post(42));
foreach (var target in new[] { b.Target1, b.Target2 })
{
var t = target.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
}
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.False(b.TryReceive(out ignoredValue));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Target1.Complete();
b.Target2.Complete();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion);
}
public void TestBatchedJoinInvalidArgumentValidation()
{
Assert.Throws<ArgumentOutOfRangeException>(() => new BatchedJoinBlock<int, int>(0));
Assert.Throws<ArgumentNullException>(() => new BatchedJoinBlock<int, int>(1, null));
Assert.Throws<ArgumentException>(() => new BatchedJoinBlock<int, int>(1, new GroupingDataflowBlockOptions { Greedy = false }));
Assert.Throws<NotSupportedException>(() => { var ignored = new BatchedJoinBlock<int, int>(2).Target1.Completion; });
Assert.True(ISourceBlockTestHelper.TestArgumentsExceptions<Tuple<IList<int>, IList<int>>>(new BatchedJoinBlock<int, int>(2)));
Assert.Throws<ArgumentOutOfRangeException>(() => new BatchedJoinBlock<int, int, int>(0));
Assert.Throws<ArgumentNullException>(() => new BatchedJoinBlock<int, int, int>(1, null));
Assert.Throws<ArgumentException>(() => new BatchedJoinBlock<int, int, int>(1, new GroupingDataflowBlockOptions { Greedy = false }));
Assert.Throws<NotSupportedException>(() => { var ignored = new BatchedJoinBlock<int, int, int>(2).Target3.Completion; });
Assert.True(ISourceBlockTestHelper.TestArgumentsExceptions<Tuple<IList<int>, IList<int>, IList<int>>>(new BatchedJoinBlock<int, int, int>(2)));
}
public async Task BatchedJoinToAction()
{
var b = new BatchedJoinBlock<int, int>(1);
int completedCount = 0;
var c = new ActionBlock<Tuple<IList<int>, IList<int>>>(i => completedCount++);
b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
for (int i = 0; i < Iterations; i++)
{
if (i % 2 == 0) b.Target1.Post(i);
else b.Target2.Post(i);
}
b.Target1.Complete();
b.Target2.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public void MaxNumberOfGroupsTest ()
{
var scheduler = new TestScheduler ();
var block = new BatchedJoinBlock<int, int, int> (1,
new GroupingDataflowBlockOptions
{ MaxNumberOfGroups = 3, TaskScheduler = scheduler });
Assert.IsTrue (block.Target1.Post (1));
Assert.IsTrue (block.Target2.Post (2));
Assert.IsTrue (block.Target3.Post (3));
Assert.IsFalse (block.Target3.Post (4));
Assert.IsFalse (block.Target2.Post (5));
Assert.IsFalse (block.Target1.Post (6));
Tuple<IList<int>, IList<int>, IList<int>> batch;
Assert.IsTrue (block.TryReceive (out batch));
CollectionAssert.AreEqual (new[] { 1 }, batch.Item1);
CollectionAssert.IsEmpty (batch.Item2);
CollectionAssert.IsEmpty (batch.Item3);
Assert.IsTrue (block.TryReceive (out batch));
CollectionAssert.IsEmpty (batch.Item1);
CollectionAssert.AreEqual (new[] { 2 }, batch.Item2);
CollectionAssert.IsEmpty (batch.Item3);
Assert.IsTrue (block.TryReceive (out batch));
CollectionAssert.IsEmpty (batch.Item1);
CollectionAssert.IsEmpty (batch.Item2);
CollectionAssert.AreEqual (new[] { 3 }, batch.Item3);
scheduler.ExecuteAll ();
Assert.IsTrue (block.Completion.Wait (1000));
}
