public void BasicUsageTest ()
{
Tuple<IList<int>, IList<int>, IList<string>> result = null;
var evt = new ManualResetEventSlim (false);
var actionBlock =
new ActionBlock<Tuple<IList<int>, IList<int>, IList<string>>> (r =>
{
result = r;
evt.Set ();
});
var block = new BatchedJoinBlock<int, int, string> (3);
block.LinkTo (actionBlock);
// all targets once
Assert.IsTrue (block.Target1.Post (1));
Assert.IsTrue (block.Target2.Post (2));
Assert.IsFalse (evt.Wait (100));
Assert.IsNull (result);
Assert.IsTrue (block.Target3.Post ("foo"));
Assert.IsTrue (evt.Wait (1000));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 1 }, result.Item1);
CollectionAssert.AreEqual (new[] { 2 }, result.Item2);
CollectionAssert.AreEqual (new[] { "foo" }, result.Item3);
}
public void Run()
{
var broadcastBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Mesaj {a} a1 tarafından işlenilmekte.");
Task.Delay(300).Wait();
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(-a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Mesaj {a} a2 tarafından işlenilmekte.");
Task.Delay(150).Wait();
return(a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1);
broadcastBlock.LinkTo(a2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(joinBlock.Target1);
a2.LinkTo(joinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a => Console.WriteLine($"Mesaj: [{string.Join(',', a.Item1)}] , [{string.Join(',', a.Item2)}]"));
joinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
broadcastBlock.SendAsync(i).ContinueWith(a =>
{
if (a.Result)
{
Console.WriteLine($"{i} mesajı kabul edildi.");
}
else
{
Console.WriteLine($"{i} mesajı reddedildi.");
}
});
}
}
public void BasicUsageTest()
{
Tuple <IList <int>, IList <int> > result = null;
var evt = new ManualResetEventSlim(false);
var actionBlock = new ActionBlock <Tuple <IList <int>, IList <int> > > (r =>
{
result = r;
evt.Set();
});
var block = new BatchedJoinBlock <int, int> (2);
block.LinkTo(actionBlock);
// both targets once
Assert.IsTrue(block.Target1.Post(1));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target2.Post(2));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 1 }, result.Item1);
CollectionAssert.AreEqual(new[] { 2 }, result.Item2);
result = null;
evt.Reset();
// target 1 twice
Assert.IsTrue(block.Target1.Post(3));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target1.Post(4));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 3, 4 }, result.Item1);
CollectionAssert.IsEmpty(result.Item2);
result = null;
evt.Reset();
// target 2 twice
Assert.IsTrue(block.Target2.Post(5));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target2.Post(6));
Assert.IsTrue(evt.Wait(100));
Assert.IsNotNull(result);
CollectionAssert.IsEmpty(result.Item1);
CollectionAssert.AreEqual(new[] { 5, 6 }, result.Item2);
}
public void BasicUsageTest()
{
Tuple <IList <int>, IList <int>, IList <string> > result = null;
var evt = new ManualResetEventSlim(false);
var actionBlock =
new ActionBlock <Tuple <IList <int>, IList <int>, IList <string> > > (r =>
{
result = r;
evt.Set();
});
var block = new BatchedJoinBlock <int, int, string> (3);
block.LinkTo(actionBlock);
// all targets once
Assert.IsTrue(block.Target1.Post(1));
Assert.IsTrue(block.Target2.Post(2));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull(result);
Assert.IsTrue(block.Target3.Post("foo"));
Assert.IsTrue(evt.Wait(1000));
Assert.IsNotNull(result);
CollectionAssert.AreEqual(new[] { 1 }, result.Item1);
CollectionAssert.AreEqual(new[] { 2 }, result.Item2);
CollectionAssert.AreEqual(new[] { "foo" }, result.Item3);
}
public async Task BatchedJoinToAction()
{
var b = new BatchedJoinBlock <int, int>(1);
int completedCount = 0;
var c = new ActionBlock <Tuple <IList <int>, IList <int> > >(i => completedCount++);
b.LinkTo(c, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < Iterations; i++)
{
if (i % 2 == 0)
{
b.Target1.Post(i);
}
else
{
b.Target2.Post(i);
}
}
b.Target1.Complete();
b.Target2.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public void BasicUsageTest()
{
Tuple<IList<int>, IList<int>> result = null;
var evt = new ManualResetEventSlim (false);
var actionBlock = new ActionBlock<Tuple<IList<int>, IList<int>>> (r =>
{
result = r;
evt.Set ();
});
var block = new BatchedJoinBlock<int, int> (2);
block.LinkTo (actionBlock);
// both targets once
Assert.IsTrue (block.Target1.Post (1));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target2.Post (2));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 1 }, result.Item1);
CollectionAssert.AreEqual (new[] { 2 }, result.Item2);
result = null;
evt.Reset ();
// target 1 twice
Assert.IsTrue (block.Target1.Post (3));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target1.Post (4));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 3, 4 }, result.Item1);
CollectionAssert.IsEmpty (result.Item2);
result = null;
evt.Reset ();
// target 2 twice
Assert.IsTrue (block.Target2.Post (5));
Assert.IsFalse(evt.Wait(100));
Assert.IsNull (result);
Assert.IsTrue (block.Target2.Post (6));
Assert.IsTrue (evt.Wait (100));
Assert.IsNotNull (result);
CollectionAssert.IsEmpty (result.Item1);
CollectionAssert.AreEqual (new[] { 5, 6 }, result.Item2);
}
public async Task TestPrecanceled3()
{
var b = new BatchedJoinBlock <int, int, int>(42,
new GroupingDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1
});
Tuple <IList <int>, IList <int>, IList <int> > ignoredValue;
IList <Tuple <IList <int>, IList <int>, IList <int> > > ignoredValues;
Assert.NotNull(b.LinkTo(new ActionBlock <Tuple <IList <int>, IList <int>, IList <int> > >(delegate { })));
Assert.False(b.Target1.Post(42));
Assert.False(b.Target2.Post(42));
foreach (var target in new[] { b.Target1, b.Target2 })
{
var t = target.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
}
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.False(b.TryReceive(out ignoredValue));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Target1.Complete();
b.Target2.Complete();
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion);
}
// BatchJoinBlocks allow you to optimize on message rate vs. message order.
// MessageCount is what is batched from all the inputs.
//
// The output is difficult to discern but you could have:
// An empty Batch1, an empty Batch2, or both empty!
// A full Batch1, a full Batch2, or both full!
// A full Batch1, a partial/empty Batch2!
// A partial/empty Batch1, a full Batch2!
// Both partially filled!
//
// This is a fast way to consume data, but you will have to add some boiler plate condtions
// to your code to handle all the possible scenarios since it is unpredictable in
// nature.
private static async Task SimpleDemoAsync()
{
Console.WriteLine("BatchJoinBlockDemo has started!");
var bufferExecutionOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 10
};
var transformOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 5
}; // add parallelism back
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
var bufferBlock = new BufferBlock <string>(bufferExecutionOptions);
var broadCastBlock = new BroadcastBlock <string>(
input => input);
var transform1 = new TransformBlock <string, string>(
(input) => $"TB1: {input}",
transformOptions);
var transform2 = new TransformBlock <string, string>(
(input) => $"TB2: {input}",
transformOptions);
bufferBlock.LinkTo(broadCastBlock, options);
broadCastBlock.LinkTo(transform1, options);
broadCastBlock.LinkTo(transform2, options);
var batchJoinBlock = new BatchedJoinBlock <string, string>(2);
transform1.LinkTo(batchJoinBlock.Target1, options); // You have to stitch up where the executions are going in your join block.
transform2.LinkTo(batchJoinBlock.Target2, options);
var actionBlock = new ActionBlock <Tuple <IList <string>, IList <string> > >(
(inputs) => Console.WriteLine($"Batch1: {string.Join(", ", inputs.Item1)}\r\nBatch2: {string.Join(", ", inputs.Item2)}\r\n-------"));
batchJoinBlock.LinkTo(actionBlock, options);
for (int i = 0; i < 20; i++)
{
await bufferBlock
.SendAsync(ProduceTimeData(i))
.ConfigureAwait(false);
}
bufferBlock.Complete();
//await bufferBlock.Completion.ConfigureAwait(false);
//await broadCastBlock.Completion.ConfigureAwait(false);
//await transform1.Completion.ConfigureAwait(false);
//await transform2.Completion.ConfigureAwait(false);
// Because we are using PropagateCompletion = true and the last block is a single ActionBlock,
// we will wait for all our marbles to reach the finish line with a single line.
await actionBlock.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
private static void TestBatchedJoinBlock()
{
//Func<int, int> DoWork = n =>
//{
// if (n < 0)
// {
// throw new ArgumentOutOfRangeException();
// }
// return n;
//};
//var batchedJoinBlock = new BatchedJoinBlock<int, Exception>(7);
//foreach (int i in new int[] { 5, 6, -7, -22, 13, 55, 0 })
//{
// try
// {
// batchedJoinBlock.Target1.Post(DoWork(i));
// }
// catch (ArgumentOutOfRangeException ex)
// {
// batchedJoinBlock.Target2.Post(ex);
// }
//}
//var res = batchedJoinBlock.Receive();
//foreach (int n in res.Item1)
//{
// Console.WriteLine(n);
//}
//foreach (Exception e in res.Item2)
//{
// Console.WriteLine(e.Message);
//}
var bjb = new BatchedJoinBlock <int, string>(3);
var ab = new ActionBlock <Tuple <IList <int>, IList <string> > >(i =>
{
Console.WriteLine("------------------------------------");
foreach (int m in i.Item1)
{
Console.WriteLine(m);
}
foreach (string s in i.Item2)
{
Console.WriteLine(s);
}
});
bjb.LinkTo(ab);
for (int i = 0; i < 5; i++)
{
bjb.Target1.Post(i);
}
for (int i = 5; i > 0; i--)
{
bjb.Target2.Post(i.ToString());
}
}
public async Task SimpleBatchedJoin()
{
var block = new BroadcastBlock <int>(a => a);
Random random = new Random();
var action1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Processed by consumer 1 : {a}");
Task.Delay(random.Next(0, 1000)).Wait();
return(a * -1);
}, new ExecutionDataflowBlockOptions());
var action2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Processed by consumer 2 : {a}");
Task.Delay(random.Next(0, 1000)).Wait();
return(a);
}, new ExecutionDataflowBlockOptions());
block.LinkTo(action1);
block.LinkTo(action2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
action1.LinkTo(joinBlock.Target1);
action2.LinkTo(joinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a =>
{
Console.WriteLine($"Processed: [{string.Join(",",a.Item1)}],[{string.Join(",", a.Item2)}]");
});
joinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
await block.SendAsync(i);
Console.WriteLine($"Message {i} was sended");
}
block.Complete();
await block.Completion;
action2.Complete();
action1.Complete();
await action2.Completion;
await action1.Completion;
Console.WriteLine("Done!");
}
static void Main(string[] args)
{
var broadcastBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(n => {
Console.WriteLine($"Message {n} received by Consumer 1");
Task.Delay(n % 2 == 0 ? 300 : 100).Wait(); // Join block will still pair them correctly
return(n * -1);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(n => {
Console.WriteLine($"Message {n} received by Consumer 2");
Task.Delay(n % 2 == 0 ? 100 : 300).Wait(); // Join block will still pair them correctly
return(n);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1); // Messages will be proposed in this order
broadcastBlock.LinkTo(a2);
var batchedJoinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(batchedJoinBlock.Target1);
a2.LinkTo(batchedJoinBlock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(
a => Console.WriteLine($"Message {string.Join(",", a.Item1)} : {string.Join(",", a.Item2)}")
);
batchedJoinBlock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
broadcastBlock.SendAsync(i) // List Post, if receivers are not ready then they will be ignored, there is no retry
.ContinueWith(a =>
{
if (a.Result)
{
Console.WriteLine($"Accepted {i}");
}
else
{
Console.WriteLine($"Rejected {i}");
}
});
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
public static void TestSync()
{
bjb.LinkTo(ab);
for (int i = 0; i < 5; i++)
{
bjb.Target1.Post(i);
}
for (int i = 5; i > 0; i--)
{
bjb.Target2.Post(i.ToString());
}
Console.WriteLine("Finished post");
}
public SynchronizedJoinBlock(Func <T1, T2, int> compareFunction)
{
_compareFunction = compareFunction
?? throw new ArgumentNullException(nameof(compareFunction));
_batchedJoinBlock = new BatchedJoinBlock <Tuple <T1, T2>, Tuple <T1, T2> >(1);
_target1Messages = new Queue <T1>();
_target2Messages = new Queue <T2>();
var syncObject = new object();
Func <ICollection <Tuple <T1, T2> > > getMessagesFunction = () =>
{
lock (syncObject)
{
if (_target1Messages.Count > 0 && _target2Messages.Count > 0)
{
return(GetMessagesRecursive(_target1Messages.Peek(), _target2Messages.Peek()).ToArray());
}
else
{
return(new Tuple <T1, T2> [0]);
}
}
};
_target1 = new TransformManyBlock <T1, Tuple <T1, T2> >((element) =>
{
_target1Messages.Enqueue(element);
return(getMessagesFunction());
});
_target1.LinkTo(_batchedJoinBlock.Target1, new DataflowLinkOptions()
{
PropagateCompletion = true
});
_target2 = new TransformManyBlock <T2, Tuple <T1, T2> >((element) =>
{
_target2Messages.Enqueue(element);
return(getMessagesFunction());
});
_target2.LinkTo(_batchedJoinBlock.Target2, new DataflowLinkOptions()
{
PropagateCompletion = true
});
_transformManyBlock = new TransformManyBlock <Tuple <IList <Tuple <T1, T2> >, IList <Tuple <T1, T2> > >, Tuple <T1, T2> >(
element => element.Item1.Concat(element.Item2)
);
_batchedJoinBlock.LinkTo(_transformManyBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
public IDictionary <string, string[]> BatchedJoinBlockUsage(int numberOfIteration, int batchsize)
{
Console.WriteLine($"Inside {nameof(TplDataflow3GroupingBlocksController)} - {nameof(BatchedJoinBlockUsage)}");
var ouputCollection = new Dictionary <string, string[]>();
Functions.ClearCounterForBatchedJoinBlockUsage();
// Create the members of the pipeline.
var broadCastBlock = new BroadcastBlock <int>(i => i);
var transformBlockDoNothing = new TransformBlock <int, int>(i => i);
var transformBlockSquare = new TransformBlock <int, int>(i => i * i);
var transformBlockMultipleByPi = new TransformBlock <int, double>(i => i * Math.PI);
var batchedJoinBlock = new BatchedJoinBlock <int, int, double>(batchsize);
var processorBlock = new ActionBlock <Tuple <IList <int>, IList <int>, IList <double> > >(tuple =>
Functions.FormatTupleForTheOuputCollection(ouputCollection, tuple)
);
// Connect the dataflow blocks to form a pipeline.
broadCastBlock.LinkTo(transformBlockDoNothing, DataflowOptions.LinkOptions);
broadCastBlock.LinkTo(transformBlockSquare, DataflowOptions.LinkOptions);
broadCastBlock.LinkTo(transformBlockMultipleByPi, DataflowOptions.LinkOptions);
transformBlockDoNothing.LinkTo(batchedJoinBlock.Target1);
transformBlockSquare.LinkTo(batchedJoinBlock.Target2);
transformBlockMultipleByPi.LinkTo(batchedJoinBlock.Target3);
batchedJoinBlock.LinkTo(processorBlock, DataflowOptions.LinkOptions);
// Start BatchedJoinBlockUsage pipeline with the input values.
for (var i = 0; i < numberOfIteration; i++)
{
broadCastBlock.Post(i);
}
// Mark the head of the pipeline as complete.
broadCastBlock.Complete();
// Wait for the last block in the pipeline to process all messages.
Task.WhenAll(transformBlockDoNothing.Completion,
transformBlockSquare.Completion,
transformBlockMultipleByPi.Completion)
.ContinueWith(_ => batchedJoinBlock.Complete());
processorBlock.Completion.Wait();
return(ouputCollection);
}
public static void GetRandomEmployee(int batchSize, int count)
{
Employee em = new Employee();
var selectEmployees = new BatchedJoinBlock <Employee, Exception>(batchSize);
int totalException = 0;
var printEmployees = new ActionBlock <Tuple <IList <Employee>, IList <Exception> > >(data =>
{
Console.WriteLine("Recieve a batch");
foreach (var emp in data.Item1)
{
Console.WriteLine("{0} {1} ID={2}", emp.FirstName, emp.LastName, emp.EmployeeId);
}
Console.WriteLine("There was {0} errors in this batch", data.Item2.Count);
totalException += data.Item2.Count;
});
selectEmployees.LinkTo(printEmployees);
selectEmployees.Completion.ContinueWith(delegate { printEmployees.Complete(); });
Console.WriteLine("Select random employee");
for (int i = 0; i < 10; i++)
{
try
{
Employee ranEmp = em.GetRandomEmployee().FirstOrDefault();
selectEmployees.Target1.Post(ranEmp);
}
catch (NullReferenceException ex)
{
Console.WriteLine("Null Exception {0}", ex.InnerException);
selectEmployees.Target2.Post(ex);
}
}
selectEmployees.Complete();
printEmployees.Completion.Wait();
Console.WriteLine("Finished get employee. There was {0} error", totalException);
}
// Displays information about several random employees to the console.
static void GetRandomEmployees(string connectionString, int batchSize, int count)
{
// Create a BatchedJoinBlock<Employee, Exception> object that holds
// both employee and exception data.
BatchedJoinBlock <Employee, Exception> selectEmployees = new BatchedJoinBlock <Employee, Exception>(batchSize);
// Holds the total number of exceptions that occurred.
int totalErrors = 0;
// Create an action block that prints employee and error information
// to the console.
ActionBlock <Tuple <IList <Employee>, IList <Exception> > > printEmployees =
new ActionBlock <Tuple <IList <Employee>, IList <Exception> > >(data =>
{
// Print information about the employees in this batch.
Console.WriteLine("Received a batch...");
foreach (Employee e in data.Item1)
{
Console.WriteLine("Last={0} First={1} ID={2}",
e.FirstName, e.LastName, e.EmployeeID);
}
// Print the error count for this batch.
Console.WriteLine("There were {0} errors in this batch...",
data.Item2.Count);
// Update total error count.
totalErrors += data.Item2.Count;
});
// Link the batched join block to the action block.
selectEmployees.LinkTo(printEmployees);
// When the batched join block completes, set the action block also to complete.
selectEmployees.Completion.ContinueWith(delegate { printEmployees.Complete(); });
// Try to retrieve the ID for several random employees.
Console.WriteLine("Selecting random entries from Employees table...");
for (int i = 0; i < count; i++)
{
try
{
// Create a random employee.
Employee e = Employee.Random();
// Try to retrieve the ID for the employee from the database.
e.EmployeeID = DatabaseUtilities.GetEmployeeID(e.LastName, e.FirstName, connectionString);
// Post the Employee object to the Employee target of
// the batched join block.
selectEmployees.Target1.Post(e);
}
catch (NullReferenceException e)
{
// GetEmployeeID throws NullReferenceException when there is
// no such employee with the given name. When this happens,
// post the Exception object to the Exception target of
// the batched join block.
selectEmployees.Target2.Post(e);
}
}
// Set the batched join block to the completed state and wait for
// all retrieval operations to complete.
selectEmployees.Complete();
printEmployees.Completion.Wait();
// Print the total error count.
Console.WriteLine("Finished. There were {0} total errors.", totalErrors);
}
public void RunBatchedJoinBlockConformanceTests()
{
// Test Post/Receive single block
var block = new BatchedJoinBlock<int, int>(2);
int iter = 10;
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
if (i % block.BatchSize == 0)
{
var msg = block.Receive();
Assert.False(msg.Item1.Count != msg.Item2.Count, "BatchedJoinBlock Post/Receive failed, returned arrays of differnet length");
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock Post/Receive failed, returned arrys items are different");
}
}
}
}
// Test PostAll then Receive single block
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.False(block.OutputCount != iter, string.Format("BatchedJoinBlock Post failed, expected, incorrect OutputCount. Expected {0} actual {1}", iter, block.OutputCount));
for (int i = 0; i < block.OutputCount; i++)
{
var msg = block.Receive();
if (msg.Item1.Count != msg.Item2.Count)
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrays of differnet length");
}
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrys items are different");
}
}
}
//Test one target Post < patchSize msg with TryReceive
block = new BatchedJoinBlock<int, int>(2);
block.Target1.Post(0);
Tuple<IList<int>, IList<int>> result;
if (block.TryReceive(out result))
{
Assert.False(true, "BatchedJoinBlock.TryReceive failed, returned true and the number of messages is less than the batch size");
}
if (block.OutputCount > 0)
{
Assert.False(true, "BatchedJoinBlock.OutputCount failed, returned count > 0 and only one target posted a message");
}
// Test handling of stragglers at end of block's life
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < 10; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
if (block.OutputCount != 11)
{
Assert.False(true, "BatchedJoinBlock last batch not generated correctly");
}
for (int i = 0; i < 10; i++) block.Receive();
var lastResult = block.Receive();
if (lastResult.Item1.Count != 1 || lastResult.Item2.Count != 0)
{
Assert.False(true, "BatchedJoinBlock last batch contains incorrect data");
}
// Test BatchedJoinBlock`2 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int>(42, dbo);
Tuple<IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`2");
}
// Test BatchedJoinBlock`3 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int, int>(42, dbo);
Tuple<IList<int>, IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target3.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.Target3.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
bjb.Target3.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`3");
}
// Test BatchedJoinBlock`2 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
batchedJoin.Target3.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`2 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
}
public void Test()
{
var action = new ActionBlock <int[]>(x => { }, new ExecutionDataflowBlockOptions {
});
var batch = new BatchBlock <int>(10, new GroupingDataflowBlockOptions {
});
using var likBA = batch.LinkTo(action, new DataflowLinkOptions { }, x => x[0] > 1); //actionBlock gets 10 elements
batch.TriggerBatch(); //for leftover elements
var batchJoined = new BatchedJoinBlock <int[], string>(10, new GroupingDataflowBlockOptions {
});
batch.LinkTo(batchJoined.Target1, new DataflowLinkOptions {
}, ints => ints[0] > 0);
var action2 = new ActionBlock <Tuple <IList <int[]>, IList <string> > >(x => { });
using var linkBJA = batchJoined.LinkTo(action2, new DataflowLinkOptions { }, x => x.Item1.Count == x.Item2.Count);//it's 10 total so 7-3 or 0-10 will propagate
var broadcast = new BroadcastBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
});
var action3 = new ActionBlock <int>(x => { });
var action4 = new ActionBlock <int>(x => { });
broadcast.LinkTo(action3, new DataflowLinkOptions {
}, x => x > 0);
broadcast.LinkTo(action4, new DataflowLinkOptions {
}, x => x > 0); //both will gete same elements
var buffer = new BufferBlock <int>(new DataflowBlockOptions {
});
var action5 = new ActionBlock <int>(x => { });
buffer.LinkTo(action5, new DataflowLinkOptions {
}, x => x == 0);
var buffer1 = new BufferBlock <int>();
var join = new JoinBlock <int, string>(new GroupingDataflowBlockOptions {
});
buffer1.LinkTo(join.Target1);
var action6 = new ActionBlock <Tuple <int, string> >(tlp => { });
join.LinkTo(action6);
var transform = new TransformBlock <int, string>(x => x.ToString(), new ExecutionDataflowBlockOptions {
});
var action7 = new ActionBlock <string>(str => { });
transform.LinkTo(action7);
var transformMany = new TransformManyBlock <int, string>(
x => Enumerable.Range(0, x).Select(i => i.ToString()));//one recived to many output
var action8 = new ActionBlock <string>(str => { });
var writeOnce = new WriteOnceBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
}); // it gets and stores 1 element but gives clones to linked blocks?
var action9 = new ActionBlock <int>(x => { });
writeOnce.LinkTo(action9);
var bufferC1 = new BufferBlock <int>();
var bufferC2 = new BufferBlock <string>();
var actionIdx = DataflowBlock.Choose <int, string>(
bufferC1, x => Console.WriteLine(x + 1),
bufferC2, str => Console.WriteLine(str + ""));
var actionE1 = new ActionBlock <int>(x => { });
var bufferE1 = new BufferBlock <string>();
var propagator = DataflowBlock.Encapsulate <int, string>(actionE1, bufferE1);
var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount, int.MaxValue);
var concurrent = scheduler.ConcurrentScheduler;
var exclusive = scheduler.ExclusiveScheduler;
}
private static async Task Main(string[] args)
{
var bufferBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Message {a} was processed by Consumer 1");
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a * -1);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
}
);
var a2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Message {a} was processed by Consumer 2");
if (a % 2 == 0)
{
Task.Delay(50).Wait();
}
else
{
Task.Delay(300).Wait();
}
return(a);
}
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
}
);
bufferBlock.LinkTo(a1);
bufferBlock.LinkTo(a2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(joinBlock.Target1);
a2.LinkTo(joinBlock.Target2);
var finalBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a =>
{
Console.WriteLine($"Message {a.Item1},{a.Item2} was processed by all consumers. Total was [A=={string.Join(",", a.Item1)}:::::B=={string.Join(",", a.Item2)}] ");
});
joinBlock.LinkTo(finalBlock);
for (var i = 0; i < 10; i++)
{
await bufferBlock.SendAsync(i);
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
static void Main(string[] args)
{
var broadcastBlock = new BroadcastBlock <int>(a => a, new DataflowBlockOptions()
{
BoundedCapacity = 1
});
var a1 = new TransformBlock <int, int>(
a =>
{
Console.WriteLine($"Message {a} was processed by consumer 1");
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a * -1);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(
a =>
{
Console.WriteLine($"Message {a} was processed by consumer 2");
if (a % 2 != 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1);
broadcastBlock.LinkTo(a2);
var joinblock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(joinblock.Target1);
a2.LinkTo(joinblock.Target2);
var printBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(
a => Console.WriteLine($"Message [{string.Join(",", a.Item1)}]" +
$",[{string.Join(",", a.Item2)}]"));
joinblock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
var i1 = i;
broadcastBlock.SendAsync(i)
.ContinueWith(a =>
{
Console.WriteLine(a.Result ? $"Messgae {i1} was accepted" : $"Messgae {i1} was rejected");
});
}
Console.WriteLine("done");
Console.ReadLine();
}
public async Task TestPrecanceled3()
{
var b = new BatchedJoinBlock<int, int, int>(42,
new GroupingDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1 });
Tuple<IList<int>, IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues;
Assert.NotNull(b.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })));
Assert.False(b.Target1.Post(42));
Assert.False(b.Target2.Post(42));
foreach (var target in new[] { b.Target1, b.Target2 })
{
var t = target.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
}
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.False(b.TryReceive(out ignoredValue));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Target1.Complete();
b.Target2.Complete();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion);
}
public async Task BatchedJoinToAction()
{
var b = new BatchedJoinBlock<int, int>(1);
int completedCount = 0;
var c = new ActionBlock<Tuple<IList<int>, IList<int>>>(i => completedCount++);
b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
for (int i = 0; i < Iterations; i++)
{
if (i % 2 == 0) b.Target1.Post(i);
else b.Target2.Post(i);
}
b.Target1.Complete();
b.Target2.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public void RunBatchedJoinBlockConformanceTests()
{
// Test Post/Receive single block
var block = new BatchedJoinBlock <int, int>(2);
int iter = 10;
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
if (i % block.BatchSize == 0)
{
var msg = block.Receive();
Assert.False(msg.Item1.Count != msg.Item2.Count, "BatchedJoinBlock Post/Receive failed, returned arrays of differnet length");
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock Post/Receive failed, returned arrys items are different");
}
}
}
}
// Test PostAll then Receive single block
block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.False(block.OutputCount != iter, string.Format("BatchedJoinBlock Post failed, expected, incorrect OutputCount. Expected {0} actual {1}", iter, block.OutputCount));
for (int i = 0; i < block.OutputCount; i++)
{
var msg = block.Receive();
if (msg.Item1.Count != msg.Item2.Count)
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrays of differnet length");
}
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrys items are different");
}
}
}
//Test one target Post < patchSize msg with TryReceive
block = new BatchedJoinBlock <int, int>(2);
block.Target1.Post(0);
Tuple <IList <int>, IList <int> > result;
if (block.TryReceive(out result))
{
Assert.False(true, "BatchedJoinBlock.TryReceive failed, returned true and the number of messages is less than the batch size");
}
if (block.OutputCount > 0)
{
Assert.False(true, "BatchedJoinBlock.OutputCount failed, returned count > 0 and only one target posted a message");
}
// Test handling of stragglers at end of block's life
block = new BatchedJoinBlock <int, int>(2);
for (int i = 0; i < 10; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
if (block.OutputCount != 11)
{
Assert.False(true, "BatchedJoinBlock last batch not generated correctly");
}
for (int i = 0; i < 10; i++)
{
block.Receive();
}
var lastResult = block.Receive();
if (lastResult.Item1.Count != 1 || lastResult.Item2.Count != 0)
{
Assert.False(true, "BatchedJoinBlock last batch contains incorrect data");
}
// Test BatchedJoinBlock`2 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions {
CancellationToken = cts.Token, MaxNumberOfGroups = 1
};
var bjb = new BatchedJoinBlock <int, int>(42, dbo);
Tuple <IList <int>, IList <int> > ignoredValue;
IList <Tuple <IList <int>, IList <int> > > ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock <Tuple <IList <int>, IList <int> > >(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`2");
}
// Test BatchedJoinBlock`3 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions {
CancellationToken = cts.Token, MaxNumberOfGroups = 1
};
var bjb = new BatchedJoinBlock <int, int, int>(42, dbo);
Tuple <IList <int>, IList <int>, IList <int> > ignoredValue;
IList <Tuple <IList <int>, IList <int>, IList <int> > > ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock <Tuple <IList <int>, IList <int>, IList <int> > >(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target3.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.Target3.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
bjb.Target3.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`3");
}
// Test BatchedJoinBlock`2 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock <int, int>(99);
var terminator = new ActionBlock <Tuple <IList <int>, IList <int> > >(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock <int, int, int>(99);
var terminator = new ActionBlock <Tuple <IList <int>, IList <int>, IList <int> > >(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
batchedJoin.Target3.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`2 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock <int, int>(99);
var terminator = new ActionBlock <Tuple <IList <int>, IList <int> > >(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock <int, int, int>(99);
var terminator = new ActionBlock <Tuple <IList <int>, IList <int>, IList <int> > >(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
}