public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.dataflowBlockOptions = dataflowBlockOptions;
this.target1 = new JoinTarget <T1> (this, SignalArrivalTarget1, new BlockingCollection <T1> (), compHelper);
this.target2 = new JoinTarget <T2> (this, SignalArrivalTarget2, new BlockingCollection <T2> (), compHelper);
this.outgoing = new MessageOutgoingQueue <Tuple <T1, T2> > (compHelper, () => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted);
}
public BatchedJoinBlock(int batchSize,
GroupingDataflowBlockOptions dataflowBlockOptions)
{
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException(
"batchSize", batchSize, "The batchSize must be positive.");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
if (!dataflowBlockOptions.Greedy)
{
throw new ArgumentException(
"Greedy must be true for this dataflow block.", "dataflowBlockOptions");
}
if (dataflowBlockOptions.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
throw new ArgumentException(
"BoundedCapacity must be Unbounded or -1 for this dataflow block.",
"dataflowBlockOptions");
}
BatchSize = batchSize;
options = dataflowBlockOptions;
completionHelper = CompletionHelper.GetNew(options);
target1 = new JoinTarget <T1> (
this, SignalTarget, completionHelper, () => outgoing.IsCompleted,
dataflowBlockOptions, true, TryAdd);
target2 = new JoinTarget <T2> (
this, SignalTarget, completionHelper, () => outgoing.IsCompleted,
dataflowBlockOptions, true, TryAdd);
target3 = new JoinTarget <T3> (
this, SignalTarget, completionHelper, () => outgoing.IsCompleted,
dataflowBlockOptions, true, TryAdd);
outgoing = new OutgoingQueue <Tuple <IList <T1>, IList <T2>, IList <T3> > > (
this, completionHelper,
() => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted ||
target3.Buffer.IsCompleted,
_ =>
{
target1.DecreaseCount();
target2.DecreaseCount();
target3.DecreaseCount();
}, options);
}
public BatchBlock(int batchSize, GroupingDataflowBlockOptions dataflowBlockOptions)
{
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException("batchSize", batchSize,
"The batchSize must be positive.");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
if (dataflowBlockOptions.BoundedCapacity != -1 &&
batchSize > dataflowBlockOptions.BoundedCapacity)
{
throw new ArgumentOutOfRangeException("batchSize",
"The batchSize must be smaller than the value of BoundedCapacity.");
}
this.batchSize = batchSize;
this.dataflowBlockOptions = dataflowBlockOptions;
this.compHelper = CompletionHelper.GetNew(dataflowBlockOptions);
Action <bool> processQueue;
Func <bool> canAccept;
if (dataflowBlockOptions.MaxNumberOfGroups == -1)
{
processQueue = newItem => BatchProcess(newItem ? 1 : 0);
canAccept = null;
}
else
{
processQueue = _ => BatchProcess();
canAccept = TryAdd;
}
this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper,
() => outgoing.IsCompleted, processQueue, dataflowBlockOptions,
dataflowBlockOptions.Greedy, canAccept);
this.outgoing = new OutgoingQueue <T[]> (this, compHelper,
() => messageQueue.IsCompleted, messageBox.DecreaseCount,
dataflowBlockOptions, batch => batch.Length);
}
public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.dataflowBlockOptions = dataflowBlockOptions;
Func <bool> checker1 = () => target2.Buffer.Count == 0 || target3.Buffer.Count == 0;
Func <bool> checker2 = () => target1.Buffer.Count == 0 || target3.Buffer.Count == 0;
Func <bool> checker3 = () => target1.Buffer.Count == 0 || target2.Buffer.Count == 0;
this.target1 = new JoinTarget <T1> (this, () => SignalArrivalTargetImpl(checker1), new BlockingCollection <T1> (), compHelper);
this.target2 = new JoinTarget <T2> (this, () => SignalArrivalTargetImpl(checker2), new BlockingCollection <T2> (), compHelper);
this.target3 = new JoinTarget <T3> (this, () => SignalArrivalTargetImpl(checker3), new BlockingCollection <T3> (), compHelper);
this.outgoing =
new MessageOutgoingQueue <Tuple <T1, T2, T3> > (compHelper,
() => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted || target3.Buffer.IsCompleted);
}
public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.dataflowBlockOptions = dataflowBlockOptions;
compHelper = CompletionHelper.GetNew(dataflowBlockOptions);
target1 = new JoinTarget <T1> (this, SignalArrivalTarget, compHelper,
() => outgoing.IsCompleted, dataflowBlockOptions,
dataflowBlockOptions.Greedy, TryAdd1);
target2 = new JoinTarget <T2> (this, SignalArrivalTarget, compHelper,
() => outgoing.IsCompleted, dataflowBlockOptions,
dataflowBlockOptions.Greedy, TryAdd2);
outgoing = new OutgoingQueue <Tuple <T1, T2> > (this, compHelper,
() => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted,
_ =>
{
target1.DecreaseCount();
target2.DecreaseCount();
}, dataflowBlockOptions);
}
/// <summary>Initializes this <see cref="BatchedJoinBlock{T1,T2}"/> with the specified configuration.</summary>
/// <param name="batchSize">The number of items to group into a batch.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BatchedJoinBlock{T1,T2}"/>.</param>
/// <exception cref="System.ArgumentOutOfRangeException">The <paramref name="batchSize"/> must be positive.</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public BatchedJoinBlock(int batchSize, GroupingDataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments
if (batchSize < 1)
{
throw new ArgumentOutOfRangeException(nameof(batchSize), SR.ArgumentOutOfRange_GenericPositive);
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
if (!dataflowBlockOptions.Greedy)
{
throw new ArgumentException(SR.Argument_NonGreedyNotSupported, nameof(dataflowBlockOptions));
}
if (dataflowBlockOptions.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
throw new ArgumentException(SR.Argument_BoundedCapacityNotSupported, nameof(dataflowBlockOptions));
}
// Store arguments
_batchSize = batchSize;
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Configure the source
_source = new SourceCore <Tuple <IList <T1>, IList <T2> > >(
this, dataflowBlockOptions, owningSource => ((BatchedJoinBlock <T1, T2>)owningSource).CompleteEachTarget());
// The action to run when a batch should be created. This is typically called
// when we have a full batch, but it will also be called when we're done receiving
// messages, and thus when there may be a few stragglers we need to make a batch out of.
Action createBatchAction = () =>
{
if (_target1.Count > 0 || _target2.Count > 0)
{
_source.AddMessage(Tuple.Create(_target1.GetAndEmptyMessages(), _target2.GetAndEmptyMessages()));
}
};
// Configure the targets
_sharedResources = new BatchedJoinBlockTargetSharedResources(
batchSize, dataflowBlockOptions,
createBatchAction,
() =>
{
createBatchAction();
_source.Complete();
},
_source.AddException,
Complete);
_target1 = new BatchedJoinBlockTarget <T1>(_sharedResources);
_target2 = new BatchedJoinBlockTarget <T2>(_sharedResources);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
_source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((BatchedJoinBlock <T1, T2>)state !) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception !);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, _source.Completion, state => ((BatchedJoinBlock <T1, T2>)state !).CompleteEachTarget(), this);
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
public void RunBatchedJoinBlockConformanceTests()
{
// Test Post/Receive single block
var block = new BatchedJoinBlock<int, int>(2);
int iter = 10;
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
if (i % block.BatchSize == 0)
{
var msg = block.Receive();
Assert.False(msg.Item1.Count != msg.Item2.Count, "BatchedJoinBlock Post/Receive failed, returned arrays of differnet length");
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock Post/Receive failed, returned arrys items are different");
}
}
}
}
// Test PostAll then Receive single block
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < iter; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
Assert.False(block.OutputCount != iter, string.Format("BatchedJoinBlock Post failed, expected, incorrect OutputCount. Expected {0} actual {1}", iter, block.OutputCount));
for (int i = 0; i < block.OutputCount; i++)
{
var msg = block.Receive();
if (msg.Item1.Count != msg.Item2.Count)
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrays of differnet length");
}
for (int j = 0; j < msg.Item1.Count; j++)
{
if (msg.Item1[j] != msg.Item2[j])
{
Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrys items are different");
}
}
}
//Test one target Post < patchSize msg with TryReceive
block = new BatchedJoinBlock<int, int>(2);
block.Target1.Post(0);
Tuple<IList<int>, IList<int>> result;
if (block.TryReceive(out result))
{
Assert.False(true, "BatchedJoinBlock.TryReceive failed, returned true and the number of messages is less than the batch size");
}
if (block.OutputCount > 0)
{
Assert.False(true, "BatchedJoinBlock.OutputCount failed, returned count > 0 and only one target posted a message");
}
// Test handling of stragglers at end of block's life
block = new BatchedJoinBlock<int, int>(2);
for (int i = 0; i < 10; i++)
{
block.Target1.Post(i);
block.Target2.Post(i);
}
block.Target1.Post(10);
block.Target1.Complete();
block.Target2.Complete();
if (block.OutputCount != 11)
{
Assert.False(true, "BatchedJoinBlock last batch not generated correctly");
}
for (int i = 0; i < 10; i++) block.Receive();
var lastResult = block.Receive();
if (lastResult.Item1.Count != 1 || lastResult.Item2.Count != 0)
{
Assert.False(true, "BatchedJoinBlock last batch contains incorrect data");
}
// Test BatchedJoinBlock`2 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int>(42, dbo);
Tuple<IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`2");
}
// Test BatchedJoinBlock`3 using a precanceled token
{
var localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var bjb = new BatchedJoinBlock<int, int, int>(42, dbo);
Tuple<IList<int>, IList<int>, IList<int>> ignoredValue;
IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues;
localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })) != null;
localPassed &= bjb.Target1.Post(42) == false;
localPassed &= bjb.Target2.Post(42) == false;
localPassed &= bjb.Target3.Post(42) == false;
localPassed &= bjb.Target1.SendAsync(42).Result == false;
localPassed &= bjb.Target2.SendAsync(42).Result == false;
localPassed &= bjb.Target3.SendAsync(42).Result == false;
localPassed &= bjb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bjb.TryReceive(out ignoredValue) == false;
localPassed &= bjb.OutputCount == 0;
localPassed &= bjb.Completion != null;
bjb.Target1.Complete();
bjb.Target2.Complete();
bjb.Target3.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`3");
}
// Test BatchedJoinBlock`2 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through all targets
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Target1.Complete();
batchedJoin.Target2.Complete();
batchedJoin.Target3.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through targets - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`2 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test BatchedJoinBlock`3 completion through block
{
var localPassed = true;
var batchedJoin = new BatchedJoinBlock<int, int, int>(99);
var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { });
batchedJoin.LinkTo(terminator);
batchedJoin.Target1.Post(1);
batchedJoin.Complete();
localPassed = batchedJoin.Completion.Wait(2000);
Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through block - {0}", localPassed ? "Passed" : "FAILED"));
}
}
private static bool TestTriggerBatchRacingWithComplete(bool greedy)
{
bool passed = true;
const int batchSize = 2;
const int iterations = 1;
const int waitTimeout = 100;
var dbo = new GroupingDataflowBlockOptions { Greedy = greedy };
for (int iter = 0; iter < iterations; iter++)
{
bool localPassed = true;
var sendAsyncTasks = new Task<bool>[batchSize - 1];
var racerReady = new ManualResetEventSlim();
int[] output1 = null;
int[] output2 = null;
// Blocks
var batch = new BatchBlock<int>(batchSize, dbo);
var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; });
batch.LinkTo(terminator);
// Queue up batchSize-1 input items
for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i);
var racer = Task.Factory.StartNew(() =>
{
racerReady.Set();
batch.Complete();
});
// Wait for the racer to get ready and trigger
localPassed &= racerReady.Wait(waitTimeout);
batch.TriggerBatch();
Assert.True(localPassed, "The racer task FAILED to start.");
if (localPassed)
{
// Wait for the SendAsync tasks to complete
localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout);
Assert.True(localPassed, "SendAsync tasks FAILED to complete");
}
// Do this verification only in greedy mode, because non-greedy is non-deterministic
if (greedy)
{
// Wait for a batch to be produced
if (localPassed)
{
localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout);
Assert.True(localPassed, "FAILED to produce a batch");
}
if (localPassed)
{
//Verify the number of input items propagated
localPassed &= output1.Length == batchSize - 1;
Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}",
batchSize, output1.Length));
}
}
// Wait for the block to complete
if (localPassed)
{
localPassed &= batch.Completion.Wait(waitTimeout);
Assert.True(localPassed, "The block FAILED to complete");
}
// There should never be a second batch produced
if (localPassed)
{
localPassed &= output2 == null;
Assert.True(localPassed, "FAILED not to produce a second batch");
}
passed &= localPassed;
if (!localPassed)
{
Assert.True(localPassed, string.Format("Iteration={0}", iter));
Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString()));
break;
}
}
return passed;
}
private static bool TestTriggerBatchRacingWithSendAsync(bool greedy)
{
bool passed = true;
const int batchSize = 2;
const int iterations = 1;
const int waitTimeout = 100;
var dbo = new GroupingDataflowBlockOptions { Greedy = greedy };
for (int iter = 0; iter < iterations; iter++)
{
bool localPassed = true;
var sendAsyncTasks = new Task<bool>[batchSize - 1];
Task<bool> lastSendAsyncTask = null;
var racerReady = new ManualResetEventSlim();
var racerDone = new ManualResetEventSlim();
int[] output1 = null;
int[] output2 = null;
// Blocks
var batch = new BatchBlock<int>(batchSize, dbo);
var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; });
batch.LinkTo(terminator);
// Queue up batchSize-1 input items
for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i);
var racer = Task.Factory.StartNew(() =>
{
racerReady.Set();
lastSendAsyncTask = batch.SendAsync(batchSize - 1);
racerDone.Set();
});
// Wait for the racer to get ready and trigger
localPassed &= (racerReady.Wait(waitTimeout));
batch.TriggerBatch();
Assert.True(localPassed, "The racer task FAILED to start.");
// Wait for the SendAsync tasks to complete
localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout);
Assert.True(localPassed, "SendAsync tasks FAILED to complete");
// Wait for a batch to be produced
if (localPassed)
{
localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout);
Assert.True(localPassed, "FAILED to produce a batch");
}
if (localPassed && output1.Length < batchSize)
{
// If the produced batch is not full, we'll trigger one more and count the items.
// However, we need to make sure the last message has been offered. Otherwise this
// trigger will have no effect.
racerDone.Wait(waitTimeout);
batch.TriggerBatch();
if (localPassed)
{
// Wait for the last SendAsync task to complete
localPassed &= SpinWait.SpinUntil(() => lastSendAsyncTask != null, waitTimeout);
localPassed &= lastSendAsyncTask.Wait(waitTimeout);
Assert.True(localPassed, "The last SendAsync task FAILED to complete");
}
// Wait for a second batch to be produced
if (localPassed)
{
localPassed &= SpinWait.SpinUntil(() => output2 != null, waitTimeout);
Assert.True(localPassed, "FAILED to produce a second batch");
}
//Verify the total number of input items propagated
if (localPassed)
{
localPassed &= output1.Length + output2.Length == batchSize;
Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}, count2={2}",
batchSize, output1.Length, output2.Length));
}
}
passed &= localPassed;
if (!localPassed)
{
Assert.True(localPassed, string.Format("Iteration={0}", iter));
Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString()));
Assert.True(localPassed, string.Format("Count2={0}", output2 == null ? "null" : output2.Length.ToString()));
break;
}
}
return passed;
}
private static bool TestTriggerBatch(int boundedCapacity)
{
bool passed = true;
// Test greedy with batch size of 1 (force should always be a nop)
{
bool localPassed = true;
const int ITERS = 2;
var b = new BatchBlock<int>(1, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int i = 0; i < ITERS; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
b.TriggerBatch();
localPassed &= outputCount == b.OutputCount;
}
localPassed &= b.OutputCount == ITERS;
for (int i = 0; i < ITERS; i++)
{
var arr = b.Receive();
localPassed &= arr.Length == 1 && arr[0] == i;
}
Assert.True(localPassed, string.Format("greedy with batch size of 1 - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test greedy with varying batch sizes and smaller queued numbers
{
bool localPassed = true;
foreach (var batchSize in new[] { 3 })
{
foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 })
{
var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int p = 1; p <= queuedBeforeTrigger; p++) b.Post(p);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided
localPassed &= b.OutputCount == 1 && b.Receive().Length == queuedBeforeTrigger;
for (int j = 0; j < batchSize; j++)
{
localPassed &= b.OutputCount == 0;
b.Post(j);
}
localPassed &= b.OutputCount == 1;
}
}
Assert.True(localPassed, string.Format("greedy with varying batch sizes - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy with empty queue
{
bool localPassed = true;
foreach (var batchSize in new[] { 1 })
{
var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int i = 0; i < 2; i++) b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("greedy with empty queue - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy after decline
{
bool localPassed = true;
{
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
localPassed &= b.OutputCount == 0;
b.Complete();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
{
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
localPassed &= b.OutputCount == 0;
b.Post(1);
localPassed &= b.OutputCount == 0;
b.Complete();
localPassed &= b.OutputCount == 1;
b.TriggerBatch();
localPassed &= b.OutputCount == 1;
}
Assert.True(localPassed, string.Format("greedy after decline - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy after canceled
{
bool localPassed = true;
{
var cts = new CancellationTokenSource();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(2, dbo);
localPassed &= b.OutputCount == 0;
cts.Cancel();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
{
var cts = new CancellationTokenSource();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(2, dbo);
localPassed &= b.OutputCount == 0;
b.Post(1);
localPassed &= b.OutputCount == 0;
cts.Cancel();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("greedy after canceled - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy with MaxNumberOfGroups == 1
{
bool localPassed = true;
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1, BoundedCapacity = boundedCapacity });
b.Post(1);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided
localPassed &= b.OutputCount == 1;
localPassed &= b.Post(2) == false;
b.TriggerBatch();
localPassed &= b.OutputCount == 1;
Assert.True(localPassed, string.Format("greedy with MaxNumberOfGroups == 1 - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test non-greedy with no queued or postponed messages
{
bool localPassed = true;
var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(3, dbo);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
Assert.True(localPassed, string.Format("non-greedy with no mesages - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test non-greedy with no queued but postponed messages
{
bool localPassed = true;
var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity };
const int BATCH_SIZE = 10;
for (int numPostponedMessages = 1; numPostponedMessages < BATCH_SIZE; numPostponedMessages++)
{
var b = new BatchBlock<int>(BATCH_SIZE, dbo);
localPassed &= b.OutputCount == 0;
for (int i = 0; i < numPostponedMessages; i++) localPassed &= !b.SendAsync(i).IsCompleted;
b.TriggerBatch();
var output = b.Receive();
localPassed &= output.Length == numPostponedMessages;
for (int i = 0; i < output.Length; i++) localPassed &= output[i] == i;
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("non-greedy with postponed, no queued - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
return passed;
}
private static bool TestMaxNumberOfGroups(bool greedy, bool sync)
{
Contract.Assert(greedy || !sync, "Non-greedy sync doesn't make sense.");
bool passed = true;
for (int maxNumberOfGroups = 1; maxNumberOfGroups <= 21; maxNumberOfGroups += 20)
{
for (int itemsPerBatch = 1; itemsPerBatch <= 1; itemsPerBatch++)
{
var options = new GroupingDataflowBlockOptions { MaxNumberOfGroups = maxNumberOfGroups, Greedy = greedy };
var batch = new BatchBlock<int>(itemsPerBatch, options);
// Feed all N batches; all should succeed
for (int batchNum = 0; batchNum < maxNumberOfGroups; batchNum++)
{
var sendAsyncs = new Task<bool>[itemsPerBatch];
for (int itemNum = 0; itemNum < itemsPerBatch; itemNum++)
{
if (sync)
{
Assert.True(batch.Post(itemNum), string.Format("FAILED batch.Post({0}) on MaxNOG {1}", itemNum, batchNum));
}
else
{
sendAsyncs[itemNum] = batch.SendAsync(itemNum);
}
}
if (!sync)
{
Assert.True(Task.WaitAll(sendAsyncs, 4000),
string.Format("FAILED batch.SendAsyncs should have been completed in batch num {0}", batchNum));
if (passed)
{
Assert.True(sendAsyncs.All(t => t.Status == TaskStatus.RanToCompletion && t.Result),
string.Format("FAILED batch.SendAsyncs should have been completed in batch num {0}", batchNum));
}
}
}
// Next message should fail in greedy mode
if (greedy)
{
if (sync)
{
Assert.False(batch.Post(1), "FAILED batch.Post(1) after completed groups should be declind");
}
else
{
var t = batch.SendAsync(1);
Assert.True(t != null && t.Status == TaskStatus.RanToCompletion && t.Result == false, "FAILED batch.SendAsync(1) after completed groups should be declined");
}
}
// Wait until the all batches are produced
Assert.True(SpinWait.SpinUntil(() => batch.OutputCount == maxNumberOfGroups, 4000), "FAILED All batches should have been produced");
// Next message should fail, even after groups have been produced
if (sync)
{
Assert.False(batch.Post(1), "FAILED batch.Post(1) after completed groups are output should be declind");
}
else
{
var t = batch.SendAsync(1);
Assert.True(t != null && t.Status == TaskStatus.RanToCompletion && t.Result == false, "FAILED batch.SendAsync(1) after completed groups are output should be declined");
}
}
}
Assert.True(passed, string.Format("{0}", passed ? "Passed" : "FAILED"));
return passed;
}
public void RunBatchBlockConformanceTests()
{
bool localPassed;
// Greedy batching
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 1;
var batch = new BatchBlock<int>(BATCH_SIZE);
for (int i = 0; i < NUM_MESSAGES * BATCH_SIZE; i++) batch.Post(i);
for (int i = 0; i < NUM_MESSAGES; i++)
{
int[] result = batch.Receive();
localPassed &= result.Length == BATCH_SIZE;
for (int j = 0; j < result.Length - 1; j++)
{
localPassed &= (result[j] + 1 == result[j + 1]);
}
}
Assert.True(localPassed, string.Format("{0}: Greedy batching", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with BATCH_SIZE sources used repeatedly
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 1;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers) buffer.LinkTo(batch);
int prevSum = -1;
for (int i = 0; i < NUM_MESSAGES; i++)
{
for (int j = 0; j < BATCH_SIZE; j++) buffers[j].Post(i);
int sum = batch.Receive().Sum();
localPassed &= (sum > prevSum);
prevSum = sum;
}
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with BATCH_SIZE sources used repeatedly", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with BATCH_SIZE * NUM_MESSAGES sources
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 2;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE * NUM_MESSAGES).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(1);
}
for (int i = 0; i < NUM_MESSAGES; i++)
{
localPassed &= batch.Receive().Sum() == BATCH_SIZE;
}
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with N*M sources", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with missed messages
{
localPassed = true;
const int BATCH_SIZE = 2;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE - 1).Select(_ => new BufferBlock<int>()).ToList();
using (var ce = new CountdownEvent(BATCH_SIZE - 1))
{
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.LinkTo(new ActionBlock<int>(i => ce.Signal()));
buffer.Post(42);
}
ce.Wait();
}
buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(42);
buffer.Complete();
}
localPassed &= Task.WaitAll(buffers.Select(b => b.Completion).ToArray(), 2000);
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with missed messages", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var b = new BatchBlock<int>(42, dbo);
int[] ignoredValue;
IList<int[]> ignoredValues;
localPassed &= b.BatchSize == 42;
localPassed &= b.LinkTo(new ActionBlock<int[]>(delegate { })) != null;
localPassed &= b.SendAsync(42).Result == false;
localPassed &= b.TryReceiveAll(out ignoredValues) == false;
localPassed &= b.Post(42) == false;
localPassed &= b.OutputCount == 0;
localPassed &= b.TryReceive(out ignoredValue) == false;
localPassed &= b.Completion != null;
b.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
// Test completing block while still items buffered
{
localPassed = true;
var b = new BatchBlock<int>(5);
b.Post(1);
b.Post(2);
b.Post(3);
b.Complete();
localPassed &= b.Receive().Length == 3;
Assert.True(localPassed, string.Format("{0}: Makes batches of remaining items", localPassed ? "Success" : "Failure"));
}
}
/// <summary>
/// Extract relative information from dataflow option to a block-level GroupingDataflowBlockOptions
/// </summary>
public GroupingDataflowBlockOptions ToGroupingBlockOption()
{
var option = new GroupingDataflowBlockOptions();
if (this.RecommendedCapacity != null)
{
option.BoundedCapacity = this.RecommendedCapacity.Value;
}
return option;
}
public void RunDataflowBlockOptionsTests()
{
// Test base DataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().TaskScheduler = null; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = 0; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = 0; });
Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().NameFormat = null; });
}
// Test default values
{
var db = new DataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "TaskScheduler should be Default");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Max messages should be unbounded.");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Bounded capacity should be unbounded.");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested,
"The cancellation token should be None.");
Assert.True(DataflowBlockOptions.Unbounded == -1, "Unbounded should be the value -1");
Assert.True(db.NameFormat == @"{0} Id={1}", @"NameFormat should be the value '{0} Id={1}'");
}
// Test that set values are retrievable
{
var db = new DataflowBlockOptions();
db.MaxMessagesPerTask = 2;
Assert.True(db.MaxMessagesPerTask == 2, "Expected max messages to be the set value 2");
db.MaxMessagesPerTask = Int32.MaxValue;
Assert.True(db.MaxMessagesPerTask == Int32.MaxValue, "Expected max messages to be the set value Int32.MaxValue");
db.MaxMessagesPerTask = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to be unbounded.");
db.BoundedCapacity = 2;
Assert.True(db.BoundedCapacity == 2, "Expected bounded capacity to be the set value 2");
db.BoundedCapacity = Int32.MaxValue;
Assert.True(db.BoundedCapacity == Int32.MaxValue, "Expected bounded capacity to be the set value Int32.MaxValue");
db.BoundedCapacity = DataflowBlockOptions.Unbounded;
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to be unbounded.");
var dummyScheduler = new DummyScheduler();
db.TaskScheduler = dummyScheduler;
Assert.True(db.TaskScheduler == dummyScheduler, "Expected task scheduler to be the dummy scheduler");
db.TaskScheduler = TaskScheduler.Default;
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to be the default scheduler");
var cts = new CancellationTokenSource();
db.CancellationToken = cts.Token;
Assert.True(db.CancellationToken == cts.Token, "Expected the token to be the one just set");
db.CancellationToken = CancellationToken.None;
Assert.True(db.CancellationToken == CancellationToken.None, "Expected the token to be none");
db.NameFormat = "none";
Assert.True(db.NameFormat.Equals("none"), "Expected name format to be the set value 'none'");
db.NameFormat = "foo {0}";
Assert.True(db.NameFormat.Equals("foo {0}"), @"Expected name format to be the set value 'foo {0}'");
db.NameFormat = "foo {0} bar {1}";
Assert.True(db.NameFormat.Equals("foo {0} bar {1}"), @"Expected name format to be the set value 'foo {0} bar {1}'");
db.NameFormat = "kaboom {0} {1} {2}";
Assert.True(db.NameFormat.Equals("kaboom {0} {1} {2}"), @"Expected name format to be the set value 'kaboom {0} {1} {2}'");
}
}
// Test base ExecutionDataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = 0; });
}
// Test default values
{
var db = new ExecutionDataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value");
Assert.True(db.MaxDegreeOfParallelism == 1, "Expected max dop to have default value");
}
// Test that set values are retrievable
{
var db = new ExecutionDataflowBlockOptions();
db.MaxDegreeOfParallelism = 2;
Assert.True(db.MaxDegreeOfParallelism == 2, "Expected max dop to be 2");
db.MaxDegreeOfParallelism = Int32.MaxValue;
Assert.True(db.MaxDegreeOfParallelism == Int32.MaxValue, "Expected max dop to be Int32.MaxValue");
db.MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxDegreeOfParallelism == DataflowBlockOptions.Unbounded, "Expected max dop to be unbounded");
}
}
// Test base GroupingDataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = 0; });
}
// Test default values
{
var db = new GroupingDataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value");
Assert.True(db.MaxNumberOfGroups == DataflowBlockOptions.Unbounded, "Expected max groups to have default value");
Assert.True(db.Greedy == true, "Expected greedy to have default value");
}
// Test that set values are retrievable
{
var db = new GroupingDataflowBlockOptions();
db.MaxNumberOfGroups = 2;
Assert.True(db.MaxNumberOfGroups == 2, "Expected max groups to be 2");
db.MaxNumberOfGroups = Int32.MaxValue;
Assert.True(db.MaxNumberOfGroups == Int32.MaxValue, "Expected max groups to be Int32.MaxValue");
db.MaxNumberOfGroups = Int64.MaxValue;
Assert.True(db.MaxNumberOfGroups == Int64.MaxValue, "Expected max groups to be Int64.MaxValue");
db.MaxNumberOfGroups = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max groups to unbounded");
db.Greedy = true;
Assert.True(db.Greedy == true, "Expected greedy to be true");
db.Greedy = false;
Assert.True(db.Greedy == false, "Expected greedy to be false");
db.Greedy = true;
Assert.True(db.Greedy == true, "Expected greedy to be true");
}
}
}