public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.dataflowBlockOptions = dataflowBlockOptions; this.target1 = new JoinTarget <T1> (this, SignalArrivalTarget1, new BlockingCollection <T1> (), compHelper); this.target2 = new JoinTarget <T2> (this, SignalArrivalTarget2, new BlockingCollection <T2> (), compHelper); this.outgoing = new MessageOutgoingQueue <Tuple <T1, T2> > (compHelper, () => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted); }
public BatchedJoinBlock(int batchSize, GroupingDataflowBlockOptions dataflowBlockOptions) { if (batchSize <= 0) { throw new ArgumentOutOfRangeException( "batchSize", batchSize, "The batchSize must be positive."); } if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } if (!dataflowBlockOptions.Greedy) { throw new ArgumentException( "Greedy must be true for this dataflow block.", "dataflowBlockOptions"); } if (dataflowBlockOptions.BoundedCapacity != DataflowBlockOptions.Unbounded) { throw new ArgumentException( "BoundedCapacity must be Unbounded or -1 for this dataflow block.", "dataflowBlockOptions"); } BatchSize = batchSize; options = dataflowBlockOptions; completionHelper = CompletionHelper.GetNew(options); target1 = new JoinTarget <T1> ( this, SignalTarget, completionHelper, () => outgoing.IsCompleted, dataflowBlockOptions, true, TryAdd); target2 = new JoinTarget <T2> ( this, SignalTarget, completionHelper, () => outgoing.IsCompleted, dataflowBlockOptions, true, TryAdd); target3 = new JoinTarget <T3> ( this, SignalTarget, completionHelper, () => outgoing.IsCompleted, dataflowBlockOptions, true, TryAdd); outgoing = new OutgoingQueue <Tuple <IList <T1>, IList <T2>, IList <T3> > > ( this, completionHelper, () => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted || target3.Buffer.IsCompleted, _ => { target1.DecreaseCount(); target2.DecreaseCount(); target3.DecreaseCount(); }, options); }
public BatchBlock(int batchSize, GroupingDataflowBlockOptions dataflowBlockOptions) { if (batchSize <= 0) { throw new ArgumentOutOfRangeException("batchSize", batchSize, "The batchSize must be positive."); } if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } if (dataflowBlockOptions.BoundedCapacity != -1 && batchSize > dataflowBlockOptions.BoundedCapacity) { throw new ArgumentOutOfRangeException("batchSize", "The batchSize must be smaller than the value of BoundedCapacity."); } this.batchSize = batchSize; this.dataflowBlockOptions = dataflowBlockOptions; this.compHelper = CompletionHelper.GetNew(dataflowBlockOptions); Action <bool> processQueue; Func <bool> canAccept; if (dataflowBlockOptions.MaxNumberOfGroups == -1) { processQueue = newItem => BatchProcess(newItem ? 1 : 0); canAccept = null; } else { processQueue = _ => BatchProcess(); canAccept = TryAdd; } this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper, () => outgoing.IsCompleted, processQueue, dataflowBlockOptions, dataflowBlockOptions.Greedy, canAccept); this.outgoing = new OutgoingQueue <T[]> (this, compHelper, () => messageQueue.IsCompleted, messageBox.DecreaseCount, dataflowBlockOptions, batch => batch.Length); }
public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.dataflowBlockOptions = dataflowBlockOptions; Func <bool> checker1 = () => target2.Buffer.Count == 0 || target3.Buffer.Count == 0; Func <bool> checker2 = () => target1.Buffer.Count == 0 || target3.Buffer.Count == 0; Func <bool> checker3 = () => target1.Buffer.Count == 0 || target2.Buffer.Count == 0; this.target1 = new JoinTarget <T1> (this, () => SignalArrivalTargetImpl(checker1), new BlockingCollection <T1> (), compHelper); this.target2 = new JoinTarget <T2> (this, () => SignalArrivalTargetImpl(checker2), new BlockingCollection <T2> (), compHelper); this.target3 = new JoinTarget <T3> (this, () => SignalArrivalTargetImpl(checker3), new BlockingCollection <T3> (), compHelper); this.outgoing = new MessageOutgoingQueue <Tuple <T1, T2, T3> > (compHelper, () => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted || target3.Buffer.IsCompleted); }
public JoinBlock(GroupingDataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.dataflowBlockOptions = dataflowBlockOptions; compHelper = CompletionHelper.GetNew(dataflowBlockOptions); target1 = new JoinTarget <T1> (this, SignalArrivalTarget, compHelper, () => outgoing.IsCompleted, dataflowBlockOptions, dataflowBlockOptions.Greedy, TryAdd1); target2 = new JoinTarget <T2> (this, SignalArrivalTarget, compHelper, () => outgoing.IsCompleted, dataflowBlockOptions, dataflowBlockOptions.Greedy, TryAdd2); outgoing = new OutgoingQueue <Tuple <T1, T2> > (this, compHelper, () => target1.Buffer.IsCompleted || target2.Buffer.IsCompleted, _ => { target1.DecreaseCount(); target2.DecreaseCount(); }, dataflowBlockOptions); }
/// <summary>Initializes this <see cref="BatchedJoinBlock{T1,T2}"/> with the specified configuration.</summary> /// <param name="batchSize">The number of items to group into a batch.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BatchedJoinBlock{T1,T2}"/>.</param> /// <exception cref="System.ArgumentOutOfRangeException">The <paramref name="batchSize"/> must be positive.</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public BatchedJoinBlock(int batchSize, GroupingDataflowBlockOptions dataflowBlockOptions) { // Validate arguments if (batchSize < 1) { throw new ArgumentOutOfRangeException(nameof(batchSize), SR.ArgumentOutOfRange_GenericPositive); } if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } if (!dataflowBlockOptions.Greedy) { throw new ArgumentException(SR.Argument_NonGreedyNotSupported, nameof(dataflowBlockOptions)); } if (dataflowBlockOptions.BoundedCapacity != DataflowBlockOptions.Unbounded) { throw new ArgumentException(SR.Argument_BoundedCapacityNotSupported, nameof(dataflowBlockOptions)); } // Store arguments _batchSize = batchSize; dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Configure the source _source = new SourceCore <Tuple <IList <T1>, IList <T2> > >( this, dataflowBlockOptions, owningSource => ((BatchedJoinBlock <T1, T2>)owningSource).CompleteEachTarget()); // The action to run when a batch should be created. This is typically called // when we have a full batch, but it will also be called when we're done receiving // messages, and thus when there may be a few stragglers we need to make a batch out of. Action createBatchAction = () => { if (_target1.Count > 0 || _target2.Count > 0) { _source.AddMessage(Tuple.Create(_target1.GetAndEmptyMessages(), _target2.GetAndEmptyMessages())); } }; // Configure the targets _sharedResources = new BatchedJoinBlockTargetSharedResources( batchSize, dataflowBlockOptions, createBatchAction, () => { createBatchAction(); _source.Complete(); }, _source.AddException, Complete); _target1 = new BatchedJoinBlockTarget <T1>(_sharedResources); _target2 = new BatchedJoinBlockTarget <T2>(_sharedResources); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. _source.Completion.ContinueWith((completed, state) => { var thisBlock = ((BatchedJoinBlock <T1, T2>)state !) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception !); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, _source.Completion, state => ((BatchedJoinBlock <T1, T2>)state !).CompleteEachTarget(), this); #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
public void RunBatchedJoinBlockConformanceTests() { // Test Post/Receive single block var block = new BatchedJoinBlock<int, int>(2); int iter = 10; for (int i = 0; i < iter; i++) { block.Target1.Post(i); block.Target2.Post(i); if (i % block.BatchSize == 0) { var msg = block.Receive(); Assert.False(msg.Item1.Count != msg.Item2.Count, "BatchedJoinBlock Post/Receive failed, returned arrays of differnet length"); for (int j = 0; j < msg.Item1.Count; j++) { if (msg.Item1[j] != msg.Item2[j]) { Assert.False(true, "BatchedJoinBlock Post/Receive failed, returned arrys items are different"); } } } } // Test PostAll then Receive single block block = new BatchedJoinBlock<int, int>(2); for (int i = 0; i < iter; i++) { block.Target1.Post(i); block.Target2.Post(i); } Assert.False(block.OutputCount != iter, string.Format("BatchedJoinBlock Post failed, expected, incorrect OutputCount. Expected {0} actual {1}", iter, block.OutputCount)); for (int i = 0; i < block.OutputCount; i++) { var msg = block.Receive(); if (msg.Item1.Count != msg.Item2.Count) { Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrays of differnet length"); } for (int j = 0; j < msg.Item1.Count; j++) { if (msg.Item1[j] != msg.Item2[j]) { Assert.False(true, "BatchedJoinBlock PostAll then Receive failed, returned arrys items are different"); } } } //Test one target Post < patchSize msg with TryReceive block = new BatchedJoinBlock<int, int>(2); block.Target1.Post(0); Tuple<IList<int>, IList<int>> result; if (block.TryReceive(out result)) { Assert.False(true, "BatchedJoinBlock.TryReceive failed, returned true and the number of messages is less than the batch size"); } if (block.OutputCount > 0) { Assert.False(true, "BatchedJoinBlock.OutputCount failed, returned count > 0 and only one target posted a message"); } // Test handling of stragglers at end of block's life block = new BatchedJoinBlock<int, int>(2); for (int i = 0; i < 10; i++) { block.Target1.Post(i); block.Target2.Post(i); } block.Target1.Post(10); block.Target1.Complete(); block.Target2.Complete(); if (block.OutputCount != 11) { Assert.False(true, "BatchedJoinBlock last batch not generated correctly"); } for (int i = 0; i < 10; i++) block.Receive(); var lastResult = block.Receive(); if (lastResult.Item1.Count != 1 || lastResult.Item2.Count != 0) { Assert.False(true, "BatchedJoinBlock last batch contains incorrect data"); } // Test BatchedJoinBlock`2 using a precanceled token { var localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 }; var bjb = new BatchedJoinBlock<int, int>(42, dbo); Tuple<IList<int>, IList<int>> ignoredValue; IList<Tuple<IList<int>, IList<int>>> ignoredValues; localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>>>(delegate { })) != null; localPassed &= bjb.Target1.Post(42) == false; localPassed &= bjb.Target2.Post(42) == false; localPassed &= bjb.Target1.SendAsync(42).Result == false; localPassed &= bjb.Target2.SendAsync(42).Result == false; localPassed &= bjb.TryReceiveAll(out ignoredValues) == false; localPassed &= bjb.TryReceive(out ignoredValue) == false; localPassed &= bjb.OutputCount == 0; localPassed &= bjb.Completion != null; bjb.Target1.Complete(); bjb.Target2.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`2"); } // Test BatchedJoinBlock`3 using a precanceled token { var localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 }; var bjb = new BatchedJoinBlock<int, int, int>(42, dbo); Tuple<IList<int>, IList<int>, IList<int>> ignoredValue; IList<Tuple<IList<int>, IList<int>, IList<int>>> ignoredValues; localPassed &= bjb.LinkTo(new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(delegate { })) != null; localPassed &= bjb.Target1.Post(42) == false; localPassed &= bjb.Target2.Post(42) == false; localPassed &= bjb.Target3.Post(42) == false; localPassed &= bjb.Target1.SendAsync(42).Result == false; localPassed &= bjb.Target2.SendAsync(42).Result == false; localPassed &= bjb.Target3.SendAsync(42).Result == false; localPassed &= bjb.TryReceiveAll(out ignoredValues) == false; localPassed &= bjb.TryReceive(out ignoredValue) == false; localPassed &= bjb.OutputCount == 0; localPassed &= bjb.Completion != null; bjb.Target1.Complete(); bjb.Target2.Complete(); bjb.Target3.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, "Precanceled tokens don't work correctly on BJB`3"); } // Test BatchedJoinBlock`2 completion through all targets { var localPassed = true; var batchedJoin = new BatchedJoinBlock<int, int>(99); var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { }); batchedJoin.LinkTo(terminator); batchedJoin.Target1.Post(1); batchedJoin.Target1.Complete(); batchedJoin.Target2.Complete(); localPassed = batchedJoin.Completion.Wait(2000); Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through targets - {0}", localPassed ? "Passed" : "FAILED")); } // Test BatchedJoinBlock`3 completion through all targets { var localPassed = true; var batchedJoin = new BatchedJoinBlock<int, int, int>(99); var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { }); batchedJoin.LinkTo(terminator); batchedJoin.Target1.Post(1); batchedJoin.Target1.Complete(); batchedJoin.Target2.Complete(); batchedJoin.Target3.Complete(); localPassed = batchedJoin.Completion.Wait(2000); Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through targets - {0}", localPassed ? "Passed" : "FAILED")); } // Test BatchedJoinBlock`2 completion through block { var localPassed = true; var batchedJoin = new BatchedJoinBlock<int, int>(99); var terminator = new ActionBlock<Tuple<IList<int>, IList<int>>>(x => { }); batchedJoin.LinkTo(terminator); batchedJoin.Target1.Post(1); batchedJoin.Complete(); localPassed = batchedJoin.Completion.Wait(2000); Assert.True(localPassed, string.Format("BatchedJoinBlock`2 completed through block - {0}", localPassed ? "Passed" : "FAILED")); } // Test BatchedJoinBlock`3 completion through block { var localPassed = true; var batchedJoin = new BatchedJoinBlock<int, int, int>(99); var terminator = new ActionBlock<Tuple<IList<int>, IList<int>, IList<int>>>(x => { }); batchedJoin.LinkTo(terminator); batchedJoin.Target1.Post(1); batchedJoin.Complete(); localPassed = batchedJoin.Completion.Wait(2000); Assert.True(localPassed, string.Format("BatchedJoinBlock`3 completed through block - {0}", localPassed ? "Passed" : "FAILED")); } }
private static bool TestTriggerBatchRacingWithComplete(bool greedy) { bool passed = true; const int batchSize = 2; const int iterations = 1; const int waitTimeout = 100; var dbo = new GroupingDataflowBlockOptions { Greedy = greedy }; for (int iter = 0; iter < iterations; iter++) { bool localPassed = true; var sendAsyncTasks = new Task<bool>[batchSize - 1]; var racerReady = new ManualResetEventSlim(); int[] output1 = null; int[] output2 = null; // Blocks var batch = new BatchBlock<int>(batchSize, dbo); var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; }); batch.LinkTo(terminator); // Queue up batchSize-1 input items for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i); var racer = Task.Factory.StartNew(() => { racerReady.Set(); batch.Complete(); }); // Wait for the racer to get ready and trigger localPassed &= racerReady.Wait(waitTimeout); batch.TriggerBatch(); Assert.True(localPassed, "The racer task FAILED to start."); if (localPassed) { // Wait for the SendAsync tasks to complete localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout); Assert.True(localPassed, "SendAsync tasks FAILED to complete"); } // Do this verification only in greedy mode, because non-greedy is non-deterministic if (greedy) { // Wait for a batch to be produced if (localPassed) { localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout); Assert.True(localPassed, "FAILED to produce a batch"); } if (localPassed) { //Verify the number of input items propagated localPassed &= output1.Length == batchSize - 1; Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}", batchSize, output1.Length)); } } // Wait for the block to complete if (localPassed) { localPassed &= batch.Completion.Wait(waitTimeout); Assert.True(localPassed, "The block FAILED to complete"); } // There should never be a second batch produced if (localPassed) { localPassed &= output2 == null; Assert.True(localPassed, "FAILED not to produce a second batch"); } passed &= localPassed; if (!localPassed) { Assert.True(localPassed, string.Format("Iteration={0}", iter)); Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString())); break; } } return passed; }
private static bool TestTriggerBatchRacingWithSendAsync(bool greedy) { bool passed = true; const int batchSize = 2; const int iterations = 1; const int waitTimeout = 100; var dbo = new GroupingDataflowBlockOptions { Greedy = greedy }; for (int iter = 0; iter < iterations; iter++) { bool localPassed = true; var sendAsyncTasks = new Task<bool>[batchSize - 1]; Task<bool> lastSendAsyncTask = null; var racerReady = new ManualResetEventSlim(); var racerDone = new ManualResetEventSlim(); int[] output1 = null; int[] output2 = null; // Blocks var batch = new BatchBlock<int>(batchSize, dbo); var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; }); batch.LinkTo(terminator); // Queue up batchSize-1 input items for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i); var racer = Task.Factory.StartNew(() => { racerReady.Set(); lastSendAsyncTask = batch.SendAsync(batchSize - 1); racerDone.Set(); }); // Wait for the racer to get ready and trigger localPassed &= (racerReady.Wait(waitTimeout)); batch.TriggerBatch(); Assert.True(localPassed, "The racer task FAILED to start."); // Wait for the SendAsync tasks to complete localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout); Assert.True(localPassed, "SendAsync tasks FAILED to complete"); // Wait for a batch to be produced if (localPassed) { localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout); Assert.True(localPassed, "FAILED to produce a batch"); } if (localPassed && output1.Length < batchSize) { // If the produced batch is not full, we'll trigger one more and count the items. // However, we need to make sure the last message has been offered. Otherwise this // trigger will have no effect. racerDone.Wait(waitTimeout); batch.TriggerBatch(); if (localPassed) { // Wait for the last SendAsync task to complete localPassed &= SpinWait.SpinUntil(() => lastSendAsyncTask != null, waitTimeout); localPassed &= lastSendAsyncTask.Wait(waitTimeout); Assert.True(localPassed, "The last SendAsync task FAILED to complete"); } // Wait for a second batch to be produced if (localPassed) { localPassed &= SpinWait.SpinUntil(() => output2 != null, waitTimeout); Assert.True(localPassed, "FAILED to produce a second batch"); } //Verify the total number of input items propagated if (localPassed) { localPassed &= output1.Length + output2.Length == batchSize; Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}, count2={2}", batchSize, output1.Length, output2.Length)); } } passed &= localPassed; if (!localPassed) { Assert.True(localPassed, string.Format("Iteration={0}", iter)); Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString())); Assert.True(localPassed, string.Format("Count2={0}", output2 == null ? "null" : output2.Length.ToString())); break; } } return passed; }
private static bool TestTriggerBatch(int boundedCapacity) { bool passed = true; // Test greedy with batch size of 1 (force should always be a nop) { bool localPassed = true; const int ITERS = 2; var b = new BatchBlock<int>(1, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int i = 0; i < ITERS; i++) { b.Post(i); int outputCount = b.OutputCount; b.TriggerBatch(); localPassed &= outputCount == b.OutputCount; } localPassed &= b.OutputCount == ITERS; for (int i = 0; i < ITERS; i++) { var arr = b.Receive(); localPassed &= arr.Length == 1 && arr[0] == i; } Assert.True(localPassed, string.Format("greedy with batch size of 1 - {0}", localPassed ? "Passed" : "FAILED")); } // Test greedy with varying batch sizes and smaller queued numbers { bool localPassed = true; foreach (var batchSize in new[] { 3 }) { foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 }) { var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int p = 1; p <= queuedBeforeTrigger; p++) b.Post(p); localPassed &= b.OutputCount == 0; b.TriggerBatch(); b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided localPassed &= b.OutputCount == 1 && b.Receive().Length == queuedBeforeTrigger; for (int j = 0; j < batchSize; j++) { localPassed &= b.OutputCount == 0; b.Post(j); } localPassed &= b.OutputCount == 1; } } Assert.True(localPassed, string.Format("greedy with varying batch sizes - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy with empty queue { bool localPassed = true; foreach (var batchSize in new[] { 1 }) { var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int i = 0; i < 2; i++) b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("greedy with empty queue - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy after decline { bool localPassed = true; { var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); localPassed &= b.OutputCount == 0; b.Complete(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } { var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); localPassed &= b.OutputCount == 0; b.Post(1); localPassed &= b.OutputCount == 0; b.Complete(); localPassed &= b.OutputCount == 1; b.TriggerBatch(); localPassed &= b.OutputCount == 1; } Assert.True(localPassed, string.Format("greedy after decline - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy after canceled { bool localPassed = true; { var cts = new CancellationTokenSource(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(2, dbo); localPassed &= b.OutputCount == 0; cts.Cancel(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } { var cts = new CancellationTokenSource(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(2, dbo); localPassed &= b.OutputCount == 0; b.Post(1); localPassed &= b.OutputCount == 0; cts.Cancel(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("greedy after canceled - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy with MaxNumberOfGroups == 1 { bool localPassed = true; var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1, BoundedCapacity = boundedCapacity }); b.Post(1); localPassed &= b.OutputCount == 0; b.TriggerBatch(); b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided localPassed &= b.OutputCount == 1; localPassed &= b.Post(2) == false; b.TriggerBatch(); localPassed &= b.OutputCount == 1; Assert.True(localPassed, string.Format("greedy with MaxNumberOfGroups == 1 - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test non-greedy with no queued or postponed messages { bool localPassed = true; var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(3, dbo); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; Assert.True(localPassed, string.Format("non-greedy with no mesages - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test non-greedy with no queued but postponed messages { bool localPassed = true; var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity }; const int BATCH_SIZE = 10; for (int numPostponedMessages = 1; numPostponedMessages < BATCH_SIZE; numPostponedMessages++) { var b = new BatchBlock<int>(BATCH_SIZE, dbo); localPassed &= b.OutputCount == 0; for (int i = 0; i < numPostponedMessages; i++) localPassed &= !b.SendAsync(i).IsCompleted; b.TriggerBatch(); var output = b.Receive(); localPassed &= output.Length == numPostponedMessages; for (int i = 0; i < output.Length; i++) localPassed &= output[i] == i; localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("non-greedy with postponed, no queued - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } return passed; }
private static bool TestMaxNumberOfGroups(bool greedy, bool sync) { Contract.Assert(greedy || !sync, "Non-greedy sync doesn't make sense."); bool passed = true; for (int maxNumberOfGroups = 1; maxNumberOfGroups <= 21; maxNumberOfGroups += 20) { for (int itemsPerBatch = 1; itemsPerBatch <= 1; itemsPerBatch++) { var options = new GroupingDataflowBlockOptions { MaxNumberOfGroups = maxNumberOfGroups, Greedy = greedy }; var batch = new BatchBlock<int>(itemsPerBatch, options); // Feed all N batches; all should succeed for (int batchNum = 0; batchNum < maxNumberOfGroups; batchNum++) { var sendAsyncs = new Task<bool>[itemsPerBatch]; for (int itemNum = 0; itemNum < itemsPerBatch; itemNum++) { if (sync) { Assert.True(batch.Post(itemNum), string.Format("FAILED batch.Post({0}) on MaxNOG {1}", itemNum, batchNum)); } else { sendAsyncs[itemNum] = batch.SendAsync(itemNum); } } if (!sync) { Assert.True(Task.WaitAll(sendAsyncs, 4000), string.Format("FAILED batch.SendAsyncs should have been completed in batch num {0}", batchNum)); if (passed) { Assert.True(sendAsyncs.All(t => t.Status == TaskStatus.RanToCompletion && t.Result), string.Format("FAILED batch.SendAsyncs should have been completed in batch num {0}", batchNum)); } } } // Next message should fail in greedy mode if (greedy) { if (sync) { Assert.False(batch.Post(1), "FAILED batch.Post(1) after completed groups should be declind"); } else { var t = batch.SendAsync(1); Assert.True(t != null && t.Status == TaskStatus.RanToCompletion && t.Result == false, "FAILED batch.SendAsync(1) after completed groups should be declined"); } } // Wait until the all batches are produced Assert.True(SpinWait.SpinUntil(() => batch.OutputCount == maxNumberOfGroups, 4000), "FAILED All batches should have been produced"); // Next message should fail, even after groups have been produced if (sync) { Assert.False(batch.Post(1), "FAILED batch.Post(1) after completed groups are output should be declind"); } else { var t = batch.SendAsync(1); Assert.True(t != null && t.Status == TaskStatus.RanToCompletion && t.Result == false, "FAILED batch.SendAsync(1) after completed groups are output should be declined"); } } } Assert.True(passed, string.Format("{0}", passed ? "Passed" : "FAILED")); return passed; }
public void RunBatchBlockConformanceTests() { bool localPassed; // Greedy batching { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 1; var batch = new BatchBlock<int>(BATCH_SIZE); for (int i = 0; i < NUM_MESSAGES * BATCH_SIZE; i++) batch.Post(i); for (int i = 0; i < NUM_MESSAGES; i++) { int[] result = batch.Receive(); localPassed &= result.Length == BATCH_SIZE; for (int j = 0; j < result.Length - 1; j++) { localPassed &= (result[j] + 1 == result[j + 1]); } } Assert.True(localPassed, string.Format("{0}: Greedy batching", localPassed ? "Success" : "Failure")); } // Non-greedy batching with BATCH_SIZE sources used repeatedly { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 1; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) buffer.LinkTo(batch); int prevSum = -1; for (int i = 0; i < NUM_MESSAGES; i++) { for (int j = 0; j < BATCH_SIZE; j++) buffers[j].Post(i); int sum = batch.Receive().Sum(); localPassed &= (sum > prevSum); prevSum = sum; } Assert.True(localPassed, string.Format("{0}: Non-greedy batching with BATCH_SIZE sources used repeatedly", localPassed ? "Success" : "Failure")); } // Non-greedy batching with BATCH_SIZE * NUM_MESSAGES sources { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 2; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE * NUM_MESSAGES).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.Post(1); } for (int i = 0; i < NUM_MESSAGES; i++) { localPassed &= batch.Receive().Sum() == BATCH_SIZE; } Assert.True(localPassed, string.Format("{0}: Non-greedy batching with N*M sources", localPassed ? "Success" : "Failure")); } // Non-greedy batching with missed messages { localPassed = true; const int BATCH_SIZE = 2; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE - 1).Select(_ => new BufferBlock<int>()).ToList(); using (var ce = new CountdownEvent(BATCH_SIZE - 1)) { foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.LinkTo(new ActionBlock<int>(i => ce.Signal())); buffer.Post(42); } ce.Wait(); } buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.Post(42); buffer.Complete(); } localPassed &= Task.WaitAll(buffers.Select(b => b.Completion).ToArray(), 2000); Assert.True(localPassed, string.Format("{0}: Non-greedy batching with missed messages", localPassed ? "Success" : "Failure")); } // Test using a precanceled token { localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 }; var b = new BatchBlock<int>(42, dbo); int[] ignoredValue; IList<int[]> ignoredValues; localPassed &= b.BatchSize == 42; localPassed &= b.LinkTo(new ActionBlock<int[]>(delegate { })) != null; localPassed &= b.SendAsync(42).Result == false; localPassed &= b.TryReceiveAll(out ignoredValues) == false; localPassed &= b.Post(42) == false; localPassed &= b.OutputCount == 0; localPassed &= b.TryReceive(out ignoredValue) == false; localPassed &= b.Completion != null; b.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure")); } // Test completing block while still items buffered { localPassed = true; var b = new BatchBlock<int>(5); b.Post(1); b.Post(2); b.Post(3); b.Complete(); localPassed &= b.Receive().Length == 3; Assert.True(localPassed, string.Format("{0}: Makes batches of remaining items", localPassed ? "Success" : "Failure")); } }
/// <summary> /// Extract relative information from dataflow option to a block-level GroupingDataflowBlockOptions /// </summary> public GroupingDataflowBlockOptions ToGroupingBlockOption() { var option = new GroupingDataflowBlockOptions(); if (this.RecommendedCapacity != null) { option.BoundedCapacity = this.RecommendedCapacity.Value; } return option; }
public void RunDataflowBlockOptionsTests() { // Test base DataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().TaskScheduler = null; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = 0; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = 0; }); Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().NameFormat = null; }); } // Test default values { var db = new DataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "TaskScheduler should be Default"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Max messages should be unbounded."); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Bounded capacity should be unbounded."); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "The cancellation token should be None."); Assert.True(DataflowBlockOptions.Unbounded == -1, "Unbounded should be the value -1"); Assert.True(db.NameFormat == @"{0} Id={1}", @"NameFormat should be the value '{0} Id={1}'"); } // Test that set values are retrievable { var db = new DataflowBlockOptions(); db.MaxMessagesPerTask = 2; Assert.True(db.MaxMessagesPerTask == 2, "Expected max messages to be the set value 2"); db.MaxMessagesPerTask = Int32.MaxValue; Assert.True(db.MaxMessagesPerTask == Int32.MaxValue, "Expected max messages to be the set value Int32.MaxValue"); db.MaxMessagesPerTask = DataflowBlockOptions.Unbounded; Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to be unbounded."); db.BoundedCapacity = 2; Assert.True(db.BoundedCapacity == 2, "Expected bounded capacity to be the set value 2"); db.BoundedCapacity = Int32.MaxValue; Assert.True(db.BoundedCapacity == Int32.MaxValue, "Expected bounded capacity to be the set value Int32.MaxValue"); db.BoundedCapacity = DataflowBlockOptions.Unbounded; Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to be unbounded."); var dummyScheduler = new DummyScheduler(); db.TaskScheduler = dummyScheduler; Assert.True(db.TaskScheduler == dummyScheduler, "Expected task scheduler to be the dummy scheduler"); db.TaskScheduler = TaskScheduler.Default; Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to be the default scheduler"); var cts = new CancellationTokenSource(); db.CancellationToken = cts.Token; Assert.True(db.CancellationToken == cts.Token, "Expected the token to be the one just set"); db.CancellationToken = CancellationToken.None; Assert.True(db.CancellationToken == CancellationToken.None, "Expected the token to be none"); db.NameFormat = "none"; Assert.True(db.NameFormat.Equals("none"), "Expected name format to be the set value 'none'"); db.NameFormat = "foo {0}"; Assert.True(db.NameFormat.Equals("foo {0}"), @"Expected name format to be the set value 'foo {0}'"); db.NameFormat = "foo {0} bar {1}"; Assert.True(db.NameFormat.Equals("foo {0} bar {1}"), @"Expected name format to be the set value 'foo {0} bar {1}'"); db.NameFormat = "kaboom {0} {1} {2}"; Assert.True(db.NameFormat.Equals("kaboom {0} {1} {2}"), @"Expected name format to be the set value 'kaboom {0} {1} {2}'"); } } // Test base ExecutionDataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = 0; }); } // Test default values { var db = new ExecutionDataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value"); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value"); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value"); Assert.True(db.MaxDegreeOfParallelism == 1, "Expected max dop to have default value"); } // Test that set values are retrievable { var db = new ExecutionDataflowBlockOptions(); db.MaxDegreeOfParallelism = 2; Assert.True(db.MaxDegreeOfParallelism == 2, "Expected max dop to be 2"); db.MaxDegreeOfParallelism = Int32.MaxValue; Assert.True(db.MaxDegreeOfParallelism == Int32.MaxValue, "Expected max dop to be Int32.MaxValue"); db.MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded; Assert.True(db.MaxDegreeOfParallelism == DataflowBlockOptions.Unbounded, "Expected max dop to be unbounded"); } } // Test base GroupingDataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = 0; }); } // Test default values { var db = new GroupingDataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value"); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value"); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value"); Assert.True(db.MaxNumberOfGroups == DataflowBlockOptions.Unbounded, "Expected max groups to have default value"); Assert.True(db.Greedy == true, "Expected greedy to have default value"); } // Test that set values are retrievable { var db = new GroupingDataflowBlockOptions(); db.MaxNumberOfGroups = 2; Assert.True(db.MaxNumberOfGroups == 2, "Expected max groups to be 2"); db.MaxNumberOfGroups = Int32.MaxValue; Assert.True(db.MaxNumberOfGroups == Int32.MaxValue, "Expected max groups to be Int32.MaxValue"); db.MaxNumberOfGroups = Int64.MaxValue; Assert.True(db.MaxNumberOfGroups == Int64.MaxValue, "Expected max groups to be Int64.MaxValue"); db.MaxNumberOfGroups = DataflowBlockOptions.Unbounded; Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max groups to unbounded"); db.Greedy = true; Assert.True(db.Greedy == true, "Expected greedy to be true"); db.Greedy = false; Assert.True(db.Greedy == false, "Expected greedy to be false"); db.Greedy = true; Assert.True(db.Greedy == true, "Expected greedy to be true"); } } }