public void ProduceLogs(int count, int buffSize) { var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize }; var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true }; LogGenerator g = new LogGenerator(); var file = new StreamWriter("basic.async.buff.log", false); BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions); ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions); buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true }); for (int i = 0; i < count; i++) { g.Next(); var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6); writer.SendAsync(line).Wait(); } buffer.Complete(); Completed = writer.Completion.ContinueWith(t => file.Close()); }
protected OutgoingQueueBase( CompletionHelper compHelper, Func <bool> externalCompleteTester, Action <int> decreaseItemsCount, DataflowBlockOptions options) { IsProcessing = new AtomicBoolean(); Store = new ConcurrentQueue <T> (); Outgoing = new BlockingCollection <T> (Store); this.compHelper = compHelper; this.externalCompleteTester = externalCompleteTester; this.options = options; this.decreaseItemsCount = decreaseItemsCount; }
public BufferBlock(DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.dataflowBlockOptions = dataflowBlockOptions; this.messageBox = new PassingMessageBox <T> (messageQueue, compHelper, () => outgoing.IsCompleted, ProcessQueue, dataflowBlockOptions); this.outgoing = new MessageOutgoingQueue <T> (compHelper, () => messageQueue.IsCompleted); this.vault = new MessageVault <T> (); }
protected MessageBox( ITargetBlock <TInput> target, BlockingCollection <TInput> messageQueue, CompletionHelper compHelper, Func <bool> externalCompleteTester, DataflowBlockOptions options, bool greedy = true, Func <bool> canAccept = null) { this.Target = target; this.CompHelper = compHelper; this.MessageQueue = messageQueue; this.externalCompleteTester = externalCompleteTester; this.options = options; this.greedy = greedy; this.canAccept = canAccept; }
public BatchBlock(int batchSize, DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.batchSize = batchSize; this.dataflowBlockOptions = dataflowBlockOptions; this.messageBox = new PassingMessageBox <T> (messageQueue, compHelper, () => outgoing.IsCompleted, BatchProcess, dataflowBlockOptions); this.outgoing = new MessageOutgoingQueue <T[]> (compHelper, () => messageQueue.IsCompleted); this.vault = new MessageVault <T[]> (); }
/// <summary> /// 初期化処理。 /// </summary> /// <param name="settingObject"></param> /// <param name="token"></param> public void Init(dynamic settingObject, CancellationToken token) { logger.Trace("Init Start"); var opt = new DataflowBlockOptions { CancellationToken = token, }; var buffer = new BufferBlock<PastaLog>(opt); var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt); buffer.LinkTo(bloadcast); Target = buffer; Source = bloadcast; logger.Trace("Init End"); }
public BufferBlock(DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.dataflowBlockOptions = dataflowBlockOptions; this.compHelper = new CompletionHelper(dataflowBlockOptions); this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper, () => outgoing.IsCompleted, _ => ProcessQueue(), dataflowBlockOptions); this.outgoing = new OutgoingQueue <T> (this, compHelper, () => messageQueue.IsCompleted, messageBox.DecreaseCount, dataflowBlockOptions); }
/// <summary>Initializes the <see cref="WriteOnceBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary> /// <param name="cloningFunction"> /// The function to use to clone the data when offered to other blocks. /// This may be null to indicate that no cloning need be performed. /// </param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="WriteOnceBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public WriteOnceBlock(Func <T, T> cloningFunction, DataflowBlockOptions dataflowBlockOptions) { // Validate arguments if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } Contract.EndContractBlock(); // Store the option _cloningFunction = cloningFunction; _dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // The target registry also serves as our ValueLock, // and thus must always be initialized, even if the block is pre-canceled, as // subsequent usage of the block may run through code paths that try to take this lock. _targetRegistry = new TargetRegistry <T>(this); // If a cancelable CancellationToken has been passed in, // we need to initialize the completion task's TCS now. if (dataflowBlockOptions.CancellationToken.CanBeCanceled) { _lazyCompletionTaskSource = new TaskCompletionSource <VoidResult>(); // If we've already had cancellation requested, do as little work as we have to // in order to be done. if (dataflowBlockOptions.CancellationToken.IsCancellationRequested) { _completionReserved = _decliningPermanently = true; // Cancel the completion task's TCS _lazyCompletionTaskSource.SetCanceled(); } else { // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, _lazyCompletionTaskSource.Task, state => ((WriteOnceBlock <T>)state).Complete(), this); } } #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
/// <summary>Initializes the <see cref="BufferBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BufferBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public BufferBlock(DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } Contract.EndContractBlock(); // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize bounding state if necessary Action <ISourceBlock <T>, int> onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((BufferBlock <T>)owningSource).OnItemsRemoved(count); _boundingState = new BoundingStateWithPostponedAndTask <T>(dataflowBlockOptions.BoundedCapacity); } // Initialize the source state _source = new SourceCore <T>(this, dataflowBlockOptions, owningSource => ((BufferBlock <T>)owningSource).Complete(), onItemsRemoved); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. _source.Completion.ContinueWith((completed, state) => { var thisBlock = ((BufferBlock <T>)state) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, _source.Completion, owningSource => ((BufferBlock <T>)owningSource).Complete(), this); #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
public BroadcastBlock(Func <T, T> cloningFunction, DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.cloningFunction = cloningFunction; this.dataflowBlockOptions = dataflowBlockOptions; this.compHelper = new CompletionHelper(dataflowBlockOptions); this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper, () => outgoing.IsCompleted, _ => BroadcastProcess(), dataflowBlockOptions); this.outgoing = new BroadcastOutgoingQueue <T> (this, compHelper, () => messageQueue.IsCompleted, messageBox.DecreaseCount, dataflowBlockOptions, cloningFunction != null); }
public WriteOnceBlock(Func <T, T> cloningFunction, DataflowBlockOptions dataflowBlockOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } this.cloningFunction = cloningFunction; this.dataflowBlockOptions = dataflowBlockOptions; this.compHelper = CompletionHelper.GetNew(dataflowBlockOptions); this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper, () => true, _ => BroadcastProcess(), dataflowBlockOptions, canAccept: () => written.TrySet()); this.outgoing = new BroadcastOutgoingQueue <T> (this, compHelper, () => messageQueue.IsCompleted, messageBox.DecreaseCount, dataflowBlockOptions, cloningFunction != null); }
/// <summary> /// 初期化処理。 /// </summary> /// <param name="settingObject"></param> /// <param name="token"></param> /// <returns></returns> public void Init(dynamic settingObject, CancellationToken token) { logger.Trace("Init Start"); token.Register(Dispose); var opt = new DataflowBlockOptions { CancellationToken = token, }; var buffer = new BufferBlock<PastaLog>(opt); var actOpt = new ExecutionDataflowBlockOptions() { CancellationToken = token, SingleProducerConstrained = true, }; var act = new ActionBlock<PastaLog>((a) => Save(a), actOpt); buffer.LinkTo(act); Target = buffer; logger.Trace("Init End"); }
/// <summary> /// Creates a new instance of ChannelBase /// </summary> /// <param name="transport"></param> /// <param name="sendTimeout"></param> /// <param name="buffersLimit"></param> /// <param name="fillEnvelopeRecipients">Indicates if the from and to properties of sent and received envelopes should be filled with the session information if not defined.</param> /// <param name="autoReplyPings">Indicates if the channel should reply automatically to ping request commands. In this case, the ping command are not returned by the ReceiveCommandAsync method.</param> public ChannelBase(ITransport transport, TimeSpan sendTimeout, int buffersLimit, bool fillEnvelopeRecipients, bool autoReplyPings) { if (transport == null) { throw new ArgumentNullException("transport"); } this.Transport = transport; this.Transport.Closing += Transport_Closing; _sendTimeout = sendTimeout; _fillEnvelopeRecipients = fillEnvelopeRecipients; _autoReplyPings = autoReplyPings; _channelCancellationTokenSource = new CancellationTokenSource(); this.State = SessionState.New; #if MONO _messageBuffer = new AsyncQueue <Message> (buffersLimit, buffersLimit); _commandBuffer = new AsyncQueue <Command> (buffersLimit, buffersLimit); _notificationBuffer = new AsyncQueue <Notification> (buffersLimit, buffersLimit); _sessionBuffer = new AsyncQueue <Session> (1, 1); #else var bufferOptions = new System.Threading.Tasks.Dataflow.DataflowBlockOptions() { BoundedCapacity = buffersLimit }; _messageBuffer = new BufferBlock <Message>(bufferOptions); _commandBuffer = new BufferBlock <Command>(bufferOptions); _notificationBuffer = new BufferBlock <Notification>(bufferOptions); _sessionBuffer = new BufferBlock <Session>( new DataflowBlockOptions() { BoundedCapacity = 1 }); #endif }
public static Task <int> Choose <T1, T2> (ISourceBlock <T1> source1, Action <T1> action1, ISourceBlock <T2> source2, Action <T2> action2, DataflowBlockOptions dataflowBlockOptions) { if (source1 == null) { throw new ArgumentNullException("source1"); } if (source2 == null) { throw new ArgumentNullException("source2"); } var chooser = new ChooserBlock <T1, T2, object> (action1, action2, null, dataflowBlockOptions); source1.LinkTo(chooser.Target1); source2.LinkTo(chooser.Target2); return(chooser.Completion); }
/// <summary>Initializes the <see cref="BufferBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BufferBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public BufferBlock(DataflowBlockOptions dataflowBlockOptions !!) {
public void RunWriteOnceBlockConformanceTests() { bool passed = true, localPassed = true; { // Test posting then receiving localPassed = true; var wob = new WriteOnceBlock<int>(i => i); int successfulPosts = 0; for (int i = 10; i <= 20; i++) { successfulPosts += wob.Post(i) ? 1 : 0; } localPassed |= successfulPosts == 1; localPassed |= wob.Receive() == 10; Console.WriteLine("{0}: Posting then receiving", localPassed ? "Success" : "Failure"); passed &= localPassed; } { // Test receiving then posting localPassed = true; var wob = new WriteOnceBlock<int>(i => i); Task.Factory.StartNew(() => { Task.Delay(1000).Wait(); wob.Post(42); }); localPassed |= wob.Receive() == 42; localPassed |= wob.Post(43) == false; wob.Completion.Wait(); Console.WriteLine("{0}: Receiving then posting", localPassed ? "Success" : "Failure"); passed &= localPassed; } { // Test broadcasting localPassed = true; var wob = new WriteOnceBlock<int>(i => i + 1); var tb1 = new TransformBlock<int, int>(i => i); var tb2 = new TransformBlock<int, int>(i => i); var tb3 = new TransformBlock<int, int>(i => i); wob.LinkTo(tb1); wob.LinkTo(tb2); wob.LinkTo(tb3); wob.Post(42); localPassed |= tb1.Receive() == 43; localPassed |= tb2.Receive() == 43; localPassed |= tb3.Receive() == 43; Console.WriteLine("{0}: Broadcasting", localPassed ? "Success" : "Failure"); passed &= localPassed; } { // Test using a precanceled token localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new DataflowBlockOptions { CancellationToken = cts.Token }; var wob = new WriteOnceBlock<int>(i => i, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= wob.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= wob.SendAsync(42).Result == false; localPassed &= ((IReceivableSourceBlock<int>)wob).TryReceiveAll(out ignoredValues) == false; localPassed &= wob.Post(42) == false; localPassed &= wob.TryReceive(out ignoredValue) == false; localPassed &= wob.Completion != null; wob.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } { // Test using token canceled after construction localPassed = true; try { var cts = new CancellationTokenSource(); var dbo = new DataflowBlockOptions { CancellationToken = cts.Token }; var wob = new WriteOnceBlock<int>(i => i, dbo); cts.Cancel(); int ignoredValue; IList<int> ignoredValues; localPassed &= wob.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= wob.SendAsync(42).Result == false; localPassed &= ((IReceivableSourceBlock<int>)wob).TryReceiveAll(out ignoredValues) == false; localPassed &= wob.Post(42) == false; localPassed &= wob.TryReceive(out ignoredValue) == false; localPassed &= wob.Completion != null; wob.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } Assert.True(passed, "Test failed."); }
public void TestCanceledLinking() { bool passed = true; var cts = new CancellationTokenSource(); cts.Cancel(); var options = new DataflowBlockOptions { CancellationToken = cts.Token }; var writeOnce = new WriteOnceBlock<int>(x => x, options); var target = new ActionBlock<int>(x => { }); try { writeOnce.LinkTo(target); Console.WriteLine("Completed without exception - Passed"); } catch (Exception) { passed = false; Console.WriteLine("Completed without exception - FAILED"); } Assert.True(passed, "Test failed."); }
//[Fact(Skip = "Outerloop")] public void RunBroadcastBlockConformanceTests() { bool localPassed = true; { // Test posting then receiving localPassed = true; var bb = new BroadcastBlock<int>(i => i); for (int i = 0; i < 2; i++) bb.Post(i); Task.Delay(1).Wait(); localPassed |= bb.Receive() == 1; Assert.True(localPassed, string.Format("{0}: Posting then receiving", localPassed ? "Success" : "Failure")); } { // Test receiving then posting localPassed = true; var bb = new BroadcastBlock<int>(i => i); Task.Factory.StartNew(() => { Task.Delay(1).Wait(); bb.Post(42); }); localPassed |= bb.Receive() == 42; Assert.True(localPassed, string.Format("{0}: Receiving then posting", localPassed ? "Success" : "Failure")); } { // Test broadcasting localPassed = true; var bb = new BroadcastBlock<int>(i => i + 1); var tb1 = new TransformBlock<int, int>(i => i); var tb2 = new TransformBlock<int, int>(i => i); var tb3 = new TransformBlock<int, int>(i => i); bb.LinkTo(tb1); bb.LinkTo(tb2); bb.LinkTo(tb3); for (int i = 0; i < 2; i++) { bb.Post(i); } for (int i = 0; i < 2; i++) { localPassed |= tb1.Receive() == i + 1; localPassed |= tb2.Receive() == i + 1; localPassed |= tb3.Receive() == i + 1; } Assert.True(localPassed, string.Format("{0}: Broadcasting", localPassed ? "Success" : "Failure")); } // Test using a precanceled token { localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new DataflowBlockOptions { CancellationToken = cts.Token }; var bb = new BroadcastBlock<int>(i => i, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= bb.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= bb.SendAsync(42).Result == false; localPassed &= ((IReceivableSourceBlock<int>)bb).TryReceiveAll(out ignoredValues) == false; localPassed &= bb.Post(42) == false; localPassed &= bb.TryReceive(out ignoredValue) == false; localPassed &= bb.Completion != null; bb.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure")); } }
/// <summary> /// Builds the internal buffers using the provided options. /// </summary> /// <param name="options"></param> public PriorityBufferBlock(DataflowBlockOptions options) : this(options, options, options) { }
/// <summary> /// Each internal BufferBlock is built using the provided options specified for that priority. /// </summary> /// <param name="highPriorityOptions"></param> /// <param name="mediumPriorityOptions"></param> /// <param name="lowPriorityOptions"></param> public PriorityBufferBlock(DataflowBlockOptions highPriorityOptions, DataflowBlockOptions mediumPriorityOptions, DataflowBlockOptions lowPriorityOptions) { _highPriorityBuffer = new BufferBlock <T>(highPriorityOptions); _mediumPriorityBuffer = new BufferBlock <T>(mediumPriorityOptions); _lowPriorityBuffer = new BufferBlock <T>(lowPriorityOptions); }
public static CompletionHelper GetNew(DataflowBlockOptions options) { return(new CompletionHelper(options)); }
/// <summary>Initializes the <see cref="WriteOnceBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary> /// <param name="cloningFunction"> /// The function to use to clone the data when offered to other blocks. /// This may be null to indicate that no cloning need be performed. /// </param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="WriteOnceBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public WriteOnceBlock(Func <T, T>?cloningFunction, DataflowBlockOptions dataflowBlockOptions !!) {
public void RunBufferBlockConformanceTests() { bool localPassed; // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> bufferBlockFactory = options => { BufferBlock<int> bufferBlock = new BufferBlock<int>(options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); bufferBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = bufferBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; Assert.True(FeedTarget(bufferBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true)); Assert.True(FeedTarget(bufferBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true)); Assert.True(FeedTarget(bufferBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true)); Assert.True(FeedTarget(bufferBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true)); Assert.True(FeedTarget(bufferBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true)); Assert.True(FeedTarget(bufferBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true)); } // Test chained Post/Receive { localPassed = true; const int ITERS = 2; var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>()); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i); } Assert.True(localPassed, string.Format("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure")); } // Test chained SendAsync/Receive { localPassed = true; const int ITERS = 2; var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>()); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i); } Assert.True(localPassed, string.Format("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure")); } // Test chained Post all then Receive { localPassed = true; const int ITERS = 2; var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>()); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i; Assert.True(localPassed, string.Format("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure")); } // Test chained SendAsync all then Receive { localPassed = true; const int ITERS = 2; var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>()); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2)); Assert.True(localPassed, string.Format("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure")); } // Test using a precanceled token { localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new DataflowBlockOptions { CancellationToken = cts.Token }; var bb = new BufferBlock<int>(dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= bb.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= bb.SendAsync(42).Result == false; localPassed &= bb.TryReceiveAll(out ignoredValues) == false; localPassed &= bb.Post(42) == false; localPassed &= bb.Count == 0; localPassed &= bb.TryReceive(out ignoredValue) == false; localPassed &= bb.Completion != null; bb.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, string.Format(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure")); } }
public void CancellationTest () { var source1 = new BufferBlock<int> (); var source2 = new BufferBlock<long> (); var tokenSource = new CancellationTokenSource (); var options = new DataflowBlockOptions { CancellationToken = tokenSource.Token }; var completion = DataflowBlock.Choose ( source1, _ => { }, source2, _ => { }, options); Assert.IsFalse (completion.IsCanceled); tokenSource.Cancel (); Thread.Sleep (100); Assert.IsTrue (completion.IsCanceled); }
public void RunDataflowBlockOptionsTests() { // Test base DataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().TaskScheduler = null; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = 0; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = 0; }); Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().NameFormat = null; }); } // Test default values { var db = new DataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "TaskScheduler should be Default"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Max messages should be unbounded."); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Bounded capacity should be unbounded."); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "The cancellation token should be None."); Assert.True(DataflowBlockOptions.Unbounded == -1, "Unbounded should be the value -1"); Assert.True(db.NameFormat == @"{0} Id={1}", @"NameFormat should be the value '{0} Id={1}'"); } // Test that set values are retrievable { var db = new DataflowBlockOptions(); db.MaxMessagesPerTask = 2; Assert.True(db.MaxMessagesPerTask == 2, "Expected max messages to be the set value 2"); db.MaxMessagesPerTask = Int32.MaxValue; Assert.True(db.MaxMessagesPerTask == Int32.MaxValue, "Expected max messages to be the set value Int32.MaxValue"); db.MaxMessagesPerTask = DataflowBlockOptions.Unbounded; Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to be unbounded."); db.BoundedCapacity = 2; Assert.True(db.BoundedCapacity == 2, "Expected bounded capacity to be the set value 2"); db.BoundedCapacity = Int32.MaxValue; Assert.True(db.BoundedCapacity == Int32.MaxValue, "Expected bounded capacity to be the set value Int32.MaxValue"); db.BoundedCapacity = DataflowBlockOptions.Unbounded; Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to be unbounded."); var dummyScheduler = new DummyScheduler(); db.TaskScheduler = dummyScheduler; Assert.True(db.TaskScheduler == dummyScheduler, "Expected task scheduler to be the dummy scheduler"); db.TaskScheduler = TaskScheduler.Default; Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to be the default scheduler"); var cts = new CancellationTokenSource(); db.CancellationToken = cts.Token; Assert.True(db.CancellationToken == cts.Token, "Expected the token to be the one just set"); db.CancellationToken = CancellationToken.None; Assert.True(db.CancellationToken == CancellationToken.None, "Expected the token to be none"); db.NameFormat = "none"; Assert.True(db.NameFormat.Equals("none"), "Expected name format to be the set value 'none'"); db.NameFormat = "foo {0}"; Assert.True(db.NameFormat.Equals("foo {0}"), @"Expected name format to be the set value 'foo {0}'"); db.NameFormat = "foo {0} bar {1}"; Assert.True(db.NameFormat.Equals("foo {0} bar {1}"), @"Expected name format to be the set value 'foo {0} bar {1}'"); db.NameFormat = "kaboom {0} {1} {2}"; Assert.True(db.NameFormat.Equals("kaboom {0} {1} {2}"), @"Expected name format to be the set value 'kaboom {0} {1} {2}'"); } } // Test base ExecutionDataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = 0; }); } // Test default values { var db = new ExecutionDataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value"); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value"); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value"); Assert.True(db.MaxDegreeOfParallelism == 1, "Expected max dop to have default value"); } // Test that set values are retrievable { var db = new ExecutionDataflowBlockOptions(); db.MaxDegreeOfParallelism = 2; Assert.True(db.MaxDegreeOfParallelism == 2, "Expected max dop to be 2"); db.MaxDegreeOfParallelism = Int32.MaxValue; Assert.True(db.MaxDegreeOfParallelism == Int32.MaxValue, "Expected max dop to be Int32.MaxValue"); db.MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded; Assert.True(db.MaxDegreeOfParallelism == DataflowBlockOptions.Unbounded, "Expected max dop to be unbounded"); } } // Test base GroupingDataflowBlockOptions { // Test invalid property values { Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = -2; }); Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = 0; }); } // Test default values { var db = new GroupingDataflowBlockOptions(); Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value"); Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value"); Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value"); Assert.True( !db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value"); Assert.True(db.MaxNumberOfGroups == DataflowBlockOptions.Unbounded, "Expected max groups to have default value"); Assert.True(db.Greedy == true, "Expected greedy to have default value"); } // Test that set values are retrievable { var db = new GroupingDataflowBlockOptions(); db.MaxNumberOfGroups = 2; Assert.True(db.MaxNumberOfGroups == 2, "Expected max groups to be 2"); db.MaxNumberOfGroups = Int32.MaxValue; Assert.True(db.MaxNumberOfGroups == Int32.MaxValue, "Expected max groups to be Int32.MaxValue"); db.MaxNumberOfGroups = Int64.MaxValue; Assert.True(db.MaxNumberOfGroups == Int64.MaxValue, "Expected max groups to be Int64.MaxValue"); db.MaxNumberOfGroups = DataflowBlockOptions.Unbounded; Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max groups to unbounded"); db.Greedy = true; Assert.True(db.Greedy == true, "Expected greedy to be true"); db.Greedy = false; Assert.True(db.Greedy == false, "Expected greedy to be false"); db.Greedy = true; Assert.True(db.Greedy == true, "Expected greedy to be true"); } } }
public static CompletionHelper GetNew (DataflowBlockOptions options) { return new CompletionHelper (options); }
public CompletionHelper (DataflowBlockOptions options) { if (options != null && options.CancellationToken != CancellationToken.None) options.CancellationToken.Register (RequestCancel); }
public void TestBufferBlockCompletionOrder() { const int ITERATIONS = 1000; for (int iter = 0; iter < ITERATIONS; iter++) { var cts = new CancellationTokenSource(); var options = new DataflowBlockOptions() { CancellationToken = cts.Token }; var buffer = new BufferBlock<int>(options); buffer.Post(1); cts.Cancel(); try { buffer.Completion.Wait(); } catch { } Assert.False(buffer.Count != 0, string.Format("Iteration {0}: Completed before clearing messages.", iter)); } }
public Task Execute(CancellationToken token) { var options = new DataflowBlockOptions {CancellationToken = token}; _buffer = new BufferBlock<long>(options); var hydrate = new TransformBlock<long, Summoner>(id => { var summoner = _lookup.Hydrate(id); return summoner; }, new ExecutionDataflowBlockOptions { CancellationToken = token, MaxDegreeOfParallelism = 2 }); var store = new TransformBlock<Summoner, Summoner>(summoner => { if (summoner != null) _storage.Store(summoner); return summoner; }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); var crawl = new TransformManyBlock<Summoner, FellowPlayerInfo>(async summoner => { var summoners = new List<FellowPlayerInfo>(); var games = new List<PlayerGameStats>(); if (summoner != null) { await _crawler.Crawl(summoner, summoners.Add, games.Add); } return summoners; }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); var storeNextBatch = new ActionBlock<FellowPlayerInfo>(async info => { if (info != null) { var data = await _lookup.Lookup(info.summonerId); _storage.StoreWhenMissing(data); } }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); _buffer.LinkTo(hydrate, new DataflowLinkOptions {PropagateCompletion = true}); hydrate.LinkTo(store, new DataflowLinkOptions {PropagateCompletion = true}); store.LinkTo(crawl, new DataflowLinkOptions {PropagateCompletion = true}); crawl.LinkTo(storeNextBatch, new DataflowLinkOptions {PropagateCompletion = true}); return Task.Run(async () => { while (!token.IsCancellationRequested) { try { var batch = _producer.Produce((int) TimeSpan.FromDays(1).TotalMinutes, 30); foreach (var id in batch) await _buffer.SendAsync(id, token); // Start the chain _buffer.Complete(); // Wait until the chain is complete before iterating again await storeNextBatch.Completion; } catch (Exception ex) { throw; } } }, token); }
/// <summary> /// Create a logging thread to process the logging queue /// </summary> private void CreateLoggingEventQueue() { // We are creating a two-node dataflow graph here. The first node is a buffer, which will hold up to the number of // logging events we have specified as the queueCapacity. The second node is the processor which will actually process each message. // When the capacity of the buffer is reached, further attempts to send messages to it will block. // The reason we can't just set the BoundedCapacity on the processing block is that ActionBlock has some weird behavior // when the queue capacity is reached. Specifically, it will block new messages from being processed until it has // entirely drained its input queue, as opposed to letting new ones in as old ones are processed. This is logged as // a perf bug (305575) against Dataflow. If they choose to fix it, we can eliminate the buffer node from the graph. var dataBlockOptions = new DataflowBlockOptions { BoundedCapacity = Convert.ToInt32(_queueCapacity) }; _loggingQueue = new BufferBlock<object>(dataBlockOptions); var executionDataBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }; _loggingQueueProcessor = new ActionBlock<object>(loggingEvent => LoggingEventProcessor(loggingEvent), executionDataBlockOptions); var dataLinkOptions = new DataflowLinkOptions { PropagateCompletion = true }; _loggingQueue.LinkTo(_loggingQueueProcessor, dataLinkOptions); }
public void ProduceLogs(int count, int buffSize) { var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize, MaxMessagesPerTask = 10 }; var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10 }; var serializerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 80, MaxDegreeOfParallelism = 8, SingleProducerConstrained = true, MaxMessagesPerTask = 10 }; LogGenerator g = new LogGenerator(); var file = new StreamWriter("basic.async.srlz.buff.log", false); BufferBlock<LogEntry> buffer = new BufferBlock<LogEntry>(bufferOptions); TransformBlock<LogEntry, string> serializer = new TransformBlock<LogEntry, string>( e => string.Format(e.format, e.parameters), serializerOptions); ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions); buffer.LinkTo(serializer, new DataflowLinkOptions() { PropagateCompletion = true }); serializer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true }); for (int i = 0; i < count; i++) { g.Next(); var entry = new LogEntry() { format = g.FormatStr, parameters = new object[] { g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6 } }; buffer.SendAsync(entry).Wait(); } buffer.Complete(); Completed = writer.Completion.ContinueWith(t => file.Close()); }
public ChooserBlock(Action <T1> action1, Action <T2> action2, Action <T3> action3, DataflowBlockOptions dataflowBlockOptions) { // TODO: take care of options and its cancellation token Target1 = new ChooseTarget <T1> (message => MessageArrived(0, action1, message)); Target2 = new ChooseTarget <T2> (message => MessageArrived(1, action2, message)); Target3 = new ChooseTarget <T3> (message => MessageArrived(2, action3, message)); }